From pypy.commits at gmail.com Sun Apr 1 01:46:51 2018 From: pypy.commits at gmail.com (mattip) Date: Sat, 31 Mar 2018 22:46:51 -0700 (PDT) Subject: [pypy-commit] pypy jit-hooks-can-be-disabled: close merged branch Message-ID: <5ac0724b.0ea6df0a.a5146.6de9@mx.google.com> Author: Matti Picus Branch: jit-hooks-can-be-disabled Changeset: r94206:bc36eabcc716 Date: 2018-04-01 08:43 +0300 http://bitbucket.org/pypy/pypy/changeset/bc36eabcc716/ Log: close merged branch diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py --- a/pypy/module/cpyext/pyobject.py +++ b/pypy/module/cpyext/pyobject.py @@ -60,7 +60,12 @@ def _cpyext_attach_pyobj(self, space, py_obj): self._cpy_ref = py_obj - rawrefcount.create_link_pyobj(self, py_obj) + if type(self) is cls: + rawrefcount.create_link_pyobj(self, py_obj) + else: + # optimization of create_link_pyobj() doesn't work for + # interp-level subclasses! + rawrefcount.create_link_pypy(self, py_obj) cls._cpyext_attach_pyobj = _cpyext_attach_pyobj add_direct_pyobj_storage(W_BaseCPyObject) @@ -128,9 +133,6 @@ w_type) raise track_reference(space, obj, w_obj) - if w_type.flag_cpytype: - assert isinstance(w_obj, W_BaseCPyObject) - w_obj._cpy_ref = obj return w_obj typedescr_cache = {} From pypy.commits at gmail.com Sun Apr 1 01:46:53 2018 From: pypy.commits at gmail.com (mattip) Date: Sat, 31 Mar 2018 22:46:53 -0700 (PDT) Subject: [pypy-commit] pypy default: document merged branch Message-ID: <5ac0724d.5b88df0a.be1c9.3333@mx.google.com> Author: Matti Picus Branch: Changeset: r94207:50bc92bdf69a Date: 2018-04-01 08:46 +0300 http://bitbucket.org/pypy/pypy/changeset/50bc92bdf69a/ Log: document merged branch diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -79,3 +79,10 @@ - in rare "trace is too long" situations, the JIT could break behaviour arbitrarily. + +.. branch: jit-hooks-can-be-disabled + +Be more efficient about JIT hooks. Make it possible for the frontend to declare +that jit hooks are currently not enabled at all. in that case, the list of ops +does not have to be created in the case of the on_abort hook (which is +expensive). From pypy.commits at gmail.com Sun Apr 1 01:53:31 2018 From: pypy.commits at gmail.com (mattip) Date: Sat, 31 Mar 2018 22:53:31 -0700 (PDT) Subject: [pypy-commit] pypy default: remove redundant code, this is done in track_reference Message-ID: <5ac073db.08b51c0a.22baf.b5a8@mx.google.com> Author: Matti Picus Branch: Changeset: r94208:eb7ebd3110b0 Date: 2018-04-01 08:52 +0300 http://bitbucket.org/pypy/pypy/changeset/eb7ebd3110b0/ Log: remove redundant code, this is done in track_reference diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py --- a/pypy/module/cpyext/pyobject.py +++ b/pypy/module/cpyext/pyobject.py @@ -128,9 +128,6 @@ w_type) raise track_reference(space, obj, w_obj) - if w_type.flag_cpytype: - assert isinstance(w_obj, W_BaseCPyObject) - w_obj._cpy_ref = obj return w_obj typedescr_cache = {} From pypy.commits at gmail.com Sun Apr 1 05:36:55 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 01 Apr 2018 02:36:55 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-subclass-setattr: add a test to show how GC inappropriately kills connection btwn pyobj and w_obj Message-ID: <5ac0a837.dcb1df0a.2da62.1f52@mx.google.com> Author: Matti Picus Branch: cpyext-subclass-setattr Changeset: r94209:90146fea6553 Date: 2018-04-01 12:35 +0300 http://bitbucket.org/pypy/pypy/changeset/90146fea6553/ Log: add a test to show how GC inappropriately kills connection btwn pyobj and w_obj Passes on CPython, fails on PyPy nightly, crashes in as_pyobj untranslated diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c --- a/pypy/module/cpyext/test/array.c +++ b/pypy/module/cpyext/test/array.c @@ -2492,6 +2492,82 @@ return PyLong_FromLong(obj1->ob_type->tp_dealloc == obj2->ob_type->tp_dealloc); } +static PyObject * +subclass_with_attribute(PyObject *self, PyObject* args) { + /* what happens when we use tp_alloc to create the subclass, then + * assign to the w_obj via python, then get the GC to collect? + * The w_obj should not be collected!! + */ + PyObject * obj, *sub, *attrib, *funcname, *attribname, *collect, *res, *tup; + PyTypeObject * subtype; + int i; + if (!PyArg_ParseTuple(args, "OOOO", &obj, &funcname, &attribname, &collect)) { + return NULL; + } + if (!PyType_Check(obj)) { + PyErr_SetString(PyExc_TypeError, + "expected type object"); + return NULL; + } + subtype = (PyTypeObject*)obj; + sub = subtype->tp_alloc(subtype, 0); + if (!sub) { + return NULL; + } + attrib = PyObject_GetAttr(sub, funcname); + if (!attrib || (attrib == Py_None) ) { + PyErr_SetString(PyExc_ValueError, + "could not find function to call"); + Py_XDECREF(attrib); + Py_DECREF(sub); + return NULL; + } + tup = PyTuple_New(0); + #ifdef PYPY_VERSION + printf("calling addattrib pypylink %lu \n", sub->ob_pypy_link); + #endif + res = PyObject_Call(attrib, tup, NULL); + #ifdef PYPY_VERSION + printf("after addattrib pypylink %lu \n", sub->ob_pypy_link); + #endif + Py_DECREF(attrib); + if (res == NULL) { + Py_DECREF(tup); + Py_DECREF(sub); + return NULL; + } + Py_DECREF(res); + for(i=0; i<10; i++) { + #ifdef PYPY_VERSION + printf("starting loop iteration %d refcnt %lu pypylink %lu \n", i, + sub->ob_refcnt, sub->ob_pypy_link); + #else + printf("starting loop iteration %d refcnt %lu\n", i, sub->ob_refcnt); + #endif + attrib = PyObject_GetAttr(sub, attribname); + if (!attrib || (attrib == Py_None)) { + PyErr_SetString(PyExc_ValueError, + "could not find attrib on object"); + Py_XDECREF(attrib); + Py_DECREF(tup); + Py_DECREF(sub); + return NULL; + } + Py_XDECREF(attrib); + res = PyObject_Call(collect, tup, NULL); + Py_XDECREF(attrib); + if (res == NULL) { + Py_DECREF(tup); + Py_DECREF(sub); + return NULL; + } + Py_DECREF(res); + } + Py_DECREF(tup); + Py_DECREF(sub); + Py_RETURN_NONE; +} + /*********************** Install Module **************************/ static PyMethodDef a_methods[] = { @@ -2502,6 +2578,7 @@ {"create_and_release_buffer", (PyCFunction)create_and_release_buffer, METH_O, NULL}, {"same_dealloc", (PyCFunction)same_dealloc, METH_VARARGS, NULL}, {"getitem", (PyCFunction)getitem, METH_VARARGS, NULL}, + {"subclass_with_attribute", (PyCFunction)subclass_with_attribute, METH_VARARGS, NULL}, {NULL, NULL, 0, NULL} /* Sentinel */ }; diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py --- a/pypy/module/cpyext/test/test_arraymodule.py +++ b/pypy/module/cpyext/test/test_arraymodule.py @@ -186,3 +186,12 @@ # array_subscr does) raises(IndexError, module.getitem, a, -5) + def test_subclass_with_attribute(self): + module = self.import_module(name='array') + class Sub(module.array): + def addattrib(self): + print('called addattrib') + self.attrib = True + import gc + module.subclass_with_attribute(Sub, "addattrib", "attrib", gc.collect) + From pypy.commits at gmail.com Sun Apr 1 06:37:20 2018 From: pypy.commits at gmail.com (cfbolz) Date: Sun, 01 Apr 2018 03:37:20 -0700 (PDT) Subject: [pypy-commit] pypy default: merge pyparser-improvements: Message-ID: <5ac0b660.32c3df0a.92065.9ece@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r94210:045483a22d66 Date: 2018-04-01 12:36 +0200 http://bitbucket.org/pypy/pypy/changeset/045483a22d66/ Log: merge pyparser-improvements: - speed up python parser - create slightly more helpful SyntaxError message (only in very clear and obvious cases) diff --git a/lib-python/2.7/test/test_genexps.py b/lib-python/2.7/test/test_genexps.py --- a/lib-python/2.7/test/test_genexps.py +++ b/lib-python/2.7/test/test_genexps.py @@ -87,7 +87,7 @@ >>> dict(a = i for i in xrange(10)) Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: invalid syntax (expected ')') Verify that parenthesis are required when used as a keyword argument value diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -86,3 +86,8 @@ that jit hooks are currently not enabled at all. in that case, the list of ops does not have to be created in the case of the on_abort hook (which is expensive). + + +.. branch: pyparser-improvements + +Improve speed of Python parser, improve ParseError messages slightly. diff --git a/pypy/interpreter/pyparser/metaparser.py b/pypy/interpreter/pyparser/metaparser.py --- a/pypy/interpreter/pyparser/metaparser.py +++ b/pypy/interpreter/pyparser/metaparser.py @@ -147,8 +147,10 @@ for label, next in state.arcs.iteritems(): arcs.append((self.make_label(gram, label), dfa.index(next))) states.append((arcs, state.is_final)) - gram.dfas.append((states, self.make_first(gram, name))) - assert len(gram.dfas) - 1 == gram.symbol_ids[name] - 256 + symbol_id = gram.symbol_ids[name] + dfa = parser.DFA(symbol_id, states, self.make_first(gram, name)) + gram.dfas.append(dfa) + assert len(gram.dfas) - 1 == symbol_id - 256 gram.start = gram.symbol_ids[self.start_symbol] return gram @@ -162,6 +164,13 @@ else: gram.labels.append(gram.symbol_ids[label]) gram.symbol_to_label[label] = label_index + first = self.first[label] + if len(first) == 1: + first, = first + if not first[0].isupper(): + first = first.strip("\"'") + assert label_index not in gram.token_to_error_string + gram.token_to_error_string[label_index] = first return label_index elif label.isupper(): token_index = gram.TOKENS[label] @@ -183,7 +192,7 @@ else: gram.labels.append(gram.KEYWORD_TOKEN) gram.keyword_ids[value] = label_index - return label_index + result = label_index else: try: token_index = gram.OPERATOR_MAP[value] @@ -194,7 +203,10 @@ else: gram.labels.append(token_index) gram.token_ids[token_index] = label_index - return label_index + result = label_index + assert result not in gram.token_to_error_string + gram.token_to_error_string[result] = value + return result def make_first(self, gram, name): original_firsts = self.first[name] diff --git a/pypy/interpreter/pyparser/parser.py b/pypy/interpreter/pyparser/parser.py --- a/pypy/interpreter/pyparser/parser.py +++ b/pypy/interpreter/pyparser/parser.py @@ -1,6 +1,7 @@ """ A CPython inspired RPython parser. """ +from rpython.rlib.objectmodel import not_rpython class Grammar(object): @@ -16,6 +17,7 @@ self.symbol_names = {} self.symbol_to_label = {} self.keyword_ids = {} + self.token_to_error_string = {} self.dfas = [] self.labels = [0] self.token_ids = {} @@ -41,6 +43,27 @@ pass return True +class DFA(object): + def __init__(self, symbol_id, states, first): + self.symbol_id = symbol_id + self.states = states + self.first = self._first_to_string(first) + + def could_match_token(self, label_index): + pos = label_index >> 3 + bit = 1 << (label_index & 0b111) + return bool(ord(self.first[label_index >> 3]) & bit) + + @staticmethod + @not_rpython + def _first_to_string(first): + l = sorted(first.keys()) + b = bytearray(32) + for label_index in l: + pos = label_index >> 3 + bit = 1 << (label_index & 0b111) + b[pos] |= bit + return str(b) class Node(object): @@ -127,14 +150,17 @@ class Nonterminal(AbstractNonterminal): __slots__ = ("_children", ) - def __init__(self, type, children): + def __init__(self, type, children=None): Node.__init__(self, type) + if children is None: + children = [] self._children = children def __repr__(self): return "Nonterminal(type=%s, children=%r)" % (self.type, self._children) def get_child(self, i): + assert self._children is not None return self._children[i] def num_children(self): @@ -168,7 +194,7 @@ class ParseError(Exception): def __init__(self, msg, token_type, value, lineno, column, line, - expected=-1): + expected=-1, expected_str=None): self.msg = msg self.token_type = token_type self.value = value @@ -176,17 +202,41 @@ self.column = column self.line = line self.expected = expected + self.expected_str = expected_str def __str__(self): return "ParserError(%s, %r)" % (self.token_type, self.value) +class StackEntry(object): + def __init__(self, next, dfa, state): + self.next = next + self.dfa = dfa + self.state = state + self.node = None + + def push(self, dfa, state): + return StackEntry(self, dfa, state) + + def pop(self): + return self.next + + def node_append_child(self, child): + node = self.node + if node is None: + self.node = Nonterminal1(self.dfa.symbol_id, child) + elif isinstance(node, Nonterminal1): + newnode = self.node = Nonterminal( + self.dfa.symbol_id, [node._child, child]) + else: + self.node.append_child(child) + + class Parser(object): def __init__(self, grammar): self.grammar = grammar self.root = None - self.stack = None def prepare(self, start=-1): """Setup the parser for parsing. @@ -196,16 +246,15 @@ if start == -1: start = self.grammar.start self.root = None - current_node = Nonterminal(start, []) - self.stack = [] - self.stack.append((self.grammar.dfas[start - 256], 0, current_node)) + self.stack = StackEntry(None, self.grammar.dfas[start - 256], 0) def add_token(self, token_type, value, lineno, column, line): label_index = self.classify(token_type, value, lineno, column, line) sym_id = 0 # for the annotator while True: - dfa, state_index, node = self.stack[-1] - states, first = dfa + dfa = self.stack.dfa + state_index = self.stack.state + states = dfa.states arcs, is_accepting = states[state_index] for i, next_state in arcs: sym_id = self.grammar.labels[i] @@ -217,16 +266,17 @@ # the stack. while state[1] and not state[0]: self.pop() - if not self.stack: + if self.stack is None: # Parsing is done. return True - dfa, state_index, node = self.stack[-1] - state = dfa[0][state_index] + dfa = self.stack.dfa + state_index = self.stack.state + state = dfa.states[state_index] return False elif sym_id >= 256: sub_node_dfa = self.grammar.dfas[sym_id - 256] # Check if this token can start a child node. - if label_index in sub_node_dfa[1]: + if sub_node_dfa.could_match_token(label_index): self.push(sub_node_dfa, next_state, sym_id, lineno, column) break @@ -235,7 +285,7 @@ # state is accepting, it's invalid input. if is_accepting: self.pop() - if not self.stack: + if self.stack is None: raise ParseError("too much input", token_type, value, lineno, column, line) else: @@ -243,10 +293,13 @@ # error. if len(arcs) == 1: expected = sym_id + expected_str = self.grammar.token_to_error_string.get( + arcs[0][0], None) else: expected = -1 + expected_str = None raise ParseError("bad input", token_type, value, lineno, - column, line, expected) + column, line, expected, expected_str) def classify(self, token_type, value, lineno, column, line): """Find the label for a token.""" @@ -262,26 +315,22 @@ def shift(self, next_state, token_type, value, lineno, column): """Shift a non-terminal and prepare for the next state.""" - dfa, state, node = self.stack[-1] new_node = Terminal(token_type, value, lineno, column) - node.append_child(new_node) - self.stack[-1] = (dfa, next_state, node) + self.stack.node_append_child(new_node) + self.stack.state = next_state def push(self, next_dfa, next_state, node_type, lineno, column): """Push a terminal and adjust the current state.""" - dfa, state, node = self.stack[-1] - new_node = Nonterminal(node_type, []) - self.stack[-1] = (dfa, next_state, node) - self.stack.append((next_dfa, 0, new_node)) + self.stack.state = next_state + self.stack = self.stack.push(next_dfa, 0) def pop(self): """Pop an entry off the stack and make its node a child of the last.""" - dfa, state, node = self.stack.pop() + top = self.stack + self.stack = top.pop() + node = top.node + assert node is not None if self.stack: - # we are now done with node, so we can store it more efficiently if - # it has just one child - if node.num_children() == 1: - node = Nonterminal1(node.type, node.get_child(0)) - self.stack[-1][2].append_child(node) + self.stack.node_append_child(node) else: self.root = node diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py --- a/pypy/interpreter/pyparser/pyparse.py +++ b/pypy/interpreter/pyparser/pyparse.py @@ -132,7 +132,11 @@ w_message = space.str(e.get_w_value(space)) raise error.SyntaxError(space.text_w(w_message)) raise + if enc is not None: + compile_info.encoding = enc + return self._parse(textsrc, compile_info) + def _parse(self, textsrc, compile_info): flags = compile_info.flags # The tokenizer is very picky about how it wants its input. @@ -181,6 +185,9 @@ else: new_err = error.SyntaxError msg = "invalid syntax" + if e.expected_str is not None: + msg += " (expected '%s')" % e.expected_str + raise new_err(msg, e.lineno, e.column, e.line, compile_info.filename) else: @@ -188,6 +195,4 @@ finally: # Avoid hanging onto the tree. self.root = None - if enc is not None: - compile_info.encoding = enc return tree diff --git a/pypy/interpreter/pyparser/test/targetparse.py b/pypy/interpreter/pyparser/test/targetparse.py new file mode 100644 --- /dev/null +++ b/pypy/interpreter/pyparser/test/targetparse.py @@ -0,0 +1,39 @@ +import sys +import os +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) +print ROOT +sys.path.insert(0, str(ROOT)) +import time +from pypy.interpreter.pyparser import pyparse + + + +with file("../../../rpython/rlib/unicodedata/unicodedb_5_2_0.py") as f: + s = f.read() + +class FakeSpace(object): + pass + +fakespace = FakeSpace() + +def bench(title): + a = time.clock() + info = pyparse.CompileInfo("", "exec") + parser = pyparse.PythonParser(fakespace) + tree = parser._parse(s, info) + b = time.clock() + print title, (b-a) + + +def entry_point(argv): + bench("foo") + + return 0 + +# _____ Define and setup target ___ + +def target(*args): + return entry_point, None + +if __name__ == '__main__': + entry_point(sys.argv) diff --git a/pypy/interpreter/pyparser/test/test_metaparser.py b/pypy/interpreter/pyparser/test/test_metaparser.py --- a/pypy/interpreter/pyparser/test/test_metaparser.py +++ b/pypy/interpreter/pyparser/test/test_metaparser.py @@ -34,8 +34,8 @@ assert len(g.dfas) == 1 eval_sym = g.symbol_ids["eval"] assert g.start == eval_sym - states, first = g.dfas[eval_sym - 256] - assert states == [([(1, 1)], False), ([], True)] + dfa = g.dfas[eval_sym - 256] + assert dfa.states == [([(1, 1)], False), ([], True)] assert g.labels[0] == 0 def test_load_python_grammars(self): @@ -51,7 +51,7 @@ def test_items(self): g = self.gram_for("foo: NAME STRING OP '+'") assert len(g.dfas) == 1 - states = g.dfas[g.symbol_ids["foo"] - 256][0] + states = g.dfas[g.symbol_ids["foo"] - 256].states last = states[0][0][0][1] for state in states[1:-1]: assert last < state[0][0][1] diff --git a/pypy/interpreter/pyparser/test/test_parser.py b/pypy/interpreter/pyparser/test/test_parser.py --- a/pypy/interpreter/pyparser/test/test_parser.py +++ b/pypy/interpreter/pyparser/test/test_parser.py @@ -7,6 +7,12 @@ from pypy.interpreter.pyparser.test.test_metaparser import MyGrammar +def test_char_set(): + first = {5: None, 9: None, 100: None, 255:None} + p = parser.DFA(None, None, first) + for i in range(256): + assert p.could_match_token(i) == (i in first) + class SimpleParser(parser.Parser): def parse(self, input): @@ -55,8 +61,7 @@ n = parser.Terminal(tp, value, 0, 0) else: tp = gram.symbol_ids[data[0]] - children = [] - n = parser.Nonterminal(tp, children) + n = parser.Nonterminal(tp) new_indent = count_indent(line) if new_indent >= last_indent: if new_indent == last_indent and node_stack: @@ -291,3 +296,37 @@ NEWLINE ENDMARKER""" assert tree_from_string(expected, gram) == p.parse("hi 42 end") + + + def test_optimized_terminal(self): + gram = """foo: bar baz 'end' NEWLINE ENDMARKER +bar: NAME +baz: NUMBER +""" + p, gram = self.parser_for(gram, False) + expected = """ + foo + bar + NAME "a_name" + baz + NUMBER "42" + NAME "end" + NEWLINE + ENDMARKER""" + input = "a_name 42 end" + tree = p.parse(input) + assert tree_from_string(expected, gram) == tree + assert isinstance(tree, parser.Nonterminal) + assert isinstance(tree.get_child(0), parser.Nonterminal1) + assert isinstance(tree.get_child(1), parser.Nonterminal1) + + + def test_error_string(self): + p, gram = self.parser_for( + "foo: 'if' NUMBER '+' NUMBER" + ) + info = py.test.raises(parser.ParseError, p.parse, "if 42") + info.value.expected_str is None + info = py.test.raises(parser.ParseError, p.parse, "if 42 42") + info.value.expected_str == '+' + diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py --- a/pypy/interpreter/pyparser/test/test_pyparse.py +++ b/pypy/interpreter/pyparser/test/test_pyparse.py @@ -165,3 +165,11 @@ for linefeed in ["\r\n","\r"]: tree = self.parse(fmt % linefeed) assert expected_tree == tree + + def test_error_forgotten_chars(self): + info = py.test.raises(SyntaxError, self.parse, "if 1\n print 4") + assert "(expected ':')" in info.value.msg + info = py.test.raises(SyntaxError, self.parse, "for i in range(10)\n print i") + assert "(expected ':')" in info.value.msg + info = py.test.raises(SyntaxError, self.parse, "def f:\n print 1") + assert "(expected '(')" in info.value.msg diff --git a/pypy/module/parser/pyparser.py b/pypy/module/parser/pyparser.py --- a/pypy/module/parser/pyparser.py +++ b/pypy/module/parser/pyparser.py @@ -152,7 +152,7 @@ # Raise an exception now and be done with it. raise parser_error(space, w_tuple, "Illegal syntax-tree; cannot start with terminal symbol.") - node = pyparse.parser.Nonterminal(type, []) + node = pyparse.parser.Nonterminal(type) build_node_children(space, w_tuple, node, node_state) return node @@ -171,7 +171,7 @@ strn = space.text_w(w_obj) child = pyparse.parser.Terminal(type, strn, node_state.lineno, 0) else: - child = pyparse.parser.Nonterminal(type, []) + child = pyparse.parser.Nonterminal(type) node.append_child(child) if type >= 256: # Nonterminal node build_node_children(space, w_elem, child, node_state) @@ -187,8 +187,7 @@ raise parse_error(space, "Unrecognized node type %d." % type) dfa = parser.grammar.dfas[type] # Run the DFA for this nonterminal - states, first = dfa - arcs, is_accepting = states[0] + arcs, is_accepting = dfa.states[0] for pos in range(tree.num_children()): ch = tree.get_child(pos) for i, next_state in arcs: @@ -198,7 +197,7 @@ if ch.type >= 256: validate_node(space, ch) # Update the state, and move on to the next child. - arcs, is_accepting = states[next_state] + arcs, is_accepting = dfa.states[next_state] break else: raise parse_error(space, "Illegal node") From pypy.commits at gmail.com Sun Apr 1 08:26:08 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 01 Apr 2018 05:26:08 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-subclass-setattr: restore rpython Py_IncRef, Py_DecRef for PYPY_DEBUG_REFCOUNT, old versions segfaulted Message-ID: <5ac0cfe0.c9c7df0a.26ace.c845@mx.google.com> Author: Matti Picus Branch: cpyext-subclass-setattr Changeset: r94211:2315922104c7 Date: 2018-04-01 15:25 +0300 http://bitbucket.org/pypy/pypy/changeset/2315922104c7/ Log: restore rpython Py_IncRef, Py_DecRef for PYPY_DEBUG_REFCOUNT, old versions segfaulted diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -640,7 +640,7 @@ 'Py_DivisionWarningFlag', 'Py_DontWriteBytecodeFlag', 'Py_NoUserSiteDirectory', '_Py_QnewFlag', 'Py_Py3kWarningFlag', 'Py_HashRandomizationFlag', '_Py_PackageContext', '_PyTraceMalloc_Track', '_PyTraceMalloc_Untrack', 'PyMem_Malloc', - 'Py_IncRef', 'Py_DecRef', 'PyObject_Free', 'PyObject_GC_Del', 'PyType_GenericAlloc', + 'PyObject_Free', 'PyObject_GC_Del', 'PyType_GenericAlloc', '_PyObject_New', '_PyObject_NewVar', '_PyObject_GC_New', '_PyObject_GC_NewVar', 'PyObject_Init', 'PyObject_InitVar', 'PyInt_FromLong', diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py --- a/pypy/module/cpyext/pyobject.py +++ b/pypy/module/cpyext/pyobject.py @@ -414,3 +414,13 @@ @cpython_api([rffi.VOIDP], lltype.Signed, error=CANNOT_FAIL) def _Py_HashPointer(space, ptr): return rffi.cast(lltype.Signed, ptr) + + at cpython_api([PyObject], lltype.Void) +def Py_IncRef(space, obj): + incref(space, obj) + + at cpython_api([PyObject], lltype.Void) +def Py_DecRef(space, obj): + decref(space, obj) + + diff --git a/pypy/module/cpyext/src/object.c b/pypy/module/cpyext/src/object.c --- a/pypy/module/cpyext/src/object.c +++ b/pypy/module/cpyext/src/object.c @@ -5,18 +5,6 @@ extern void _PyPy_Free(void *ptr); extern void *_PyPy_Malloc(Py_ssize_t size); -void -Py_IncRef(PyObject *o) -{ - Py_XINCREF(o); -} - -void -Py_DecRef(PyObject *o) -{ - Py_XDECREF(o); -} - /* * The actual value of this variable will be the address of * pyobject.w_marker_deallocating, and will be set by From pypy.commits at gmail.com Sun Apr 1 08:36:32 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 01 Apr 2018 05:36:32 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-subclass-setattr: remove excess Py_XDECREF, test now passes untranslated unfortunately Message-ID: <5ac0d250.0fbcdf0a.b08c9.9bca@mx.google.com> Author: Matti Picus Branch: cpyext-subclass-setattr Changeset: r94212:866aa67a514f Date: 2018-04-01 15:35 +0300 http://bitbucket.org/pypy/pypy/changeset/866aa67a514f/ Log: remove excess Py_XDECREF, test now passes untranslated unfortunately diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c --- a/pypy/module/cpyext/test/array.c +++ b/pypy/module/cpyext/test/array.c @@ -2555,7 +2555,6 @@ } Py_XDECREF(attrib); res = PyObject_Call(collect, tup, NULL); - Py_XDECREF(attrib); if (res == NULL) { Py_DECREF(tup); Py_DECREF(sub); From pypy.commits at gmail.com Sun Apr 1 10:16:02 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 01 Apr 2018 07:16:02 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-subclass-setattr: fix for NULL obj in Py_IncRef Message-ID: <5ac0e9a2.958b1c0a.d043a.d744@mx.google.com> Author: Matti Picus Branch: cpyext-subclass-setattr Changeset: r94213:3f81ccde1932 Date: 2018-04-01 17:15 +0300 http://bitbucket.org/pypy/pypy/changeset/3f81ccde1932/ Log: fix for NULL obj in Py_IncRef diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py --- a/pypy/module/cpyext/pyobject.py +++ b/pypy/module/cpyext/pyobject.py @@ -417,10 +417,11 @@ @cpython_api([PyObject], lltype.Void) def Py_IncRef(space, obj): - incref(space, obj) + # used only ifdef PYPY_DEBUG_REFCOUNT + if obj: + incref(space, obj) @cpython_api([PyObject], lltype.Void) def Py_DecRef(space, obj): + # used only ifdef PYPY_DEBUG_REFCOUNT decref(space, obj) - - From pypy.commits at gmail.com Sun Apr 1 11:00:19 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 01 Apr 2018 08:00:19 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-subclass-setattr: be more selective in setting w_obj.flag_cpytype, fixes translated test Message-ID: <5ac0f403.4a8a1c0a.d836b.1a5f@mx.google.com> Author: Matti Picus Branch: cpyext-subclass-setattr Changeset: r94214:36ff2c150968 Date: 2018-04-01 17:59 +0300 http://bitbucket.org/pypy/pypy/changeset/36ff2c150968/ Log: be more selective in setting w_obj.flag_cpytype, fixes translated test diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -858,6 +858,7 @@ track_reference(space, py_obj, w_obj) # __init__ wraps all slotdefs functions from py_type via add_operators w_obj.__init__(space, py_type) + w_obj.flag_cpytype = True w_obj.ready() finish_type_2(space, py_type, w_obj) diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py --- a/pypy/objspace/std/typeobject.py +++ b/pypy/objspace/std/typeobject.py @@ -1199,7 +1199,9 @@ for w_base in w_self.bases_w: if not isinstance(w_base, W_TypeObject): continue - w_self.flag_cpytype |= w_base.flag_cpytype + # set only in cpyext type_realize, so user-created types will + # still use w_root_attach_pyobj (in cpyext.pyobject) + #w_self.flag_cpytype |= w_base.flag_cpytype w_self.flag_abstract |= w_base.flag_abstract if w_self.flag_map_or_seq == '?': w_self.flag_map_or_seq = w_base.flag_map_or_seq From pypy.commits at gmail.com Mon Apr 2 07:09:07 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 02 Apr 2018 04:09:07 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-subclass-setattr: test places that use is_cpytype, reomve noisy print statements Message-ID: <5ac20f53.8a9e1c0a.1a201.1ac8@mx.google.com> Author: Matti Picus Branch: cpyext-subclass-setattr Changeset: r94215:dc1bf38e9aed Date: 2018-04-02 14:08 +0300 http://bitbucket.org/pypy/pypy/changeset/dc1bf38e9aed/ Log: test places that use is_cpytype, reomve noisy print statements diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c --- a/pypy/module/cpyext/test/array.c +++ b/pypy/module/cpyext/test/array.c @@ -2523,13 +2523,17 @@ return NULL; } tup = PyTuple_New(0); + /* #ifdef PYPY_VERSION printf("calling addattrib pypylink %lu \n", sub->ob_pypy_link); #endif + */ res = PyObject_Call(attrib, tup, NULL); + /* #ifdef PYPY_VERSION printf("after addattrib pypylink %lu \n", sub->ob_pypy_link); #endif + */ Py_DECREF(attrib); if (res == NULL) { Py_DECREF(tup); @@ -2538,12 +2542,14 @@ } Py_DECREF(res); for(i=0; i<10; i++) { + /* #ifdef PYPY_VERSION printf("starting loop iteration %d refcnt %lu pypylink %lu \n", i, sub->ob_refcnt, sub->ob_pypy_link); #else printf("starting loop iteration %d refcnt %lu\n", i, sub->ob_refcnt); #endif + */ attrib = PyObject_GetAttr(sub, attribname); if (!attrib || (attrib == Py_None)) { PyErr_SetString(PyExc_ValueError, diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py --- a/pypy/module/cpyext/test/test_arraymodule.py +++ b/pypy/module/cpyext/test/test_arraymodule.py @@ -194,4 +194,7 @@ self.attrib = True import gc module.subclass_with_attribute(Sub, "addattrib", "attrib", gc.collect) + if not self.runappdirect: + assert Sub.__module__ == 'pypy.module.cpyext.test.test_arraymodule' + assert str(Sub) == "" diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -412,33 +412,42 @@ def test_type_dict(self): foo = self.import_module("foo") module = self.import_extension('test', [ - ("hack_tp_dict", "METH_O", + ("hack_tp_dict", "METH_VARARGS", ''' - PyTypeObject *type = args->ob_type; + PyTypeObject *type, *obj; PyObject *a1 = PyLong_FromLong(1); PyObject *a2 = PyLong_FromLong(2); PyObject *value; + PyObject * key; + if (!PyArg_ParseTuple(args, "OO", &obj, &key)) + return NULL; + type = obj->ob_type; - if (PyDict_SetItemString(type->tp_dict, "a", + if (PyDict_SetItem(type->tp_dict, key, a1) < 0) return NULL; Py_DECREF(a1); PyType_Modified(type); - value = PyObject_GetAttrString((PyObject *)type, "a"); + value = PyObject_GetAttr((PyObject *)type, key); Py_DECREF(value); - if (PyDict_SetItemString(type->tp_dict, "a", + if (PyDict_SetItem(type->tp_dict, key, a2) < 0) return NULL; Py_DECREF(a2); PyType_Modified(type); - value = PyObject_GetAttrString((PyObject *)type, "a"); + value = PyObject_GetAttr((PyObject *)type, key); return value; ''' ) ]) obj = foo.new() - assert module.hack_tp_dict(obj) == 2 + assert module.hack_tp_dict(obj, "a") == 2 + class Sub(foo.fooType): + pass + obj = Sub() + assert module.hack_tp_dict(obj, "b") == 2 + def test_tp_descr_get(self): module = self.import_extension('foo', [ From pypy.commits at gmail.com Mon Apr 2 08:13:48 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 02 Apr 2018 05:13:48 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-subclass-setattr: reverse logic Message-ID: <5ac21e7c.0999df0a.b07e3.4234@mx.google.com> Author: Matti Picus Branch: cpyext-subclass-setattr Changeset: r94216:268b6bc566ac Date: 2018-04-02 15:13 +0300 http://bitbucket.org/pypy/pypy/changeset/268b6bc566ac/ Log: reverse logic diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py --- a/pypy/module/cpyext/test/test_arraymodule.py +++ b/pypy/module/cpyext/test/test_arraymodule.py @@ -194,7 +194,7 @@ self.attrib = True import gc module.subclass_with_attribute(Sub, "addattrib", "attrib", gc.collect) - if not self.runappdirect: + if self.runappdirect: assert Sub.__module__ == 'pypy.module.cpyext.test.test_arraymodule' assert str(Sub) == "" From pypy.commits at gmail.com Mon Apr 2 23:07:41 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 02 Apr 2018 20:07:41 -0700 (PDT) Subject: [pypy-commit] pypy ioctl-arg-size: copy cpython approach to sizeof(arg) in ioctl - allocate at least IOCTL_BUFSZ Message-ID: <5ac2effd.4a061c0a.e2bb8.838e@mx.google.com> Author: Matti Picus Branch: ioctl-arg-size Changeset: r94217:3e2c7c20e319 Date: 2018-04-02 20:23 +0300 http://bitbucket.org/pypy/pypy/changeset/3e2c7c20e319/ Log: copy cpython approach to sizeof(arg) in ioctl - allocate at least IOCTL_BUFSZ diff --git a/pypy/module/fcntl/interp_fcntl.py b/pypy/module/fcntl/interp_fcntl.py --- a/pypy/module/fcntl/interp_fcntl.py +++ b/pypy/module/fcntl/interp_fcntl.py @@ -204,6 +204,7 @@ # XXX this function's interface is a mess. # We try to emulate the behavior of Python >= 2.5 w.r.t. mutate_flag + IOCTL_BUFSZ = 1024 # like cpython fd = space.c_filedescriptor_w(w_fd) op = rffi.cast(rffi.INT, op) # C long => C int @@ -216,15 +217,19 @@ else: arg = rwbuffer.as_str() ll_arg = rffi.str2charp(arg) + to_alloc = max(IOCTL_BUFSZ, len(arg)) try: - rv = ioctl_str(fd, op, ll_arg) - if rv < 0: - raise _get_error(space, "ioctl") - arg = rffi.charpsize2str(ll_arg, len(arg)) - if mutate_flag != 0: - rwbuffer.setslice(0, arg) - return space.newint(rv) - return space.newbytes(arg) + with rffi.scoped_alloc_buffer(to_alloc) as buf: + rffi.c_memcpy(rffi.cast(rffi.VOIDP, buf.raw), + rffi.cast(rffi.VOIDP, ll_arg), len(arg)) + rv = ioctl_str(fd, op, buf.raw) + if rv < 0: + raise _get_error(space, "ioctl") + arg = rffi.charpsize2str(buf.raw, len(arg)) + if mutate_flag != 0: + rwbuffer.setslice(0, arg) + return space.newint(rv) + return space.newbytes(arg) finally: lltype.free(ll_arg, flavor='raw') @@ -240,11 +245,15 @@ raise else: ll_arg = rffi.str2charp(arg) + to_alloc = max(IOCTL_BUFSZ, len(arg)) try: - rv = ioctl_str(fd, op, ll_arg) - if rv < 0: - raise _get_error(space, "ioctl") - arg = rffi.charpsize2str(ll_arg, len(arg)) + with rffi.scoped_alloc_buffer(to_alloc) as buf: + rffi.c_memcpy(rffi.cast(rffi.VOIDP, buf.raw), + rffi.cast(rffi.VOIDP, ll_arg), len(arg)) + rv = ioctl_str(fd, op, buf.raw) + if rv < 0: + raise _get_error(space, "ioctl") + arg = rffi.charpsize2str(buf.raw, len(arg)) return space.newbytes(arg) finally: lltype.free(ll_arg, flavor='raw') From pypy.commits at gmail.com Mon Apr 2 23:07:43 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 02 Apr 2018 20:07:43 -0700 (PDT) Subject: [pypy-commit] pypy default: decref, fixes issue #2781 Message-ID: <5ac2efff.96e81c0a.ce368.b916@mx.google.com> Author: Matti Picus Branch: Changeset: r94218:c20052c907ba Date: 2018-04-03 06:05 +0300 http://bitbucket.org/pypy/pypy/changeset/c20052c907ba/ Log: decref, fixes issue #2781 diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -13,7 +13,7 @@ ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc, cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc, readbufferproc, getbufferproc, ssizessizeobjargproc) -from pypy.module.cpyext.pyobject import make_ref, from_ref, as_pyobj +from pypy.module.cpyext.pyobject import make_ref, from_ref, as_pyobj, decref from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.module.cpyext.memoryobject import fill_Py_buffer from pypy.module.cpyext.state import State @@ -63,6 +63,7 @@ py_args = tuple_from_args_w(space, __args__.arguments_w) w_kwargs = w_kwargs_from_args(space, __args__) res = generic_cpy_call(space, func_init, w_self, py_args, w_kwargs) + decref(space, py_args) if rffi.cast(lltype.Signed, res) == -1: space.fromcache(State).check_and_raise_exception(always=True) return None @@ -244,7 +245,9 @@ func_target = rffi.cast(ternaryfunc, func) py_args = tuple_from_args_w(space, __args__.arguments_w) w_kwargs = w_kwargs_from_args(space, __args__) - return generic_cpy_call(space, func_target, w_self, py_args, w_kwargs) + ret = generic_cpy_call(space, func_target, w_self, py_args, w_kwargs) + decref(space, py_args) + return ret class wrap_ssizessizeobjargproc(W_PyCWrapperObject): def call(self, space, w_self, __args__): From pypy.commits at gmail.com Mon Apr 2 23:07:47 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 02 Apr 2018 20:07:47 -0700 (PDT) Subject: [pypy-commit] pypy default: merge branch which fixes issue #2776 Message-ID: <5ac2f003.ab87df0a.213da.7f07@mx.google.com> Author: Matti Picus Branch: Changeset: r94220:03739fa3fc5c Date: 2018-04-03 06:06 +0300 http://bitbucket.org/pypy/pypy/changeset/03739fa3fc5c/ Log: merge branch which fixes issue #2776 diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -91,3 +91,9 @@ .. branch: pyparser-improvements Improve speed of Python parser, improve ParseError messages slightly. + +.. branch: ioctl-arg-size + +Work around possible bugs in upstream ioctl users, like CPython allocate at +least 1024 bytes for the arg in calls to ``ioctl(fd, request, arg)``. Fixes +issue #2776 diff --git a/pypy/module/fcntl/interp_fcntl.py b/pypy/module/fcntl/interp_fcntl.py --- a/pypy/module/fcntl/interp_fcntl.py +++ b/pypy/module/fcntl/interp_fcntl.py @@ -204,6 +204,7 @@ # XXX this function's interface is a mess. # We try to emulate the behavior of Python >= 2.5 w.r.t. mutate_flag + IOCTL_BUFSZ = 1024 # like cpython fd = space.c_filedescriptor_w(w_fd) op = rffi.cast(rffi.INT, op) # C long => C int @@ -216,15 +217,19 @@ else: arg = rwbuffer.as_str() ll_arg = rffi.str2charp(arg) + to_alloc = max(IOCTL_BUFSZ, len(arg)) try: - rv = ioctl_str(fd, op, ll_arg) - if rv < 0: - raise _get_error(space, "ioctl") - arg = rffi.charpsize2str(ll_arg, len(arg)) - if mutate_flag != 0: - rwbuffer.setslice(0, arg) - return space.newint(rv) - return space.newbytes(arg) + with rffi.scoped_alloc_buffer(to_alloc) as buf: + rffi.c_memcpy(rffi.cast(rffi.VOIDP, buf.raw), + rffi.cast(rffi.VOIDP, ll_arg), len(arg)) + rv = ioctl_str(fd, op, buf.raw) + if rv < 0: + raise _get_error(space, "ioctl") + arg = rffi.charpsize2str(buf.raw, len(arg)) + if mutate_flag != 0: + rwbuffer.setslice(0, arg) + return space.newint(rv) + return space.newbytes(arg) finally: lltype.free(ll_arg, flavor='raw') @@ -240,11 +245,15 @@ raise else: ll_arg = rffi.str2charp(arg) + to_alloc = max(IOCTL_BUFSZ, len(arg)) try: - rv = ioctl_str(fd, op, ll_arg) - if rv < 0: - raise _get_error(space, "ioctl") - arg = rffi.charpsize2str(ll_arg, len(arg)) + with rffi.scoped_alloc_buffer(to_alloc) as buf: + rffi.c_memcpy(rffi.cast(rffi.VOIDP, buf.raw), + rffi.cast(rffi.VOIDP, ll_arg), len(arg)) + rv = ioctl_str(fd, op, buf.raw) + if rv < 0: + raise _get_error(space, "ioctl") + arg = rffi.charpsize2str(buf.raw, len(arg)) return space.newbytes(arg) finally: lltype.free(ll_arg, flavor='raw') From pypy.commits at gmail.com Mon Apr 2 23:07:45 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 02 Apr 2018 20:07:45 -0700 (PDT) Subject: [pypy-commit] pypy ioctl-arg-size: document, close branch to be merged Message-ID: <5ac2f001.6ca0df0a.2b35f.83db@mx.google.com> Author: Matti Picus Branch: ioctl-arg-size Changeset: r94219:357e97475d38 Date: 2018-04-03 06:05 +0300 http://bitbucket.org/pypy/pypy/changeset/357e97475d38/ Log: document, close branch to be merged diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -91,3 +91,9 @@ .. branch: pyparser-improvements Improve speed of Python parser, improve ParseError messages slightly. + +.. branch: ioctl-arg-size + +Work around possible bugs in upstream ioctl users, like CPython allocate at +least 1024 bytes for the arg in calls to ``ioctl(fd, request, arg)``. Fixes +issue #2776 From pypy.commits at gmail.com Tue Apr 3 04:56:41 2018 From: pypy.commits at gmail.com (arigo) Date: Tue, 03 Apr 2018 01:56:41 -0700 (PDT) Subject: [pypy-commit] pypy default: Issue #2784 Message-ID: <5ac341c9.096c1c0a.37e2d.0930@mx.google.com> Author: Armin Rigo Branch: Changeset: r94221:1fe837dccd7b Date: 2018-04-03 10:55 +0200 http://bitbucket.org/pypy/pypy/changeset/1fe837dccd7b/ Log: Issue #2784 Bump the value of re._MAXCACHE. diff --git a/lib-python/2.7/re.py b/lib-python/2.7/re.py --- a/lib-python/2.7/re.py +++ b/lib-python/2.7/re.py @@ -225,7 +225,7 @@ _pattern_type = type(sre_compile.compile("", 0)) -_MAXCACHE = 100 +_MAXCACHE = 1000 def _compile(*key): # internal: compile pattern From pypy.commits at gmail.com Tue Apr 3 05:30:57 2018 From: pypy.commits at gmail.com (arigo) Date: Tue, 03 Apr 2018 02:30:57 -0700 (PDT) Subject: [pypy-commit] pypy default: Issue #2780 Message-ID: <5ac349d1.d4e41c0a.475ea.16a3@mx.google.com> Author: Armin Rigo Branch: Changeset: r94222:fb6f7b8f2635 Date: 2018-04-03 11:30 +0200 http://bitbucket.org/pypy/pypy/changeset/fb6f7b8f2635/ Log: Issue #2780 Ignore any non-keyword args in signal.default_int_handler() diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -166,13 +166,14 @@ return handlers_w[signum] -def default_int_handler(space, w_signum, w_frame): +def default_int_handler(space, args_w): """ default_int_handler(...) The default handler for SIGINT installed by Python. It raises KeyboardInterrupt. """ + # issue #2780: accept and ignore any non-keyword arguments raise OperationError(space.w_KeyboardInterrupt, space.w_None) diff --git a/pypy/module/signal/test/test_signal.py b/pypy/module/signal/test/test_signal.py --- a/pypy/module/signal/test/test_signal.py +++ b/pypy/module/signal/test/test_signal.py @@ -260,6 +260,17 @@ finally: signal.signal(signum, oldhandler) + def test_default_int_handler(self): + import signal + for args in [(), (1, 2)]: + try: + signal.default_int_handler(*args) + except KeyboardInterrupt: + pass + else: + raise AssertionError("did not raise!") + + class AppTestSignalSocket: spaceconfig = dict(usemodules=['signal', '_socket']) From pypy.commits at gmail.com Tue Apr 3 10:25:44 2018 From: pypy.commits at gmail.com (antocuni) Date: Tue, 03 Apr 2018 07:25:44 -0700 (PDT) Subject: [pypy-commit] pypy gc-hooks: turn AbstractActionFlag._fired_actions into a linked list, so that we can manipulate it freely without doing GC allocations: this is needed to call AsyncAction.fire() from a GC hook Message-ID: <5ac38ee8.ad8adf0a.63a78.0321@mx.google.com> Author: Antonio Cuni Branch: gc-hooks Changeset: r94223:a88536ae8554 Date: 2018-04-03 16:24 +0200 http://bitbucket.org/pypy/pypy/changeset/a88536ae8554/ Log: turn AbstractActionFlag._fired_actions into a linked list, so that we can manipulate it freely without doing GC allocations: this is needed to call AsyncAction.fire() from a GC hook diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -404,7 +404,7 @@ self._periodic_actions = [] self._nonperiodic_actions = [] self.has_bytecode_counter = False - self.fired_actions = None + self._fired_actions_reset() # the default value is not 100, unlike CPython 2.7, but a much # larger value, because we use a technique that not only allows # but actually *forces* another thread to run whenever the counter @@ -416,13 +416,27 @@ """Request for the action to be run before the next opcode.""" if not action._fired: action._fired = True - if self.fired_actions is None: - self.fired_actions = [] - self.fired_actions.append(action) + self._fired_actions_append(action) # set the ticker to -1 in order to force action_dispatcher() # to run at the next possible bytecode self.reset_ticker(-1) + def _fired_actions_reset(self): + # linked list of actions. We cannot use a normal RPython list because + # we want AsyncAction.fire() to be marked as @rgc.collect: this way, + # we can call it from e.g. GcHooks or cpyext's dealloc_trigger. + self._fired_actions_first = None + self._fired_actions_last = None + + @rgc.no_collect + def _fired_actions_append(self, action): + if self._fired_actions_first is None: + self._fired_actions_first = action + self._fired_actions_last = action + else: + self._fired_actions_last._next = action + self._fired_actions_last = action + @not_rpython def register_periodic_action(self, action, use_bytecode_counter): """ @@ -467,9 +481,9 @@ action.perform(ec, frame) # nonperiodic actions - list = self.fired_actions - if list is not None: - self.fired_actions = None + action = self._fired_actions_first + if action: + self._fired_actions_reset() # NB. in case there are several actions, we reset each # 'action._fired' to false only when we're about to call # 'action.perform()'. This means that if @@ -477,9 +491,10 @@ # the corresponding perform(), the fire() has no # effect---which is the effect we want, because # perform() will be called anyway. - for action in list: + while action is not None: action._fired = False action.perform(ec, frame) + action = action._next self.action_dispatcher = action_dispatcher @@ -512,10 +527,12 @@ to occur between two opcodes, not at a completely random time. """ _fired = False + _next = None def __init__(self, space): self.space = space + @rgc.no_collect def fire(self): """Request for the action to be run before the next opcode. The action must have been registered at space initalization time.""" From pypy.commits at gmail.com Tue Apr 3 19:12:18 2018 From: pypy.commits at gmail.com (antocuni) Date: Tue, 03 Apr 2018 16:12:18 -0700 (PDT) Subject: [pypy-commit] pypy gc-hooks: refactor to avoid the ugly 'gchooks' global prebuilt variable. Now we can simply use space.fromcache to get the singleton Message-ID: <5ac40a52.09e61c0a.2b958.36a4@mx.google.com> Author: Antonio Cuni Branch: gc-hooks Changeset: r94224:b853aac49cd8 Date: 2018-04-04 01:11 +0200 http://bitbucket.org/pypy/pypy/changeset/b853aac49cd8/ Log: refactor to avoid the ugly 'gchooks' global prebuilt variable. Now we can simply use space.fromcache to get the singleton diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -215,6 +215,7 @@ usage = SUPPRESS_USAGE take_options = True + space = None def opt_parser(self, config): parser = to_optparse(config, useoptions=["objspace.*"], @@ -365,18 +366,20 @@ return PyPyJitPolicy(pypy_hooks) def get_gchooks(self): - from pypy.module.gc.hook import gchooks - return gchooks + from pypy.module.gc.hook import LowLevelGcHooks + if self.space is None: + raise Exception("get_gchooks must be called afeter get_entry_point") + return self.space.fromcache(LowLevelGcHooks) def get_entry_point(self, config): - space = make_objspace(config) + self.space = make_objspace(config) # manually imports app_main.py filename = os.path.join(pypydir, 'interpreter', 'app_main.py') app = gateway.applevel(open(filename).read(), 'app_main.py', 'app_main') app.hidden_applevel = False - w_dict = app.getwdict(space) - entry_point, _ = create_entry_point(space, w_dict) + w_dict = app.getwdict(self.space) + entry_point, _ = create_entry_point(self.space, w_dict) return entry_point, None, PyPyAnnotatorPolicy() @@ -385,7 +388,7 @@ 'jitpolicy', 'get_entry_point', 'get_additional_config_options']: ns[name] = getattr(self, name) - ns['gchooks'] = self.get_gchooks() + ns['get_gchooks'] = self.get_gchooks PyPyTarget().interface(globals()) diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py --- a/pypy/module/gc/__init__.py +++ b/pypy/module/gc/__init__.py @@ -37,7 +37,3 @@ 'set_hooks': 'hook.set_hooks', }) MixedModule.__init__(self, space, w_name) - - def setup_after_space_initialization(self): - from pypy.module.gc.hook import gchooks - gchooks.setspace(self.space) diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py --- a/pypy/module/gc/hook.py +++ b/pypy/module/gc/hook.py @@ -8,7 +8,7 @@ class LowLevelGcHooks(GcHooks): - def setspace(self, space): + def __init__(self, space): self.space = space self.hooks = space.fromcache(AppLevelHooks) @@ -30,8 +30,6 @@ pass -gchooks = LowLevelGcHooks() - class AppLevelHooks(object): def __init__(self, space): diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py --- a/pypy/module/gc/test/test_hook.py +++ b/pypy/module/gc/test/test_hook.py @@ -1,5 +1,5 @@ from rpython.rlib.rarithmetic import r_uint -from pypy.module.gc.hook import gchooks +from pypy.module.gc.hook import LowLevelGcHooks from pypy.interpreter.baseobjspace import ObjSpace from pypy.interpreter.gateway import interp2app, unwrap_spec @@ -7,6 +7,7 @@ def setup_class(cls): space = cls.space + gchooks = space.fromcache(LowLevelGcHooks) @unwrap_spec(ObjSpace, r_uint, int) def fire_gc_minor(space, total_memory_used, pinned_objects): diff --git a/rpython/memory/test/test_transformed_gc.py b/rpython/memory/test/test_transformed_gc.py --- a/rpython/memory/test/test_transformed_gc.py +++ b/rpython/memory/test/test_transformed_gc.py @@ -1409,8 +1409,8 @@ class MyGcHooks(GcHooks): - def __init__(self): - self.stats = GcHooksStats() + def __init__(self, stats=None): + self.stats = stats or GcHooksStats() def is_gc_minor_enabled(self): return True diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py --- a/rpython/translator/driver.py +++ b/rpython/translator/driver.py @@ -413,7 +413,8 @@ translator.frozen = True standalone = self.standalone - gchooks = self.extra.get('gchooks', None) + get_gchooks = self.extra.get('get_gchooks', lambda: None) + gchooks = get_gchooks() if standalone: from rpython.translator.c.genc import CStandaloneBuilder diff --git a/rpython/translator/goal/targetgcbench.py b/rpython/translator/goal/targetgcbench.py --- a/rpython/translator/goal/targetgcbench.py +++ b/rpython/translator/goal/targetgcbench.py @@ -1,8 +1,10 @@ from rpython.translator.goal import gcbench -from rpython.memory.test.test_transformed_gc import MyGcHooks +from rpython.memory.test.test_transformed_gc import MyGcHooks, GcHooksStats # _____ Define and setup target ___ +GC_HOOKS_STATS = GcHooksStats() + def entry_point(argv): GC_HOOKS_STATS.reset() ret = gcbench.entry_point(argv) @@ -15,10 +17,9 @@ print ' gc-collect: ', collects return ret -gchooks = MyGcHooks() -GC_HOOKS_STATS = gchooks.stats +def get_gchooks(): + return MyGcHooks(GC_HOOKS_STATS) def target(*args): gcbench.ENABLE_THREADS = False # not RPython return entry_point, None - From pypy.commits at gmail.com Wed Apr 4 12:26:09 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 04 Apr 2018 09:26:09 -0700 (PDT) Subject: [pypy-commit] buildbot default: move pypy-c-jit-freebsd-9-x86-64 to inactive_slaves Message-ID: <5ac4fca1.0ac0df0a.cfa2f.59f5@mx.google.com> Author: Matti Picus Branch: Changeset: r1060:1fbe9ae3024b Date: 2018-04-04 19:25 +0300 http://bitbucket.org/pypy/buildbot/changeset/1fbe9ae3024b/ Log: move pypy-c-jit-freebsd-9-x86-64 to inactive_slaves diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -262,7 +262,13 @@ 'factory' : pypyJITTranslatedTestFactoryFreeBSD, "category": 'freebsd64' }, - ] + {"name" : JITFREEBSD964, + "slavenames": ['hybridlogic', 'tavendo-freebsd-9.2-amd64'], + 'builddir' : JITFREEBSD964, + 'factory' : pypyJITTranslatedTestFactoryFreeBSD, + "category": 'freebsd64' + }, + ] extra_opts = {'xerxes': {'keepalive_interval': 15}, 'aurora': {'max_builds': 1}, 'salsa': {'max_builds': 1}, @@ -562,13 +568,7 @@ "locks": [WinSlaveLock.access('counting')], 'category' : 'win32', }, - {"name" : JITFREEBSD964, - "slavenames": ['hybridlogic', 'tavendo-freebsd-9.2-amd64'], - 'builddir' : JITFREEBSD964, - 'factory' : pypyJITTranslatedTestFactoryFreeBSD, - "category": 'freebsd64' - }, - # PPC + # PPC {"name": JITONLYLINUXPPC64, "slavenames": ['gcc1'], "builddir": JITONLYLINUXPPC64, From pypy.commits at gmail.com Wed Apr 4 12:48:16 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 04 Apr 2018 09:48:16 -0700 (PDT) Subject: [pypy-commit] pypy default: prepare new release Message-ID: <5ac501d0.41711c0a.fef48.ac32@mx.google.com> Author: Matti Picus Branch: Changeset: r94228:2e04adf1b89f Date: 2018-04-04 18:50 +0300 http://bitbucket.org/pypy/pypy/changeset/2e04adf1b89f/ Log: prepare new release diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-6.0.0.rst rename from pypy/doc/whatsnew-head.rst rename to pypy/doc/whatsnew-6.0.0.rst diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -29,8 +29,8 @@ #define PY_VERSION "2.7.13" /* PyPy version as a string */ -#define PYPY_VERSION "5.11.0-alpha0" -#define PYPY_VERSION_NUM 0x050B0000 +#define PYPY_VERSION "6.1.0-alpha0" +#define PYPY_VERSION_NUM 0x06010000 /* Defined to mean a PyPy where cpyext holds more regular references to PyObjects, e.g. staying alive as long as the internal PyPy object diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -10,7 +10,7 @@ #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py CPYTHON_API_VERSION = 1013 #XXX # sync with include/modsupport.h -PYPY_VERSION = (5, 11, 0, "alpha", 0) #XXX # sync patchlevel.h +PYPY_VERSION = (6, 1, 0, "alpha", 0) #XXX # sync patchlevel.h import pypy From pypy.commits at gmail.com Wed Apr 4 12:48:18 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 04 Apr 2018 09:48:18 -0700 (PDT) Subject: [pypy-commit] pypy default: restart whatsnew-head Message-ID: <5ac501d2.832b1c0a.c8ea2.262d@mx.google.com> Author: Matti Picus Branch: Changeset: r94229:0dbf12ced3b7 Date: 2018-04-04 18:54 +0300 http://bitbucket.org/pypy/pypy/changeset/0dbf12ced3b7/ Log: restart whatsnew-head diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-head.rst @@ -0,0 +1,8 @@ +========================== +What's new in PyPy2.7 6.0+ +========================== + +.. this is a revision shortly after release-pypy-6.0.0 +.. startrev: 2e04adf1b89f + + From pypy.commits at gmail.com Wed Apr 4 12:48:22 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 04 Apr 2018 09:48:22 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: prepare new release Message-ID: <5ac501d6.0ac0df0a.cfa2f.5fda@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r94231:2dcc8ff1c4c3 Date: 2018-04-04 19:36 +0300 http://bitbucket.org/pypy/pypy/changeset/2dcc8ff1c4c3/ Log: prepare new release diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst --- a/pypy/doc/whatsnew-pypy3-head.rst +++ b/pypy/doc/whatsnew-pypy3-head.rst @@ -17,3 +17,12 @@ Add missing ``__doc__``, ``__module__``, ``__name__`` attributes to ``instancemethod`` + +.. branch: winapi + +Update support for _winapi cffi module for python3 + +.. branch: py3.5-refactor-slots + +Refactor cpyext slots. + diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -29,8 +29,8 @@ #define PY_VERSION "3.5.3" /* PyPy version as a string */ -#define PYPY_VERSION "5.11.0-alpha0" -#define PYPY_VERSION_NUM 0x050B0000 +#define PYPY_VERSION "6.1.0-alpha0" +#define PYPY_VERSION_NUM 0x06010000 /* Defined to mean a PyPy where cpyext holds more regular references to PyObjects, e.g. staying alive as long as the internal PyPy object diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -10,7 +10,7 @@ #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py CPYTHON_API_VERSION = 1013 #XXX # sync with include/modsupport.h -PYPY_VERSION = (5, 11, 0, "alpha", 0) #XXX # sync patchlevel.h +PYPY_VERSION = (6, 1, 0, "alpha", 0) #XXX # sync patchlevel.h import pypy From pypy.commits at gmail.com Wed Apr 4 12:48:24 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 04 Apr 2018 09:48:24 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: merge default into branch Message-ID: <5ac501d8.95b5df0a.fa481.8664@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r94232:552c05a134d7 Date: 2018-04-04 19:40 +0300 http://bitbucket.org/pypy/pypy/changeset/552c05a134d7/ Log: merge default into branch diff --git a/lib-python/2.7/re.py b/lib-python/2.7/re.py --- a/lib-python/2.7/re.py +++ b/lib-python/2.7/re.py @@ -225,7 +225,7 @@ _pattern_type = type(sre_compile.compile("", 0)) -_MAXCACHE = 100 +_MAXCACHE = 1000 def _compile(*key): # internal: compile pattern diff --git a/lib-python/2.7/test/test_genexps.py b/lib-python/2.7/test/test_genexps.py --- a/lib-python/2.7/test/test_genexps.py +++ b/lib-python/2.7/test/test_genexps.py @@ -87,7 +87,7 @@ >>> dict(a = i for i in xrange(10)) Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: invalid syntax (expected ')') Verify that parenthesis are required when used as a keyword argument value diff --git a/pypy/doc/whatsnew-6.0.0.rst b/pypy/doc/whatsnew-6.0.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-6.0.0.rst @@ -0,0 +1,103 @@ +=========================== +What's new in PyPy2.7 5.10+ +=========================== + +.. this is a revision shortly after release-pypy2.7-v5.10.0 +.. startrev: 6b024edd9d12 + +.. branch: cpyext-avoid-roundtrip + +Big refactoring of some cpyext code, which avoids a lot of nonsense when +calling C from Python and vice-versa: the result is a big speedup in +function/method calls, up to 6 times faster. + +.. branch: cpyext-datetime2 + +Support ``tzinfo`` field on C-API datetime objects, fixes latest pandas HEAD + + +.. branch: mapdict-size-limit + +Fix a corner case of mapdict: When an instance is used like a dict (using +``setattr`` and ``getattr``, or ``.__dict__``) and a lot of attributes are +added, then the performance using mapdict is linear in the number of +attributes. This is now fixed (by switching to a regular dict after 80 +attributes). + + +.. branch: cpyext-faster-arg-passing + +When using cpyext, improve the speed of passing certain objects from PyPy to C +code, most notably None, True, False, types, all instances of C-defined types. +Before, a dict lookup was needed every time such an object crossed over, now it +is just a field read. + + +.. branch: 2634_datetime_timedelta_performance + +Improve datetime + timedelta performance. + +.. branch: memory-accounting + +Improve way to describe memory + +.. branch: msvc14 + +Allow compilaiton with Visual Studio 2017 compiler suite on windows + +.. branch: winapi + +Update _winapi and internal _winbase_cffi (via _winbase_build) for python 3 + +.. branch: refactor-slots + +Refactor cpyext slots. + + +.. branch: call-loopinvariant-into-bridges + +Speed up branchy code that does a lot of function inlining by saving one call +to read the TLS in most bridges. + +.. branch: rpython-sprint + +Refactor in rpython signatures + +.. branch: cpyext-tls-operror2 + +Store error state thread-locally in executioncontext, fixes issue #2764 + +.. branch: cpyext-fast-typecheck + +Optimize `Py*_Check` for `Bool`, `Float`, `Set`. Also refactor and simplify +`W_PyCWrapperObject` which is used to call slots from the C-API, greatly +improving microbenchmarks in https://github.com/antocuni/cpyext-benchmarks + + +.. branch: fix-sre-problems + +Fix two (unrelated) JIT bugs manifesting in the re module: + +- green fields are broken and were thus disabled, plus their usage removed from + the _sre implementation + +- in rare "trace is too long" situations, the JIT could break behaviour + arbitrarily. + +.. branch: jit-hooks-can-be-disabled + +Be more efficient about JIT hooks. Make it possible for the frontend to declare +that jit hooks are currently not enabled at all. in that case, the list of ops +does not have to be created in the case of the on_abort hook (which is +expensive). + + +.. branch: pyparser-improvements + +Improve speed of Python parser, improve ParseError messages slightly. + +.. branch: ioctl-arg-size + +Work around possible bugs in upstream ioctl users, like CPython allocate at +least 1024 bytes for the arg in calls to ``ioctl(fd, request, arg)``. Fixes +issue #2776 diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -1,85 +1,8 @@ -=========================== -What's new in PyPy2.7 5.10+ -=========================== +========================== +What's new in PyPy2.7 6.0+ +========================== -.. this is a revision shortly after release-pypy2.7-v5.10.0 -.. startrev: 6b024edd9d12 +.. this is a revision shortly after release-pypy-6.0.0 +.. startrev: 2e04adf1b89f -.. branch: cpyext-avoid-roundtrip -Big refactoring of some cpyext code, which avoids a lot of nonsense when -calling C from Python and vice-versa: the result is a big speedup in -function/method calls, up to 6 times faster. - -.. branch: cpyext-datetime2 - -Support ``tzinfo`` field on C-API datetime objects, fixes latest pandas HEAD - - -.. branch: mapdict-size-limit - -Fix a corner case of mapdict: When an instance is used like a dict (using -``setattr`` and ``getattr``, or ``.__dict__``) and a lot of attributes are -added, then the performance using mapdict is linear in the number of -attributes. This is now fixed (by switching to a regular dict after 80 -attributes). - - -.. branch: cpyext-faster-arg-passing - -When using cpyext, improve the speed of passing certain objects from PyPy to C -code, most notably None, True, False, types, all instances of C-defined types. -Before, a dict lookup was needed every time such an object crossed over, now it -is just a field read. - - -.. branch: 2634_datetime_timedelta_performance - -Improve datetime + timedelta performance. - -.. branch: memory-accounting - -Improve way to describe memory - -.. branch: msvc14 - -Allow compilaiton with Visual Studio 2017 compiler suite on windows - -.. branch: winapi - -Update _winapi and internal _winbase_cffi (via _winbase_build) for python 3 - -.. branch: refactor-slots - -Refactor cpyext slots. - - -.. branch: call-loopinvariant-into-bridges - -Speed up branchy code that does a lot of function inlining by saving one call -to read the TLS in most bridges. - -.. branch: rpython-sprint - -Refactor in rpython signatures - -.. branch: cpyext-tls-operror2 - -Store error state thread-locally in executioncontext, fixes issue #2764 - -.. branch: cpyext-fast-typecheck - -Optimize `Py*_Check` for `Bool`, `Float`, `Set`. Also refactor and simplify -`W_PyCWrapperObject` which is used to call slots from the C-API, greatly -improving microbenchmarks in https://github.com/antocuni/cpyext-benchmarks - - -.. branch: fix-sre-problems - -Fix two (unrelated) JIT bugs manifesting in the re module: - -- green fields are broken and were thus disabled, plus their usage removed from - the _sre implementation - -- in rare "trace is too long" situations, the JIT could break behaviour - arbitrarily. diff --git a/pypy/interpreter/pyparser/metaparser.py b/pypy/interpreter/pyparser/metaparser.py --- a/pypy/interpreter/pyparser/metaparser.py +++ b/pypy/interpreter/pyparser/metaparser.py @@ -147,8 +147,10 @@ for label, next in state.arcs.iteritems(): arcs.append((self.make_label(gram, label), dfa.index(next))) states.append((arcs, state.is_final)) - gram.dfas.append((states, self.make_first(gram, name))) - assert len(gram.dfas) - 1 == gram.symbol_ids[name] - 256 + symbol_id = gram.symbol_ids[name] + dfa = parser.DFA(symbol_id, states, self.make_first(gram, name)) + gram.dfas.append(dfa) + assert len(gram.dfas) - 1 == symbol_id - 256 gram.start = gram.symbol_ids[self.start_symbol] return gram @@ -162,6 +164,13 @@ else: gram.labels.append(gram.symbol_ids[label]) gram.symbol_to_label[label] = label_index + first = self.first[label] + if len(first) == 1: + first, = first + if not first[0].isupper(): + first = first.strip("\"'") + assert label_index not in gram.token_to_error_string + gram.token_to_error_string[label_index] = first return label_index elif label.isupper(): token_index = gram.TOKENS[label] @@ -183,7 +192,7 @@ else: gram.labels.append(gram.KEYWORD_TOKEN) gram.keyword_ids[value] = label_index - return label_index + result = label_index else: try: token_index = gram.OPERATOR_MAP[value] @@ -194,7 +203,10 @@ else: gram.labels.append(token_index) gram.token_ids[token_index] = label_index - return label_index + result = label_index + assert result not in gram.token_to_error_string + gram.token_to_error_string[result] = value + return result def make_first(self, gram, name): original_firsts = self.first[name] diff --git a/pypy/interpreter/pyparser/parser.py b/pypy/interpreter/pyparser/parser.py --- a/pypy/interpreter/pyparser/parser.py +++ b/pypy/interpreter/pyparser/parser.py @@ -1,6 +1,7 @@ """ A CPython inspired RPython parser. """ +from rpython.rlib.objectmodel import not_rpython class Grammar(object): @@ -16,6 +17,7 @@ self.symbol_names = {} self.symbol_to_label = {} self.keyword_ids = {} + self.token_to_error_string = {} self.dfas = [] self.labels = [0] self.token_ids = {} @@ -41,6 +43,27 @@ pass return True +class DFA(object): + def __init__(self, symbol_id, states, first): + self.symbol_id = symbol_id + self.states = states + self.first = self._first_to_string(first) + + def could_match_token(self, label_index): + pos = label_index >> 3 + bit = 1 << (label_index & 0b111) + return bool(ord(self.first[label_index >> 3]) & bit) + + @staticmethod + @not_rpython + def _first_to_string(first): + l = sorted(first.keys()) + b = bytearray(32) + for label_index in l: + pos = label_index >> 3 + bit = 1 << (label_index & 0b111) + b[pos] |= bit + return str(b) class Node(object): @@ -127,14 +150,17 @@ class Nonterminal(AbstractNonterminal): __slots__ = ("_children", ) - def __init__(self, type, children): + def __init__(self, type, children=None): Node.__init__(self, type) + if children is None: + children = [] self._children = children def __repr__(self): return "Nonterminal(type=%s, children=%r)" % (self.type, self._children) def get_child(self, i): + assert self._children is not None return self._children[i] def num_children(self): @@ -168,7 +194,7 @@ class ParseError(Exception): def __init__(self, msg, token_type, value, lineno, column, line, - expected=-1): + expected=-1, expected_str=None): self.msg = msg self.token_type = token_type self.value = value @@ -176,17 +202,41 @@ self.column = column self.line = line self.expected = expected + self.expected_str = expected_str def __str__(self): return "ParserError(%s, %r)" % (self.token_type, self.value) +class StackEntry(object): + def __init__(self, next, dfa, state): + self.next = next + self.dfa = dfa + self.state = state + self.node = None + + def push(self, dfa, state): + return StackEntry(self, dfa, state) + + def pop(self): + return self.next + + def node_append_child(self, child): + node = self.node + if node is None: + self.node = Nonterminal1(self.dfa.symbol_id, child) + elif isinstance(node, Nonterminal1): + newnode = self.node = Nonterminal( + self.dfa.symbol_id, [node._child, child]) + else: + self.node.append_child(child) + + class Parser(object): def __init__(self, grammar): self.grammar = grammar self.root = None - self.stack = None def prepare(self, start=-1): """Setup the parser for parsing. @@ -196,16 +246,15 @@ if start == -1: start = self.grammar.start self.root = None - current_node = Nonterminal(start, []) - self.stack = [] - self.stack.append((self.grammar.dfas[start - 256], 0, current_node)) + self.stack = StackEntry(None, self.grammar.dfas[start - 256], 0) def add_token(self, token_type, value, lineno, column, line): label_index = self.classify(token_type, value, lineno, column, line) sym_id = 0 # for the annotator while True: - dfa, state_index, node = self.stack[-1] - states, first = dfa + dfa = self.stack.dfa + state_index = self.stack.state + states = dfa.states arcs, is_accepting = states[state_index] for i, next_state in arcs: sym_id = self.grammar.labels[i] @@ -217,16 +266,17 @@ # the stack. while state[1] and not state[0]: self.pop() - if not self.stack: + if self.stack is None: # Parsing is done. return True - dfa, state_index, node = self.stack[-1] - state = dfa[0][state_index] + dfa = self.stack.dfa + state_index = self.stack.state + state = dfa.states[state_index] return False elif sym_id >= 256: sub_node_dfa = self.grammar.dfas[sym_id - 256] # Check if this token can start a child node. - if label_index in sub_node_dfa[1]: + if sub_node_dfa.could_match_token(label_index): self.push(sub_node_dfa, next_state, sym_id, lineno, column) break @@ -235,7 +285,7 @@ # state is accepting, it's invalid input. if is_accepting: self.pop() - if not self.stack: + if self.stack is None: raise ParseError("too much input", token_type, value, lineno, column, line) else: @@ -243,10 +293,13 @@ # error. if len(arcs) == 1: expected = sym_id + expected_str = self.grammar.token_to_error_string.get( + arcs[0][0], None) else: expected = -1 + expected_str = None raise ParseError("bad input", token_type, value, lineno, - column, line, expected) + column, line, expected, expected_str) def classify(self, token_type, value, lineno, column, line): """Find the label for a token.""" @@ -262,26 +315,22 @@ def shift(self, next_state, token_type, value, lineno, column): """Shift a non-terminal and prepare for the next state.""" - dfa, state, node = self.stack[-1] new_node = Terminal(token_type, value, lineno, column) - node.append_child(new_node) - self.stack[-1] = (dfa, next_state, node) + self.stack.node_append_child(new_node) + self.stack.state = next_state def push(self, next_dfa, next_state, node_type, lineno, column): """Push a terminal and adjust the current state.""" - dfa, state, node = self.stack[-1] - new_node = Nonterminal(node_type, []) - self.stack[-1] = (dfa, next_state, node) - self.stack.append((next_dfa, 0, new_node)) + self.stack.state = next_state + self.stack = self.stack.push(next_dfa, 0) def pop(self): """Pop an entry off the stack and make its node a child of the last.""" - dfa, state, node = self.stack.pop() + top = self.stack + self.stack = top.pop() + node = top.node + assert node is not None if self.stack: - # we are now done with node, so we can store it more efficiently if - # it has just one child - if node.num_children() == 1: - node = Nonterminal1(node.type, node.get_child(0)) - self.stack[-1][2].append_child(node) + self.stack.node_append_child(node) else: self.root = node diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py --- a/pypy/interpreter/pyparser/pyparse.py +++ b/pypy/interpreter/pyparser/pyparse.py @@ -146,7 +146,11 @@ w_message = space.str(e.get_w_value(space)) raise error.SyntaxError(space.text_w(w_message)) raise + if enc is not None: + compile_info.encoding = enc + return self._parse(textsrc, compile_info) + def _parse(self, textsrc, compile_info): flags = compile_info.flags if explicit_encoding: flags |= consts.PyCF_FOUND_ENCODING @@ -225,6 +229,9 @@ last_value_seen,) else: msg = "invalid syntax" + if e.expected_str is not None: + msg += " (expected '%s')" % e.expected_str + raise new_err(msg, e.lineno, e.column, e.line, compile_info.filename) else: @@ -232,6 +239,4 @@ finally: # Avoid hanging onto the tree. self.root = None - if enc is not None: - compile_info.encoding = enc return tree diff --git a/pypy/interpreter/pyparser/test/targetparse.py b/pypy/interpreter/pyparser/test/targetparse.py new file mode 100644 --- /dev/null +++ b/pypy/interpreter/pyparser/test/targetparse.py @@ -0,0 +1,39 @@ +import sys +import os +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) +print ROOT +sys.path.insert(0, str(ROOT)) +import time +from pypy.interpreter.pyparser import pyparse + + + +with file("../../../rpython/rlib/unicodedata/unicodedb_5_2_0.py") as f: + s = f.read() + +class FakeSpace(object): + pass + +fakespace = FakeSpace() + +def bench(title): + a = time.clock() + info = pyparse.CompileInfo("", "exec") + parser = pyparse.PythonParser(fakespace) + tree = parser._parse(s, info) + b = time.clock() + print title, (b-a) + + +def entry_point(argv): + bench("foo") + + return 0 + +# _____ Define and setup target ___ + +def target(*args): + return entry_point, None + +if __name__ == '__main__': + entry_point(sys.argv) diff --git a/pypy/interpreter/pyparser/test/test_metaparser.py b/pypy/interpreter/pyparser/test/test_metaparser.py --- a/pypy/interpreter/pyparser/test/test_metaparser.py +++ b/pypy/interpreter/pyparser/test/test_metaparser.py @@ -34,8 +34,8 @@ assert len(g.dfas) == 1 eval_sym = g.symbol_ids["eval"] assert g.start == eval_sym - states, first = g.dfas[eval_sym - 256] - assert states == [([(1, 1)], False), ([], True)] + dfa = g.dfas[eval_sym - 256] + assert dfa.states == [([(1, 1)], False), ([], True)] assert g.labels[0] == 0 def test_load_python_grammars(self): @@ -51,7 +51,7 @@ def test_items(self): g = self.gram_for("foo: NAME STRING OP '+'") assert len(g.dfas) == 1 - states = g.dfas[g.symbol_ids["foo"] - 256][0] + states = g.dfas[g.symbol_ids["foo"] - 256].states last = states[0][0][0][1] for state in states[1:-1]: assert last < state[0][0][1] diff --git a/pypy/interpreter/pyparser/test/test_parser.py b/pypy/interpreter/pyparser/test/test_parser.py --- a/pypy/interpreter/pyparser/test/test_parser.py +++ b/pypy/interpreter/pyparser/test/test_parser.py @@ -7,6 +7,12 @@ from pypy.interpreter.pyparser.test.test_metaparser import MyGrammar +def test_char_set(): + first = {5: None, 9: None, 100: None, 255:None} + p = parser.DFA(None, None, first) + for i in range(256): + assert p.could_match_token(i) == (i in first) + class SimpleParser(parser.Parser): def parse(self, input): @@ -55,8 +61,7 @@ n = parser.Terminal(tp, value, 0, 0) else: tp = gram.symbol_ids[data[0]] - children = [] - n = parser.Nonterminal(tp, children) + n = parser.Nonterminal(tp) new_indent = count_indent(line) if new_indent >= last_indent: if new_indent == last_indent and node_stack: @@ -291,3 +296,37 @@ NEWLINE ENDMARKER""" assert tree_from_string(expected, gram) == p.parse("hi 42 end") + + + def test_optimized_terminal(self): + gram = """foo: bar baz 'end' NEWLINE ENDMARKER +bar: NAME +baz: NUMBER +""" + p, gram = self.parser_for(gram, False) + expected = """ + foo + bar + NAME "a_name" + baz + NUMBER "42" + NAME "end" + NEWLINE + ENDMARKER""" + input = "a_name 42 end" + tree = p.parse(input) + assert tree_from_string(expected, gram) == tree + assert isinstance(tree, parser.Nonterminal) + assert isinstance(tree.get_child(0), parser.Nonterminal1) + assert isinstance(tree.get_child(1), parser.Nonterminal1) + + + def test_error_string(self): + p, gram = self.parser_for( + "foo: 'if' NUMBER '+' NUMBER" + ) + info = py.test.raises(parser.ParseError, p.parse, "if 42") + info.value.expected_str is None + info = py.test.raises(parser.ParseError, p.parse, "if 42 42") + info.value.expected_str == '+' + diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py --- a/pypy/interpreter/pyparser/test/test_pyparse.py +++ b/pypy/interpreter/pyparser/test/test_pyparse.py @@ -245,3 +245,12 @@ exc = py.test.raises(SyntaxError, self.parse, input).value assert exc.msg == ("'ascii' codec can't decode byte 0xc3 " "in position 16: ordinal not in range(128)") + + def test_error_forgotten_chars(self): + info = py.test.raises(SyntaxError, self.parse, "if 1\n print 4") + assert "(expected ':')" in info.value.msg + info = py.test.raises(SyntaxError, self.parse, "for i in range(10)\n print i") + assert "(expected ':')" in info.value.msg + info = py.test.raises(SyntaxError, self.parse, "def f:\n print 1") + assert "(expected '(')" in info.value.msg + diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py --- a/pypy/module/cpyext/pyobject.py +++ b/pypy/module/cpyext/pyobject.py @@ -128,9 +128,6 @@ w_type) raise track_reference(space, obj, w_obj) - if w_type.flag_cpytype: - assert isinstance(w_obj, W_BaseCPyObject) - w_obj._cpy_ref = obj return w_obj typedescr_cache = {} diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -13,7 +13,7 @@ ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc, cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc, getbufferproc, ssizessizeobjargproc) -from pypy.module.cpyext.pyobject import make_ref, from_ref, as_pyobj +from pypy.module.cpyext.pyobject import make_ref, from_ref, as_pyobj, decref from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.module.cpyext.memoryobject import fill_Py_buffer from pypy.module.cpyext.state import State @@ -63,6 +63,7 @@ py_args = tuple_from_args_w(space, __args__.arguments_w) w_kwargs = w_kwargs_from_args(space, __args__) res = generic_cpy_call(space, func_init, w_self, py_args, w_kwargs) + decref(space, py_args) if rffi.cast(lltype.Signed, res) == -1: space.fromcache(State).check_and_raise_exception(always=True) return None @@ -232,7 +233,9 @@ func_target = rffi.cast(ternaryfunc, func) py_args = tuple_from_args_w(space, __args__.arguments_w) w_kwargs = w_kwargs_from_args(space, __args__) - return generic_cpy_call(space, func_target, w_self, py_args, w_kwargs) + ret = generic_cpy_call(space, func_target, w_self, py_args, w_kwargs) + decref(space, py_args) + return ret class wrap_ssizessizeobjargproc(W_PyCWrapperObject): def call(self, space, w_self, __args__): diff --git a/pypy/module/fcntl/interp_fcntl.py b/pypy/module/fcntl/interp_fcntl.py --- a/pypy/module/fcntl/interp_fcntl.py +++ b/pypy/module/fcntl/interp_fcntl.py @@ -204,6 +204,7 @@ # XXX this function's interface is a mess. # We try to emulate the behavior of Python >= 2.5 w.r.t. mutate_flag + IOCTL_BUFSZ = 1024 # like cpython fd = space.c_filedescriptor_w(w_fd) op = rffi.cast(rffi.INT, op) # C long => C int @@ -217,15 +218,19 @@ else: arg = rwbuffer.as_str() ll_arg = rffi.str2charp(arg) + to_alloc = max(IOCTL_BUFSZ, len(arg)) try: - rv = ioctl_str(fd, op, ll_arg) - if rv < 0: - raise _get_error(space, "ioctl") - arg = rffi.charpsize2str(ll_arg, len(arg)) - if mutate_flag != 0: - rwbuffer.setslice(0, arg) - return space.newint(rv) - return space.newbytes(arg) + with rffi.scoped_alloc_buffer(to_alloc) as buf: + rffi.c_memcpy(rffi.cast(rffi.VOIDP, buf.raw), + rffi.cast(rffi.VOIDP, ll_arg), len(arg)) + rv = ioctl_str(fd, op, buf.raw) + if rv < 0: + raise _get_error(space, "ioctl") + arg = rffi.charpsize2str(buf.raw, len(arg)) + if mutate_flag != 0: + rwbuffer.setslice(0, arg) + return space.newint(rv) + return space.newbytes(arg) finally: lltype.free(ll_arg, flavor='raw') @@ -241,11 +246,15 @@ raise else: ll_arg = rffi.str2charp(arg) + to_alloc = max(IOCTL_BUFSZ, len(arg)) try: - rv = ioctl_str(fd, op, ll_arg) - if rv < 0: - raise _get_error(space, "ioctl") - arg = rffi.charpsize2str(ll_arg, len(arg)) + with rffi.scoped_alloc_buffer(to_alloc) as buf: + rffi.c_memcpy(rffi.cast(rffi.VOIDP, buf.raw), + rffi.cast(rffi.VOIDP, ll_arg), len(arg)) + rv = ioctl_str(fd, op, buf.raw) + if rv < 0: + raise _get_error(space, "ioctl") + arg = rffi.charpsize2str(buf.raw, len(arg)) return space.newbytes(arg) finally: lltype.free(ll_arg, flavor='raw') diff --git a/pypy/module/parser/pyparser.py b/pypy/module/parser/pyparser.py --- a/pypy/module/parser/pyparser.py +++ b/pypy/module/parser/pyparser.py @@ -153,7 +153,7 @@ # Raise an exception now and be done with it. raise parser_error(space, w_tuple, "Illegal syntax-tree; cannot start with terminal symbol.") - node = pyparse.parser.Nonterminal(type, []) + node = pyparse.parser.Nonterminal(type) build_node_children(space, w_tuple, node, node_state) return node @@ -172,7 +172,7 @@ strn = space.text_w(w_obj) child = pyparse.parser.Terminal(type, strn, node_state.lineno, 0) else: - child = pyparse.parser.Nonterminal(type, []) + child = pyparse.parser.Nonterminal(type) node.append_child(child) if type >= 256: # Nonterminal node build_node_children(space, w_elem, child, node_state) @@ -188,8 +188,7 @@ raise parse_error(space, "Unrecognized node type %d." % type) dfa = parser.grammar.dfas[type] # Run the DFA for this nonterminal - states, first = dfa - arcs, is_accepting = states[0] + arcs, is_accepting = dfa.states[0] for pos in range(tree.num_children()): ch = tree.get_child(pos) for i, next_state in arcs: @@ -199,7 +198,7 @@ if ch.type >= 256: validate_node(space, ch) # Update the state, and move on to the next child. - arcs, is_accepting = states[next_state] + arcs, is_accepting = dfa.states[next_state] break else: raise parse_error(space, "Illegal node") diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py --- a/pypy/module/pypyjit/interp_resop.py +++ b/pypy/module/pypyjit/interp_resop.py @@ -125,8 +125,13 @@ op.getarg(2).getint(), w_greenkey)) elif op.is_guard(): + descr = op.getdescr() + if descr is not None: # can be none in on_abort! + hash = op.getdescr().get_jitcounter_hash() + else: + hash = r_uint(0) l_w.append(GuardOp(name, ofs, logops.repr_of_resop(op), - op.getdescr().get_jitcounter_hash())) + hash)) else: l_w.append(WrappedOp(name, ofs, logops.repr_of_resop(op))) return l_w diff --git a/pypy/module/pypyjit/test/test_jit_hook.py b/pypy/module/pypyjit/test/test_jit_hook.py --- a/pypy/module/pypyjit/test/test_jit_hook.py +++ b/pypy/module/pypyjit/test/test_jit_hook.py @@ -65,6 +65,17 @@ if i != 1: offset[op] = i + oplist_no_descrs = parse(""" + [i1, i2, p2] + i3 = int_add(i1, i2) + debug_merge_point(0, 0, 0, 0, 0, ConstPtr(ptr0)) + guard_nonnull(p2) [] + guard_true(i3) [] + """, namespace={'ptr0': code_gcref}).operations + for op in oplist_no_descrs: + if op.is_guard(): + op.setdescr(None) + class FailDescr(BasicFailDescr): def get_jitcounter_hash(self): from rpython.rlib.rarithmetic import r_uint @@ -101,7 +112,8 @@ def interp_on_abort(): if pypy_hooks.are_hooks_enabled(): pypy_hooks.on_abort(Counters.ABORT_TOO_LONG, pypyjitdriver, - greenkey, 'blah', Logger(MockSD), []) + greenkey, 'blah', Logger(MockSD), + cls.oplist_no_descrs) space = cls.space cls.w_on_compile = space.wrap(interp2app(interp_on_compile)) @@ -111,10 +123,12 @@ cls.w_dmp_num = space.wrap(rop.DEBUG_MERGE_POINT) cls.w_on_optimize = space.wrap(interp2app(interp_on_optimize)) cls.orig_oplist = oplist + cls.orig_oplist_no_descrs = oplist_no_descrs cls.w_sorted_keys = space.wrap(sorted(Counters.counter_names)) def setup_method(self, meth): self.__class__.oplist = self.orig_oplist[:] + self.__class__.oplist_no_descrs = self.orig_oplist_no_descrs[:] def test_on_compile(self): import pypyjit @@ -224,7 +238,12 @@ pypyjit.set_abort_hook(hook) self.on_abort() - assert l == [('pypyjit', 'ABORT_TOO_LONG', [])] + assert len(l) == 1 + name, reason, ops = l[0] + assert name == 'pypyjit' + assert reason == 'ABORT_TOO_LONG' + assert len(ops) == 4 + assert ops[2].hash == 0 def test_creation(self): from pypyjit import ResOperation diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -166,13 +166,14 @@ return handlers_w[signum] -def default_int_handler(space, w_signum, w_frame): +def default_int_handler(space, args_w): """ default_int_handler(...) The default handler for SIGINT installed by Python. It raises KeyboardInterrupt. """ + # issue #2780: accept and ignore any non-keyword arguments raise OperationError(space.w_KeyboardInterrupt, space.w_None) diff --git a/pypy/module/signal/test/test_signal.py b/pypy/module/signal/test/test_signal.py --- a/pypy/module/signal/test/test_signal.py +++ b/pypy/module/signal/test/test_signal.py @@ -261,6 +261,17 @@ finally: signal.signal(signum, oldhandler) + def test_default_int_handler(self): + import signal + for args in [(), (1, 2)]: + try: + signal.default_int_handler(*args) + except KeyboardInterrupt: + pass + else: + raise AssertionError("did not raise!") + + class AppTestSignalSocket: spaceconfig = dict(usemodules=['signal', '_socket']) From pypy.commits at gmail.com Wed Apr 4 12:48:20 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 04 Apr 2018 09:48:20 -0700 (PDT) Subject: [pypy-commit] pypy release-pypy2.7-6.x: start release branch Message-ID: <5ac501d4.91acdf0a.c9f24.1476@mx.google.com> Author: Matti Picus Branch: release-pypy2.7-6.x Changeset: r94230:d2c8301f4536 Date: 2018-04-04 19:04 +0300 http://bitbucket.org/pypy/pypy/changeset/d2c8301f4536/ Log: start release branch diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -29,8 +29,8 @@ #define PY_VERSION "2.7.13" /* PyPy version as a string */ -#define PYPY_VERSION "6.1.0-alpha0" -#define PYPY_VERSION_NUM 0x06010000 +#define PYPY_VERSION "6.0.0" +#define PYPY_VERSION_NUM 0x06000000 /* Defined to mean a PyPy where cpyext holds more regular references to PyObjects, e.g. staying alive as long as the internal PyPy object diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -10,7 +10,7 @@ #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py CPYTHON_API_VERSION = 1013 #XXX # sync with include/modsupport.h -PYPY_VERSION = (6, 1, 0, "alpha", 0) #XXX # sync patchlevel.h +PYPY_VERSION = (6, 0, 0, "final", 0) #XXX # sync patchlevel.h import pypy From pypy.commits at gmail.com Wed Apr 4 12:48:26 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 04 Apr 2018 09:48:26 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: restart whatsnew-pypy3-head.rst part 1 Message-ID: <5ac501da.13151c0a.c7d0d.2c66@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r94233:bf74662ee4fa Date: 2018-04-04 19:44 +0300 http://bitbucket.org/pypy/pypy/changeset/bf74662ee4fa/ Log: restart whatsnew-pypy3-head.rst part 1 diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-6.0.0.rst rename from pypy/doc/whatsnew-pypy3-head.rst rename to pypy/doc/whatsnew-pypy3-6.0.0.rst From pypy.commits at gmail.com Wed Apr 4 12:48:28 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 04 Apr 2018 09:48:28 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: restart whatsnew-pypy3-head.rst part 2 Message-ID: <5ac501dc.c7e2df0a.bd56.c93a@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r94234:cfe5b1a3c30e Date: 2018-04-04 19:46 +0300 http://bitbucket.org/pypy/pypy/changeset/cfe5b1a3c30e/ Log: restart whatsnew-pypy3-head.rst part 2 diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-pypy3-head.rst @@ -0,0 +1,7 @@ +======================== +What's new in PyPy3 6.0+ +======================== + +.. this is the revision after release-pypy3.5-v6.0 +.. startrev: bf74662ee4fa + From pypy.commits at gmail.com Wed Apr 4 12:48:30 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 04 Apr 2018 09:48:30 -0700 (PDT) Subject: [pypy-commit] pypy release-pypy3.5-6.x: start release branch Message-ID: <5ac501de.079e1c0a.d49e1.6074@mx.google.com> Author: Matti Picus Branch: release-pypy3.5-6.x Changeset: r94235:bfd21fbbe693 Date: 2018-04-04 19:46 +0300 http://bitbucket.org/pypy/pypy/changeset/bfd21fbbe693/ Log: start release branch diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -29,8 +29,8 @@ #define PY_VERSION "3.5.3" /* PyPy version as a string */ -#define PYPY_VERSION "6.1.0-alpha0" -#define PYPY_VERSION_NUM 0x06010000 +#define PYPY_VERSION "6.0.0" +#define PYPY_VERSION_NUM 0x06000000 /* Defined to mean a PyPy where cpyext holds more regular references to PyObjects, e.g. staying alive as long as the internal PyPy object diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -10,7 +10,7 @@ #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py CPYTHON_API_VERSION = 1013 #XXX # sync with include/modsupport.h -PYPY_VERSION = (6, 1, 0, "alpha", 0) #XXX # sync patchlevel.h +PYPY_VERSION = (6, 0, 0, "final", 0) #XXX # sync patchlevel.h import pypy From pypy.commits at gmail.com Wed Apr 4 15:37:59 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 04 Apr 2018 12:37:59 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-subclass-setattr: backed out changeset 36ff2c150968, try a different approach (arigato) Message-ID: <5ac52997.d6b11c0a.afba.7713@mx.google.com> Author: Matti Picus Branch: cpyext-subclass-setattr Changeset: r94236:98123cec9206 Date: 2018-04-04 19:50 +0300 http://bitbucket.org/pypy/pypy/changeset/98123cec9206/ Log: backed out changeset 36ff2c150968, try a different approach (arigato) diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -858,7 +858,6 @@ track_reference(space, py_obj, w_obj) # __init__ wraps all slotdefs functions from py_type via add_operators w_obj.__init__(space, py_type) - w_obj.flag_cpytype = True w_obj.ready() finish_type_2(space, py_type, w_obj) diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py --- a/pypy/objspace/std/typeobject.py +++ b/pypy/objspace/std/typeobject.py @@ -1199,9 +1199,7 @@ for w_base in w_self.bases_w: if not isinstance(w_base, W_TypeObject): continue - # set only in cpyext type_realize, so user-created types will - # still use w_root_attach_pyobj (in cpyext.pyobject) - #w_self.flag_cpytype |= w_base.flag_cpytype + w_self.flag_cpytype |= w_base.flag_cpytype w_self.flag_abstract |= w_base.flag_abstract if w_self.flag_map_or_seq == '?': w_self.flag_map_or_seq = w_base.flag_map_or_seq From pypy.commits at gmail.com Wed Apr 4 15:38:01 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 04 Apr 2018 12:38:01 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-subclass-setattr: rework _cpyext_attach_pyobj (arigato) Message-ID: <5ac52999.7a86df0a.4df23.859d@mx.google.com> Author: Matti Picus Branch: cpyext-subclass-setattr Changeset: r94237:ccfd28850680 Date: 2018-04-04 22:36 +0300 http://bitbucket.org/pypy/pypy/changeset/ccfd28850680/ Log: rework _cpyext_attach_pyobj (arigato) diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py --- a/pypy/module/cpyext/pyobject.py +++ b/pypy/module/cpyext/pyobject.py @@ -60,10 +60,10 @@ def _cpyext_attach_pyobj(self, space, py_obj): self._cpy_ref = py_obj - rawrefcount.create_link_pyobj(self, py_obj) + rawrefcount.create_link_pypy(self, py_obj) cls._cpyext_attach_pyobj = _cpyext_attach_pyobj -add_direct_pyobj_storage(W_BaseCPyObject) +add_direct_pyobj_storage(W_BaseCPyObject) add_direct_pyobj_storage(W_TypeObject) add_direct_pyobj_storage(W_NoneObject) add_direct_pyobj_storage(W_BoolObject) diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -1,6 +1,6 @@ import os -from rpython.rlib import jit +from rpython.rlib import jit, rawrefcount from rpython.rlib.objectmodel import specialize, we_are_translated from rpython.rtyper.lltypesystem import rffi, lltype @@ -517,6 +517,10 @@ self.w_doc = space.newtext( rffi.charp2str(cts.cast('char*', pto.c_tp_doc))) + def _cpyext_attach_pyobj(self, space, py_obj): + self._cpy_ref = py_obj + rawrefcount.create_link_pyobj(self, py_obj) + @bootstrap_function def init_typeobject(space): make_typedescr(space.w_type.layout.typedef, From pypy.commits at gmail.com Wed Apr 4 15:38:03 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 04 Apr 2018 12:38:03 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-subclass-setattr: remove unused variable Message-ID: <5ac5299b.c2101c0a.2451a.9953@mx.google.com> Author: Matti Picus Branch: cpyext-subclass-setattr Changeset: r94238:2f6fa12ef5e6 Date: 2018-04-04 22:37 +0300 http://bitbucket.org/pypy/pypy/changeset/2f6fa12ef5e6/ Log: remove unused variable diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -781,7 +781,6 @@ try: w_obj = _type_realize(space, py_obj) finally: - name = rffi.charp2str(cts.cast('char*', pto.c_tp_name)) pto.c_tp_flags &= ~Py_TPFLAGS_READYING pto.c_tp_flags |= Py_TPFLAGS_READY return w_obj From pypy.commits at gmail.com Wed Apr 4 16:14:47 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 04 Apr 2018 13:14:47 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-subclass-setattr: remove unused variable Message-ID: <5ac53237.69b9df0a.1ffc5.a910@mx.google.com> Author: Matti Picus Branch: cpyext-subclass-setattr Changeset: r94239:a7f1ba785e54 Date: 2018-04-04 22:55 +0300 http://bitbucket.org/pypy/pypy/changeset/a7f1ba785e54/ Log: remove unused variable diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -887,7 +887,6 @@ base = pto.c_tp_base base_pyo = rffi.cast(PyObject, pto.c_tp_base) if base and not base.c_tp_flags & Py_TPFLAGS_READY: - name = rffi.charp2str(cts.cast('char*', base.c_tp_name)) type_realize(space, base_pyo) if base and not pto.c_ob_type: # will be filled later pto.c_ob_type = base.c_ob_type From pypy.commits at gmail.com Wed Apr 4 16:14:50 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 04 Apr 2018 13:14:50 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-subclass-setattr: add a passing test Message-ID: <5ac5323a.c7851c0a.776e5.b32a@mx.google.com> Author: Matti Picus Branch: cpyext-subclass-setattr Changeset: r94240:85beaaee87c8 Date: 2018-04-04 23:14 +0300 http://bitbucket.org/pypy/pypy/changeset/85beaaee87c8/ Log: add a passing test diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -5,7 +5,7 @@ from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.api import generic_cpy_call from pypy.module.cpyext.pyobject import make_ref, from_ref, decref, as_pyobj -from pypy.module.cpyext.typeobject import PyTypeObjectPtr +from pypy.module.cpyext.typeobject import PyTypeObjectPtr, W_PyCTypeObject class AppTestTypeObject(AppTestCpythonExtensionBase): @@ -569,6 +569,23 @@ assert w_obj is None assert api.PyErr_Occurred() is None + def test_subclass_not_PyCTypeObject(self, space, api): + pyobj = make_ref(space, api.PyLong_Type) + py_type = rffi.cast(PyTypeObjectPtr, pyobj) + w_pyclass = W_PyCTypeObject(space, py_type) + w_class = space.appexec([w_pyclass], """(base): + class Sub(base): + def addattrib(self, value): + self.attrib = value + return Sub + """) + assert w_pyclass in w_class.mro_w + assert isinstance(w_pyclass, W_PyCTypeObject) + assert not isinstance(w_class, W_PyCTypeObject) + assert w_pyclass.is_cpytype() + # XXX document the current status, not clear if this is desirable + assert w_class.is_cpytype() + class AppTestSlots(AppTestCpythonExtensionBase): def setup_class(cls): From pypy.commits at gmail.com Wed Apr 4 17:34:54 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 04 Apr 2018 14:34:54 -0700 (PDT) Subject: [pypy-commit] pypy default: start release notes (up to 24.2.2018), rename whatsnew to fit convention Message-ID: <5ac544fe.902e1c0a.d94b2.d560@mx.google.com> Author: Matti Picus Branch: Changeset: r94241:dbe5e735e9f7 Date: 2018-04-04 23:49 +0300 http://bitbucket.org/pypy/pypy/changeset/dbe5e735e9f7/ Log: start release notes (up to 24.2.2018), rename whatsnew to fit convention diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-v6.0.0.rst release-v5.10.1.rst release-v5.10.0.rst release-v5.9.0.rst diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -7,6 +7,8 @@ .. toctree:: whatsnew-head.rst + whatsnew-pypy2-6.0.0.rst + whatsnew-pypy2-5.10.0.rst whatsnew-pypy2-5.10.0.rst whatsnew-pypy2-5.9.0.rst whatsnew-pypy2-5.8.0.rst diff --git a/pypy/doc/release-v6.0.0.rst b/pypy/doc/release-v6.0.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-v6.0.0.rst @@ -0,0 +1,89 @@ +====================================== +PyPy2.7 and PyPy3.5 v6.0 dual release +====================================== + +The PyPy team is proud to release both PyPy2.7 v6.0 (an interpreter supporting +Python 2.7 syntax), and a PyPy3.5 v6.0 (an interpreter supporting Python +3.5 syntax). The two releases are both based on much the same codebase, thus +the dual release. + +This release is a feature release following our previous 5.10 incremental +release in late December 2017, with many improvements in the C-API +compatability layer and other improvements in speed and CPython compatibility. +Since the changes affect the included python development header files, all +c-extension modules must be recompiled for this version. + +The Windows PyPy3.5 release is still considered beta-quality. There are issues +with unicode handling especially around system calls and c-extensions. + +The Matplotlib TkAgg backend now works with PyPy. PyGame and pygtk also now can +work with PyPy. + +As always, this release is 100% compatible with the previous one and fixed +several issues and bugs raised by the growing community of PyPy users. +We strongly recommend updating. + +The utf8 branch that changes internal representation of unicode to utf8 did not +make it into the release. We also began working on a Python3.6 implementation, +help is welcome. + +We updated the cffi module included in PyPy to version 1.11.4 + +You can download the v6.0 releases here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project. + +We would also like to thank our contributors and +encourage new people to join the project. PyPy has many +layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation +improvements, tweaking popular `modules`_ to run on pypy, or general `help`_ +with making RPython's JIT even better. + +.. _`PyPy`: index.html +.. _`RPython`: https://rpython.readthedocs.org +.. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly +.. _`help`: project-ideas.html + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7 and CPython 3.5. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +The PyPy release supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html + +Changelog +========= + +* support ``hastzinfo`` and ``tzinfo`` in the C-API ``PyDateTime*`` structures +* datetime.h is now more similar to CPython +* We now support ``PyUnicode_AsUTF{16,32}String``, ``_PyLong_AsByteArray``, + ``_PyLong_AsByteArrayO``, +* PyPy3.5 on Windows is compiled with the Microsoft Visual Compiler v14, like + CPython +* Fix performance of attribute lookup when more than 80 attributes are used +* Improve performance on passing built-in types to C-API C code +* Improve the performance of datetime and timedelta by skipping the consistency + checks of the datetime values (they are correct by construction) +* Improve handling of ``bigint`` s, including fixing ``int_divmod`` +* Improve reporting of GC statistics +* Accept unicode filenames in ``dbm.open()`` diff --git a/pypy/doc/whatsnew-6.0.0.rst b/pypy/doc/whatsnew-pypy2-6.0.0.rst rename from pypy/doc/whatsnew-6.0.0.rst rename to pypy/doc/whatsnew-pypy2-6.0.0.rst From pypy.commits at gmail.com Thu Apr 5 01:44:36 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 04 Apr 2018 22:44:36 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: fix merge Message-ID: <5ac5b7c4.f4a0df0a.bcdf8.2f44@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r94242:4e6fcbd80661 Date: 2018-04-05 08:43 +0300 http://bitbucket.org/pypy/pypy/changeset/4e6fcbd80661/ Log: fix merge diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py --- a/pypy/interpreter/pyparser/pyparse.py +++ b/pypy/interpreter/pyparser/pyparse.py @@ -148,12 +148,12 @@ raise if enc is not None: compile_info.encoding = enc + if explicit_encoding: + compile_info.flags |= consts.PyCF_FOUND_ENCODING return self._parse(textsrc, compile_info) def _parse(self, textsrc, compile_info): flags = compile_info.flags - if explicit_encoding: - flags |= consts.PyCF_FOUND_ENCODING # The tokenizer is very picky about how it wants its input. source_lines = textsrc.splitlines(True) From pypy.commits at gmail.com Thu Apr 5 04:38:06 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 05 Apr 2018 01:38:06 -0700 (PDT) Subject: [pypy-commit] pypy gc-hooks: hg merge default Message-ID: <5ac5e06e.0ea6df0a.b8f8.f08c@mx.google.com> Author: Antonio Cuni Branch: gc-hooks Changeset: r94243:00c8c55ee465 Date: 2018-04-05 09:37 +0100 http://bitbucket.org/pypy/pypy/changeset/00c8c55ee465/ Log: hg merge default diff --git a/lib-python/2.7/re.py b/lib-python/2.7/re.py --- a/lib-python/2.7/re.py +++ b/lib-python/2.7/re.py @@ -225,7 +225,7 @@ _pattern_type = type(sre_compile.compile("", 0)) -_MAXCACHE = 100 +_MAXCACHE = 1000 def _compile(*key): # internal: compile pattern diff --git a/lib-python/2.7/test/test_genexps.py b/lib-python/2.7/test/test_genexps.py --- a/lib-python/2.7/test/test_genexps.py +++ b/lib-python/2.7/test/test_genexps.py @@ -87,7 +87,7 @@ >>> dict(a = i for i in xrange(10)) Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: invalid syntax (expected ')') Verify that parenthesis are required when used as a keyword argument value diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -79,3 +79,21 @@ - in rare "trace is too long" situations, the JIT could break behaviour arbitrarily. + +.. branch: jit-hooks-can-be-disabled + +Be more efficient about JIT hooks. Make it possible for the frontend to declare +that jit hooks are currently not enabled at all. in that case, the list of ops +does not have to be created in the case of the on_abort hook (which is +expensive). + + +.. branch: pyparser-improvements + +Improve speed of Python parser, improve ParseError messages slightly. + +.. branch: ioctl-arg-size + +Work around possible bugs in upstream ioctl users, like CPython allocate at +least 1024 bytes for the arg in calls to ``ioctl(fd, request, arg)``. Fixes +issue #2776 diff --git a/pypy/interpreter/pyparser/metaparser.py b/pypy/interpreter/pyparser/metaparser.py --- a/pypy/interpreter/pyparser/metaparser.py +++ b/pypy/interpreter/pyparser/metaparser.py @@ -147,8 +147,10 @@ for label, next in state.arcs.iteritems(): arcs.append((self.make_label(gram, label), dfa.index(next))) states.append((arcs, state.is_final)) - gram.dfas.append((states, self.make_first(gram, name))) - assert len(gram.dfas) - 1 == gram.symbol_ids[name] - 256 + symbol_id = gram.symbol_ids[name] + dfa = parser.DFA(symbol_id, states, self.make_first(gram, name)) + gram.dfas.append(dfa) + assert len(gram.dfas) - 1 == symbol_id - 256 gram.start = gram.symbol_ids[self.start_symbol] return gram @@ -162,6 +164,13 @@ else: gram.labels.append(gram.symbol_ids[label]) gram.symbol_to_label[label] = label_index + first = self.first[label] + if len(first) == 1: + first, = first + if not first[0].isupper(): + first = first.strip("\"'") + assert label_index not in gram.token_to_error_string + gram.token_to_error_string[label_index] = first return label_index elif label.isupper(): token_index = gram.TOKENS[label] @@ -183,7 +192,7 @@ else: gram.labels.append(gram.KEYWORD_TOKEN) gram.keyword_ids[value] = label_index - return label_index + result = label_index else: try: token_index = gram.OPERATOR_MAP[value] @@ -194,7 +203,10 @@ else: gram.labels.append(token_index) gram.token_ids[token_index] = label_index - return label_index + result = label_index + assert result not in gram.token_to_error_string + gram.token_to_error_string[result] = value + return result def make_first(self, gram, name): original_firsts = self.first[name] diff --git a/pypy/interpreter/pyparser/parser.py b/pypy/interpreter/pyparser/parser.py --- a/pypy/interpreter/pyparser/parser.py +++ b/pypy/interpreter/pyparser/parser.py @@ -1,6 +1,7 @@ """ A CPython inspired RPython parser. """ +from rpython.rlib.objectmodel import not_rpython class Grammar(object): @@ -16,6 +17,7 @@ self.symbol_names = {} self.symbol_to_label = {} self.keyword_ids = {} + self.token_to_error_string = {} self.dfas = [] self.labels = [0] self.token_ids = {} @@ -41,6 +43,27 @@ pass return True +class DFA(object): + def __init__(self, symbol_id, states, first): + self.symbol_id = symbol_id + self.states = states + self.first = self._first_to_string(first) + + def could_match_token(self, label_index): + pos = label_index >> 3 + bit = 1 << (label_index & 0b111) + return bool(ord(self.first[label_index >> 3]) & bit) + + @staticmethod + @not_rpython + def _first_to_string(first): + l = sorted(first.keys()) + b = bytearray(32) + for label_index in l: + pos = label_index >> 3 + bit = 1 << (label_index & 0b111) + b[pos] |= bit + return str(b) class Node(object): @@ -127,14 +150,17 @@ class Nonterminal(AbstractNonterminal): __slots__ = ("_children", ) - def __init__(self, type, children): + def __init__(self, type, children=None): Node.__init__(self, type) + if children is None: + children = [] self._children = children def __repr__(self): return "Nonterminal(type=%s, children=%r)" % (self.type, self._children) def get_child(self, i): + assert self._children is not None return self._children[i] def num_children(self): @@ -168,7 +194,7 @@ class ParseError(Exception): def __init__(self, msg, token_type, value, lineno, column, line, - expected=-1): + expected=-1, expected_str=None): self.msg = msg self.token_type = token_type self.value = value @@ -176,17 +202,41 @@ self.column = column self.line = line self.expected = expected + self.expected_str = expected_str def __str__(self): return "ParserError(%s, %r)" % (self.token_type, self.value) +class StackEntry(object): + def __init__(self, next, dfa, state): + self.next = next + self.dfa = dfa + self.state = state + self.node = None + + def push(self, dfa, state): + return StackEntry(self, dfa, state) + + def pop(self): + return self.next + + def node_append_child(self, child): + node = self.node + if node is None: + self.node = Nonterminal1(self.dfa.symbol_id, child) + elif isinstance(node, Nonterminal1): + newnode = self.node = Nonterminal( + self.dfa.symbol_id, [node._child, child]) + else: + self.node.append_child(child) + + class Parser(object): def __init__(self, grammar): self.grammar = grammar self.root = None - self.stack = None def prepare(self, start=-1): """Setup the parser for parsing. @@ -196,16 +246,15 @@ if start == -1: start = self.grammar.start self.root = None - current_node = Nonterminal(start, []) - self.stack = [] - self.stack.append((self.grammar.dfas[start - 256], 0, current_node)) + self.stack = StackEntry(None, self.grammar.dfas[start - 256], 0) def add_token(self, token_type, value, lineno, column, line): label_index = self.classify(token_type, value, lineno, column, line) sym_id = 0 # for the annotator while True: - dfa, state_index, node = self.stack[-1] - states, first = dfa + dfa = self.stack.dfa + state_index = self.stack.state + states = dfa.states arcs, is_accepting = states[state_index] for i, next_state in arcs: sym_id = self.grammar.labels[i] @@ -217,16 +266,17 @@ # the stack. while state[1] and not state[0]: self.pop() - if not self.stack: + if self.stack is None: # Parsing is done. return True - dfa, state_index, node = self.stack[-1] - state = dfa[0][state_index] + dfa = self.stack.dfa + state_index = self.stack.state + state = dfa.states[state_index] return False elif sym_id >= 256: sub_node_dfa = self.grammar.dfas[sym_id - 256] # Check if this token can start a child node. - if label_index in sub_node_dfa[1]: + if sub_node_dfa.could_match_token(label_index): self.push(sub_node_dfa, next_state, sym_id, lineno, column) break @@ -235,7 +285,7 @@ # state is accepting, it's invalid input. if is_accepting: self.pop() - if not self.stack: + if self.stack is None: raise ParseError("too much input", token_type, value, lineno, column, line) else: @@ -243,10 +293,13 @@ # error. if len(arcs) == 1: expected = sym_id + expected_str = self.grammar.token_to_error_string.get( + arcs[0][0], None) else: expected = -1 + expected_str = None raise ParseError("bad input", token_type, value, lineno, - column, line, expected) + column, line, expected, expected_str) def classify(self, token_type, value, lineno, column, line): """Find the label for a token.""" @@ -262,26 +315,22 @@ def shift(self, next_state, token_type, value, lineno, column): """Shift a non-terminal and prepare for the next state.""" - dfa, state, node = self.stack[-1] new_node = Terminal(token_type, value, lineno, column) - node.append_child(new_node) - self.stack[-1] = (dfa, next_state, node) + self.stack.node_append_child(new_node) + self.stack.state = next_state def push(self, next_dfa, next_state, node_type, lineno, column): """Push a terminal and adjust the current state.""" - dfa, state, node = self.stack[-1] - new_node = Nonterminal(node_type, []) - self.stack[-1] = (dfa, next_state, node) - self.stack.append((next_dfa, 0, new_node)) + self.stack.state = next_state + self.stack = self.stack.push(next_dfa, 0) def pop(self): """Pop an entry off the stack and make its node a child of the last.""" - dfa, state, node = self.stack.pop() + top = self.stack + self.stack = top.pop() + node = top.node + assert node is not None if self.stack: - # we are now done with node, so we can store it more efficiently if - # it has just one child - if node.num_children() == 1: - node = Nonterminal1(node.type, node.get_child(0)) - self.stack[-1][2].append_child(node) + self.stack.node_append_child(node) else: self.root = node diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py --- a/pypy/interpreter/pyparser/pyparse.py +++ b/pypy/interpreter/pyparser/pyparse.py @@ -132,7 +132,11 @@ w_message = space.str(e.get_w_value(space)) raise error.SyntaxError(space.text_w(w_message)) raise + if enc is not None: + compile_info.encoding = enc + return self._parse(textsrc, compile_info) + def _parse(self, textsrc, compile_info): flags = compile_info.flags # The tokenizer is very picky about how it wants its input. @@ -181,6 +185,9 @@ else: new_err = error.SyntaxError msg = "invalid syntax" + if e.expected_str is not None: + msg += " (expected '%s')" % e.expected_str + raise new_err(msg, e.lineno, e.column, e.line, compile_info.filename) else: @@ -188,6 +195,4 @@ finally: # Avoid hanging onto the tree. self.root = None - if enc is not None: - compile_info.encoding = enc return tree diff --git a/pypy/interpreter/pyparser/test/targetparse.py b/pypy/interpreter/pyparser/test/targetparse.py new file mode 100644 --- /dev/null +++ b/pypy/interpreter/pyparser/test/targetparse.py @@ -0,0 +1,39 @@ +import sys +import os +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) +print ROOT +sys.path.insert(0, str(ROOT)) +import time +from pypy.interpreter.pyparser import pyparse + + + +with file("../../../rpython/rlib/unicodedata/unicodedb_5_2_0.py") as f: + s = f.read() + +class FakeSpace(object): + pass + +fakespace = FakeSpace() + +def bench(title): + a = time.clock() + info = pyparse.CompileInfo("", "exec") + parser = pyparse.PythonParser(fakespace) + tree = parser._parse(s, info) + b = time.clock() + print title, (b-a) + + +def entry_point(argv): + bench("foo") + + return 0 + +# _____ Define and setup target ___ + +def target(*args): + return entry_point, None + +if __name__ == '__main__': + entry_point(sys.argv) diff --git a/pypy/interpreter/pyparser/test/test_metaparser.py b/pypy/interpreter/pyparser/test/test_metaparser.py --- a/pypy/interpreter/pyparser/test/test_metaparser.py +++ b/pypy/interpreter/pyparser/test/test_metaparser.py @@ -34,8 +34,8 @@ assert len(g.dfas) == 1 eval_sym = g.symbol_ids["eval"] assert g.start == eval_sym - states, first = g.dfas[eval_sym - 256] - assert states == [([(1, 1)], False), ([], True)] + dfa = g.dfas[eval_sym - 256] + assert dfa.states == [([(1, 1)], False), ([], True)] assert g.labels[0] == 0 def test_load_python_grammars(self): @@ -51,7 +51,7 @@ def test_items(self): g = self.gram_for("foo: NAME STRING OP '+'") assert len(g.dfas) == 1 - states = g.dfas[g.symbol_ids["foo"] - 256][0] + states = g.dfas[g.symbol_ids["foo"] - 256].states last = states[0][0][0][1] for state in states[1:-1]: assert last < state[0][0][1] diff --git a/pypy/interpreter/pyparser/test/test_parser.py b/pypy/interpreter/pyparser/test/test_parser.py --- a/pypy/interpreter/pyparser/test/test_parser.py +++ b/pypy/interpreter/pyparser/test/test_parser.py @@ -7,6 +7,12 @@ from pypy.interpreter.pyparser.test.test_metaparser import MyGrammar +def test_char_set(): + first = {5: None, 9: None, 100: None, 255:None} + p = parser.DFA(None, None, first) + for i in range(256): + assert p.could_match_token(i) == (i in first) + class SimpleParser(parser.Parser): def parse(self, input): @@ -55,8 +61,7 @@ n = parser.Terminal(tp, value, 0, 0) else: tp = gram.symbol_ids[data[0]] - children = [] - n = parser.Nonterminal(tp, children) + n = parser.Nonterminal(tp) new_indent = count_indent(line) if new_indent >= last_indent: if new_indent == last_indent and node_stack: @@ -291,3 +296,37 @@ NEWLINE ENDMARKER""" assert tree_from_string(expected, gram) == p.parse("hi 42 end") + + + def test_optimized_terminal(self): + gram = """foo: bar baz 'end' NEWLINE ENDMARKER +bar: NAME +baz: NUMBER +""" + p, gram = self.parser_for(gram, False) + expected = """ + foo + bar + NAME "a_name" + baz + NUMBER "42" + NAME "end" + NEWLINE + ENDMARKER""" + input = "a_name 42 end" + tree = p.parse(input) + assert tree_from_string(expected, gram) == tree + assert isinstance(tree, parser.Nonterminal) + assert isinstance(tree.get_child(0), parser.Nonterminal1) + assert isinstance(tree.get_child(1), parser.Nonterminal1) + + + def test_error_string(self): + p, gram = self.parser_for( + "foo: 'if' NUMBER '+' NUMBER" + ) + info = py.test.raises(parser.ParseError, p.parse, "if 42") + info.value.expected_str is None + info = py.test.raises(parser.ParseError, p.parse, "if 42 42") + info.value.expected_str == '+' + diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py --- a/pypy/interpreter/pyparser/test/test_pyparse.py +++ b/pypy/interpreter/pyparser/test/test_pyparse.py @@ -165,3 +165,11 @@ for linefeed in ["\r\n","\r"]: tree = self.parse(fmt % linefeed) assert expected_tree == tree + + def test_error_forgotten_chars(self): + info = py.test.raises(SyntaxError, self.parse, "if 1\n print 4") + assert "(expected ':')" in info.value.msg + info = py.test.raises(SyntaxError, self.parse, "for i in range(10)\n print i") + assert "(expected ':')" in info.value.msg + info = py.test.raises(SyntaxError, self.parse, "def f:\n print 1") + assert "(expected '(')" in info.value.msg diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py --- a/pypy/module/cpyext/pyobject.py +++ b/pypy/module/cpyext/pyobject.py @@ -128,9 +128,6 @@ w_type) raise track_reference(space, obj, w_obj) - if w_type.flag_cpytype: - assert isinstance(w_obj, W_BaseCPyObject) - w_obj._cpy_ref = obj return w_obj typedescr_cache = {} diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -13,7 +13,7 @@ ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc, cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc, readbufferproc, getbufferproc, ssizessizeobjargproc) -from pypy.module.cpyext.pyobject import make_ref, from_ref, as_pyobj +from pypy.module.cpyext.pyobject import make_ref, from_ref, as_pyobj, decref from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.module.cpyext.memoryobject import fill_Py_buffer from pypy.module.cpyext.state import State @@ -63,6 +63,7 @@ py_args = tuple_from_args_w(space, __args__.arguments_w) w_kwargs = w_kwargs_from_args(space, __args__) res = generic_cpy_call(space, func_init, w_self, py_args, w_kwargs) + decref(space, py_args) if rffi.cast(lltype.Signed, res) == -1: space.fromcache(State).check_and_raise_exception(always=True) return None @@ -244,7 +245,9 @@ func_target = rffi.cast(ternaryfunc, func) py_args = tuple_from_args_w(space, __args__.arguments_w) w_kwargs = w_kwargs_from_args(space, __args__) - return generic_cpy_call(space, func_target, w_self, py_args, w_kwargs) + ret = generic_cpy_call(space, func_target, w_self, py_args, w_kwargs) + decref(space, py_args) + return ret class wrap_ssizessizeobjargproc(W_PyCWrapperObject): def call(self, space, w_self, __args__): diff --git a/pypy/module/fcntl/interp_fcntl.py b/pypy/module/fcntl/interp_fcntl.py --- a/pypy/module/fcntl/interp_fcntl.py +++ b/pypy/module/fcntl/interp_fcntl.py @@ -204,6 +204,7 @@ # XXX this function's interface is a mess. # We try to emulate the behavior of Python >= 2.5 w.r.t. mutate_flag + IOCTL_BUFSZ = 1024 # like cpython fd = space.c_filedescriptor_w(w_fd) op = rffi.cast(rffi.INT, op) # C long => C int @@ -216,15 +217,19 @@ else: arg = rwbuffer.as_str() ll_arg = rffi.str2charp(arg) + to_alloc = max(IOCTL_BUFSZ, len(arg)) try: - rv = ioctl_str(fd, op, ll_arg) - if rv < 0: - raise _get_error(space, "ioctl") - arg = rffi.charpsize2str(ll_arg, len(arg)) - if mutate_flag != 0: - rwbuffer.setslice(0, arg) - return space.newint(rv) - return space.newbytes(arg) + with rffi.scoped_alloc_buffer(to_alloc) as buf: + rffi.c_memcpy(rffi.cast(rffi.VOIDP, buf.raw), + rffi.cast(rffi.VOIDP, ll_arg), len(arg)) + rv = ioctl_str(fd, op, buf.raw) + if rv < 0: + raise _get_error(space, "ioctl") + arg = rffi.charpsize2str(buf.raw, len(arg)) + if mutate_flag != 0: + rwbuffer.setslice(0, arg) + return space.newint(rv) + return space.newbytes(arg) finally: lltype.free(ll_arg, flavor='raw') @@ -240,11 +245,15 @@ raise else: ll_arg = rffi.str2charp(arg) + to_alloc = max(IOCTL_BUFSZ, len(arg)) try: - rv = ioctl_str(fd, op, ll_arg) - if rv < 0: - raise _get_error(space, "ioctl") - arg = rffi.charpsize2str(ll_arg, len(arg)) + with rffi.scoped_alloc_buffer(to_alloc) as buf: + rffi.c_memcpy(rffi.cast(rffi.VOIDP, buf.raw), + rffi.cast(rffi.VOIDP, ll_arg), len(arg)) + rv = ioctl_str(fd, op, buf.raw) + if rv < 0: + raise _get_error(space, "ioctl") + arg = rffi.charpsize2str(buf.raw, len(arg)) return space.newbytes(arg) finally: lltype.free(ll_arg, flavor='raw') diff --git a/pypy/module/parser/pyparser.py b/pypy/module/parser/pyparser.py --- a/pypy/module/parser/pyparser.py +++ b/pypy/module/parser/pyparser.py @@ -152,7 +152,7 @@ # Raise an exception now and be done with it. raise parser_error(space, w_tuple, "Illegal syntax-tree; cannot start with terminal symbol.") - node = pyparse.parser.Nonterminal(type, []) + node = pyparse.parser.Nonterminal(type) build_node_children(space, w_tuple, node, node_state) return node @@ -171,7 +171,7 @@ strn = space.text_w(w_obj) child = pyparse.parser.Terminal(type, strn, node_state.lineno, 0) else: - child = pyparse.parser.Nonterminal(type, []) + child = pyparse.parser.Nonterminal(type) node.append_child(child) if type >= 256: # Nonterminal node build_node_children(space, w_elem, child, node_state) @@ -187,8 +187,7 @@ raise parse_error(space, "Unrecognized node type %d." % type) dfa = parser.grammar.dfas[type] # Run the DFA for this nonterminal - states, first = dfa - arcs, is_accepting = states[0] + arcs, is_accepting = dfa.states[0] for pos in range(tree.num_children()): ch = tree.get_child(pos) for i, next_state in arcs: @@ -198,7 +197,7 @@ if ch.type >= 256: validate_node(space, ch) # Update the state, and move on to the next child. - arcs, is_accepting = states[next_state] + arcs, is_accepting = dfa.states[next_state] break else: raise parse_error(space, "Illegal node") diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py --- a/pypy/module/pypyjit/interp_resop.py +++ b/pypy/module/pypyjit/interp_resop.py @@ -113,14 +113,13 @@ ofs = ops_offset.get(op, 0) num = op.getopnum() name = op.getopname() - repr = logops.repr_of_resop(op) if num == rop.DEBUG_MERGE_POINT: jd_sd = jitdrivers_sd[op.getarg(0).getint()] greenkey = op.getarglist()[3:] repr = jd_sd.warmstate.get_location_str(greenkey) w_greenkey = wrap_greenkey(space, jd_sd.jitdriver, greenkey, repr) l_w.append(DebugMergePoint(space, name, - repr, + logops.repr_of_resop(op), jd_sd.jitdriver.name, op.getarg(1).getint(), op.getarg(2).getint(), @@ -130,10 +129,11 @@ if descr is not None: # can be none in on_abort! hash = op.getdescr().get_jitcounter_hash() else: - hash = -1 - l_w.append(GuardOp(name, ofs, repr, hash)) + hash = r_uint(0) + l_w.append(GuardOp(name, ofs, logops.repr_of_resop(op), + hash)) else: - l_w.append(WrappedOp(name, ofs, repr)) + l_w.append(WrappedOp(name, ofs, logops.repr_of_resop(op))) return l_w @unwrap_spec(offset=int, repr='text', name='text') diff --git a/pypy/module/pypyjit/test/test_jit_hook.py b/pypy/module/pypyjit/test/test_jit_hook.py --- a/pypy/module/pypyjit/test/test_jit_hook.py +++ b/pypy/module/pypyjit/test/test_jit_hook.py @@ -242,6 +242,7 @@ assert name == 'pypyjit' assert reason == 'ABORT_TOO_LONG' assert len(ops) == 4 + assert ops[2].hash == 0 def test_creation(self): from pypyjit import ResOperation diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -166,13 +166,14 @@ return handlers_w[signum] -def default_int_handler(space, w_signum, w_frame): +def default_int_handler(space, args_w): """ default_int_handler(...) The default handler for SIGINT installed by Python. It raises KeyboardInterrupt. """ + # issue #2780: accept and ignore any non-keyword arguments raise OperationError(space.w_KeyboardInterrupt, space.w_None) diff --git a/pypy/module/signal/test/test_signal.py b/pypy/module/signal/test/test_signal.py --- a/pypy/module/signal/test/test_signal.py +++ b/pypy/module/signal/test/test_signal.py @@ -260,6 +260,17 @@ finally: signal.signal(signum, oldhandler) + def test_default_int_handler(self): + import signal + for args in [(), (1, 2)]: + try: + signal.default_int_handler(*args) + except KeyboardInterrupt: + pass + else: + raise AssertionError("did not raise!") + + class AppTestSignalSocket: spaceconfig = dict(usemodules=['signal', '_socket']) From pypy.commits at gmail.com Thu Apr 5 06:56:03 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 05 Apr 2018 03:56:03 -0700 (PDT) Subject: [pypy-commit] pypy default: progress toward release Message-ID: <5ac600c3.47d81c0a.722f1.6b1f@mx.google.com> Author: Matti Picus Branch: Changeset: r94244:baec80cd31df Date: 2018-04-05 13:41 +0300 http://bitbucket.org/pypy/pypy/changeset/baec80cd31df/ Log: progress toward release diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -6,36 +6,36 @@ Except when otherwise stated (look for LICENSE files in directories or information at the beginning of each file) all software and documentation in the 'rpython', 'pypy', 'ctype_configure', 'dotviewer', 'demo', 'lib_pypy', -'py', and '_pytest' directories is licensed as follows: +'py', and '_pytest' directories is licensed as follows: The MIT License - Permission is hereby granted, free of charge, to any person - obtaining a copy of this software and associated documentation - files (the "Software"), to deal in the Software without - restriction, including without limitation the rights to use, - copy, modify, merge, publish, distribute, sublicense, and/or - sell copies of the Software, and to permit persons to whom the + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation + files (the "Software"), to deal in the Software without + restriction, including without limitation the rights to use, + copy, modify, merge, publish, distribute, sublicense, and/or + sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included + The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. PyPy Copyright holders 2003-2018 ------------------------------------ +-------------------------------- Except when otherwise stated (look for LICENSE files or information at the beginning of each file) the files in the 'pypy' directory are each -copyrighted by one or more of the following people and organizations: +copyrighted by one or more of the following people and organizations: Armin Rigo Maciej Fijalkowski @@ -89,13 +89,13 @@ Niko Matsakis Alexander Hesse Ludovic Aubry + stian Jacob Hallen Jason Creighton Mark Young Alex Martelli Spenser Bauman Michal Bendowski - stian Jan de Mooij Tyler Wade Vincent Legoll @@ -123,10 +123,10 @@ Wenzhu Man Konstantin Lopuhin John Witulski + Jeremy Thurgood Greg Price Ivan Sichmann Freitas Dario Bertini - Jeremy Thurgood Mark Pearse Simon Cross Tobias Pape @@ -145,18 +145,19 @@ Adrian Kuhn tav Georg Brandl + Joannah Nanjekye Bert Freudenberg Stian Andreassen Wanja Saatkamp Mike Blume - Joannah Nanjekye Gerald Klix Oscar Nierstrasz Rami Chowdhury Stefan H. Muller + Dodan Mihai Tim Felgentreff Eugene Oden - Dodan Mihai + Colin Valliant Jeff Terrace Henry Mason Vasily Kuznetsov @@ -225,12 +226,14 @@ Vaibhav Sood Reuben Cummings Attila Gobi + Floris Bruynooghe Christopher Pope Tristan Arthur Christian Tismer Dan Stromberg Carl Meyer Florin Papa + Arianna Avanzini Jens-Uwe Mager Valentina Mukhamedzhanova Stefano Parmesan @@ -250,9 +253,11 @@ Alejandro J. Cura Vladimir Kryachko Gabriel + Thomas Hisch Mark Williams Kunal Grover Nathan Taylor + Barry Hart Travis Francis Athougies Yasir Suhail Sergey Kishchenko @@ -260,6 +265,7 @@ Lutz Paelike Ian Foote Philipp Rustemeuer + Logan Chien Catalin Gabriel Manciu Jacob Oscarson Ryan Gonzalez @@ -295,7 +301,6 @@ Akira Li Gustavo Niemeyer Rafał Gałczyński - Logan Chien Lucas Stadler roberto at goyle Matt Bogosian @@ -308,6 +313,7 @@ Anna Katrina Dominguez Kim Jin Su Amber Brown + Miro Hrončok Anthony Sottile Nate Bragg Ben Darnell @@ -315,7 +321,6 @@ Godefroid Chappelle Julian Berman Michael Hudson-Doyle - Floris Bruynooghe Stephan Busemann Dan Colish timo @@ -357,6 +362,7 @@ Michael Chermside Anna Ravencroft remarkablerocket + Pauli Virtanen Petre Vijiac Berker Peksag Christian Muirhead @@ -381,6 +387,7 @@ Graham Markall Dan Loewenherz werat + Andrew Stepanov Niclas Olofsson Chris Pressey Tobias Diaz @@ -395,14 +402,14 @@ m at funkyhat.org Stefan Marr - Heinrich-Heine University, Germany + Heinrich-Heine University, Germany Open End AB (formerly AB Strakt), Sweden - merlinux GmbH, Germany - tismerysoft GmbH, Germany - Logilab Paris, France - DFKI GmbH, Germany + merlinux GmbH, Germany + tismerysoft GmbH, Germany + Logilab Paris, France + DFKI GmbH, Germany Impara, Germany - Change Maker, Sweden + Change Maker, Sweden University of California Berkeley, USA Google Inc. King's College London @@ -410,14 +417,14 @@ The PyPy Logo as used by http://speed.pypy.org and others was created by Samuel Reis and is distributed on terms of Creative Commons Share Alike License. - -License for 'lib-python/2.7' -============================ + +License for 'lib-python/2.7, lib-python/3' +========================================== Except when otherwise stated (look for LICENSE files or copyright/license -information at the beginning of each file) the files in the 'lib-python/2.7' +information at the beginning of each file) the files in the 'lib-python' directory are all copyrighted by the Python Software Foundation and licensed -under the terms that you can find here: https://docs.python.org/2/license.html +under the terms that you can find here: https://docs.python.org/3/license.html License for 'pypy/module/unicodedata/' ====================================== @@ -441,9 +448,9 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -56,13 +56,13 @@ Niko Matsakis Alexander Hesse Ludovic Aubry + stian Jacob Hallen Jason Creighton Mark Young Alex Martelli Spenser Bauman Michal Bendowski - stian Jan de Mooij Tyler Wade Vincent Legoll @@ -90,10 +90,10 @@ Wenzhu Man Konstantin Lopuhin John Witulski + Jeremy Thurgood Greg Price Ivan Sichmann Freitas Dario Bertini - Jeremy Thurgood Mark Pearse Simon Cross Tobias Pape @@ -112,18 +112,19 @@ Adrian Kuhn tav Georg Brandl + Joannah Nanjekye Bert Freudenberg Stian Andreassen Wanja Saatkamp Mike Blume - Joannah Nanjekye Gerald Klix Oscar Nierstrasz Rami Chowdhury Stefan H. Muller + Dodan Mihai Tim Felgentreff Eugene Oden - Dodan Mihai + Colin Valliant Jeff Terrace Henry Mason Vasily Kuznetsov @@ -192,12 +193,14 @@ Vaibhav Sood Reuben Cummings Attila Gobi + Floris Bruynooghe Christopher Pope Tristan Arthur Christian Tismer Dan Stromberg Carl Meyer Florin Papa + Arianna Avanzini Jens-Uwe Mager Valentina Mukhamedzhanova Stefano Parmesan @@ -221,6 +224,7 @@ Mark Williams Kunal Grover Nathan Taylor + Barry Hart Travis Francis Athougies Yasir Suhail Sergey Kishchenko @@ -228,6 +232,7 @@ Lutz Paelike Ian Foote Philipp Rustemeuer + Logan Chien Catalin Gabriel Manciu Jacob Oscarson Ryan Gonzalez @@ -263,7 +268,6 @@ Akira Li Gustavo Niemeyer Rafał Gałczyński - Logan Chien Lucas Stadler roberto at goyle Matt Bogosian @@ -276,6 +280,7 @@ Anna Katrina Dominguez Kim Jin Su Amber Brown + Miro Hrončok Anthony Sottile Nate Bragg Ben Darnell @@ -283,7 +288,6 @@ Godefroid Chappelle Julian Berman Michael Hudson-Doyle - Floris Bruynooghe Stephan Busemann Dan Colish timo @@ -325,6 +329,7 @@ Michael Chermside Anna Ravencroft remarkablerocket + Pauli Virtanen Petre Vijiac Berker Peksag Christian Muirhead @@ -349,6 +354,7 @@ Graham Markall Dan Loewenherz werat + Andrew Stepanov Niclas Olofsson Chris Pressey Tobias Diaz diff --git a/pypy/doc/release-v6.0.0.rst b/pypy/doc/release-v6.0.0.rst --- a/pypy/doc/release-v6.0.0.rst +++ b/pypy/doc/release-v6.0.0.rst @@ -27,20 +27,20 @@ make it into the release. We also began working on a Python3.6 implementation, help is welcome. -We updated the cffi module included in PyPy to version 1.11.4 +We updated the cffi module included in PyPy to version 1.11.5 You can download the v6.0 releases here: http://pypy.org/download.html We would like to thank our donors for the continued support of the PyPy -project. +project. If PyPy is not quite good enough for your needs, we are available for +direct consulting work. -We would also like to thank our contributors and -encourage new people to join the project. PyPy has many -layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation -improvements, tweaking popular `modules`_ to run on pypy, or general `help`_ -with making RPython's JIT even better. +We would also like to thank our contributors and encourage new people to join +the project. PyPy has many layers and we need help with all of them: `PyPy`_ +and `RPython`_ documentation improvements, tweaking popular `modules`_ to run +on pypy, or general `help`_ with making RPython's JIT even better. .. _`PyPy`: index.html .. _`RPython`: https://rpython.readthedocs.org @@ -74,6 +74,9 @@ Changelog ========= +* Speed up C-API method calls, and make most Py*_Check calls C macros +* Speed up C-API slot method calls +* Enable TkAgg backend support for matplotlib * support ``hastzinfo`` and ``tzinfo`` in the C-API ``PyDateTime*`` structures * datetime.h is now more similar to CPython * We now support ``PyUnicode_AsUTF{16,32}String``, ``_PyLong_AsByteArray``, @@ -87,3 +90,13 @@ * Improve handling of ``bigint`` s, including fixing ``int_divmod`` * Improve reporting of GC statistics * Accept unicode filenames in ``dbm.open()`` +* Improve RPython support for half-floats +* Added missing attributes to C-API ``instancemethod`` on pypy3 +* Store error state in thread-local storage for C-API. +* Fix JIT bugs exposed in the sre module +* Improve speed of Python parser, improve ParseError messages slightly +* Handle JIT hooks more efficiently + +We also refactored many parts of the JIT bridge optimizations, as well as cpyext +internals, and together with new contributors fixed issues, added new +documentation, and cleaned up the codebase. diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh --- a/pypy/tool/release/repackage.sh +++ b/pypy/tool/release/repackage.sh @@ -1,8 +1,8 @@ # Edit these appropriately before running this script pmaj=2 # python main version pmin=7 # python minor version -maj=5 -min=8 +maj=6 +min=0 rev=0 branchname=release-pypy$pmaj.$pmin-$maj.x # ==OR== release-$maj.x # ==OR== release-$maj.$min.x tagname=release-pypy$pmaj.$pmin-v$maj.$min.$rev # ==OR== release-$maj.$min From pypy.commits at gmail.com Thu Apr 5 06:56:05 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 05 Apr 2018 03:56:05 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: merge default into branch Message-ID: <5ac600c5.929cdf0a.5c3fb.8a87@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r94245:bcbefe2f5e14 Date: 2018-04-05 13:51 +0300 http://bitbucket.org/pypy/pypy/changeset/bcbefe2f5e14/ Log: merge default into branch diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -6,36 +6,36 @@ Except when otherwise stated (look for LICENSE files in directories or information at the beginning of each file) all software and documentation in the 'rpython', 'pypy', 'ctype_configure', 'dotviewer', 'demo', 'lib_pypy', -'py', and '_pytest' directories is licensed as follows: +'py', and '_pytest' directories is licensed as follows: The MIT License - Permission is hereby granted, free of charge, to any person - obtaining a copy of this software and associated documentation - files (the "Software"), to deal in the Software without - restriction, including without limitation the rights to use, - copy, modify, merge, publish, distribute, sublicense, and/or - sell copies of the Software, and to permit persons to whom the + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation + files (the "Software"), to deal in the Software without + restriction, including without limitation the rights to use, + copy, modify, merge, publish, distribute, sublicense, and/or + sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included + The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. PyPy Copyright holders 2003-2018 ------------------------------------ +-------------------------------- Except when otherwise stated (look for LICENSE files or information at the beginning of each file) the files in the 'pypy' directory are each -copyrighted by one or more of the following people and organizations: +copyrighted by one or more of the following people and organizations: Armin Rigo Maciej Fijalkowski @@ -89,13 +89,13 @@ Niko Matsakis Alexander Hesse Ludovic Aubry + stian Jacob Hallen Jason Creighton Mark Young Alex Martelli Spenser Bauman Michal Bendowski - stian Jan de Mooij Tyler Wade Vincent Legoll @@ -123,10 +123,10 @@ Wenzhu Man Konstantin Lopuhin John Witulski + Jeremy Thurgood Greg Price Ivan Sichmann Freitas Dario Bertini - Jeremy Thurgood Mark Pearse Simon Cross Tobias Pape @@ -145,18 +145,19 @@ Adrian Kuhn tav Georg Brandl + Joannah Nanjekye Bert Freudenberg Stian Andreassen Wanja Saatkamp Mike Blume - Joannah Nanjekye Gerald Klix Oscar Nierstrasz Rami Chowdhury Stefan H. Muller + Dodan Mihai Tim Felgentreff Eugene Oden - Dodan Mihai + Colin Valliant Jeff Terrace Henry Mason Vasily Kuznetsov @@ -225,12 +226,14 @@ Vaibhav Sood Reuben Cummings Attila Gobi + Floris Bruynooghe Christopher Pope Tristan Arthur Christian Tismer Dan Stromberg Carl Meyer Florin Papa + Arianna Avanzini Jens-Uwe Mager Valentina Mukhamedzhanova Stefano Parmesan @@ -250,9 +253,11 @@ Alejandro J. Cura Vladimir Kryachko Gabriel + Thomas Hisch Mark Williams Kunal Grover Nathan Taylor + Barry Hart Travis Francis Athougies Yasir Suhail Sergey Kishchenko @@ -260,6 +265,7 @@ Lutz Paelike Ian Foote Philipp Rustemeuer + Logan Chien Catalin Gabriel Manciu Jacob Oscarson Ryan Gonzalez @@ -295,7 +301,6 @@ Akira Li Gustavo Niemeyer Rafał Gałczyński - Logan Chien Lucas Stadler roberto at goyle Matt Bogosian @@ -308,6 +313,7 @@ Anna Katrina Dominguez Kim Jin Su Amber Brown + Miro Hrončok Anthony Sottile Nate Bragg Ben Darnell @@ -315,7 +321,6 @@ Godefroid Chappelle Julian Berman Michael Hudson-Doyle - Floris Bruynooghe Stephan Busemann Dan Colish timo @@ -357,6 +362,7 @@ Michael Chermside Anna Ravencroft remarkablerocket + Pauli Virtanen Petre Vijiac Berker Peksag Christian Muirhead @@ -381,6 +387,7 @@ Graham Markall Dan Loewenherz werat + Andrew Stepanov Niclas Olofsson Chris Pressey Tobias Diaz @@ -395,14 +402,14 @@ m at funkyhat.org Stefan Marr - Heinrich-Heine University, Germany + Heinrich-Heine University, Germany Open End AB (formerly AB Strakt), Sweden - merlinux GmbH, Germany - tismerysoft GmbH, Germany - Logilab Paris, France - DFKI GmbH, Germany + merlinux GmbH, Germany + tismerysoft GmbH, Germany + Logilab Paris, France + DFKI GmbH, Germany Impara, Germany - Change Maker, Sweden + Change Maker, Sweden University of California Berkeley, USA Google Inc. King's College London @@ -410,14 +417,14 @@ The PyPy Logo as used by http://speed.pypy.org and others was created by Samuel Reis and is distributed on terms of Creative Commons Share Alike License. - -License for 'lib-python/2.7' -============================ + +License for 'lib-python/2.7, lib-python/3' +========================================== Except when otherwise stated (look for LICENSE files or copyright/license -information at the beginning of each file) the files in the 'lib-python/2.7' +information at the beginning of each file) the files in the 'lib-python' directory are all copyrighted by the Python Software Foundation and licensed -under the terms that you can find here: https://docs.python.org/2/license.html +under the terms that you can find here: https://docs.python.org/3/license.html License for 'pypy/module/unicodedata/' ====================================== @@ -441,9 +448,9 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -56,13 +56,13 @@ Niko Matsakis Alexander Hesse Ludovic Aubry + stian Jacob Hallen Jason Creighton Mark Young Alex Martelli Spenser Bauman Michal Bendowski - stian Jan de Mooij Tyler Wade Vincent Legoll @@ -90,10 +90,10 @@ Wenzhu Man Konstantin Lopuhin John Witulski + Jeremy Thurgood Greg Price Ivan Sichmann Freitas Dario Bertini - Jeremy Thurgood Mark Pearse Simon Cross Tobias Pape @@ -112,18 +112,19 @@ Adrian Kuhn tav Georg Brandl + Joannah Nanjekye Bert Freudenberg Stian Andreassen Wanja Saatkamp Mike Blume - Joannah Nanjekye Gerald Klix Oscar Nierstrasz Rami Chowdhury Stefan H. Muller + Dodan Mihai Tim Felgentreff Eugene Oden - Dodan Mihai + Colin Valliant Jeff Terrace Henry Mason Vasily Kuznetsov @@ -192,12 +193,14 @@ Vaibhav Sood Reuben Cummings Attila Gobi + Floris Bruynooghe Christopher Pope Tristan Arthur Christian Tismer Dan Stromberg Carl Meyer Florin Papa + Arianna Avanzini Jens-Uwe Mager Valentina Mukhamedzhanova Stefano Parmesan @@ -221,6 +224,7 @@ Mark Williams Kunal Grover Nathan Taylor + Barry Hart Travis Francis Athougies Yasir Suhail Sergey Kishchenko @@ -228,6 +232,7 @@ Lutz Paelike Ian Foote Philipp Rustemeuer + Logan Chien Catalin Gabriel Manciu Jacob Oscarson Ryan Gonzalez @@ -263,7 +268,6 @@ Akira Li Gustavo Niemeyer Rafał Gałczyński - Logan Chien Lucas Stadler roberto at goyle Matt Bogosian @@ -276,6 +280,7 @@ Anna Katrina Dominguez Kim Jin Su Amber Brown + Miro Hrončok Anthony Sottile Nate Bragg Ben Darnell @@ -283,7 +288,6 @@ Godefroid Chappelle Julian Berman Michael Hudson-Doyle - Floris Bruynooghe Stephan Busemann Dan Colish timo @@ -325,6 +329,7 @@ Michael Chermside Anna Ravencroft remarkablerocket + Pauli Virtanen Petre Vijiac Berker Peksag Christian Muirhead @@ -349,6 +354,7 @@ Graham Markall Dan Loewenherz werat + Andrew Stepanov Niclas Olofsson Chris Pressey Tobias Diaz diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-v6.0.0.rst release-v5.10.1.rst release-v5.10.0.rst release-v5.9.0.rst diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -7,6 +7,8 @@ .. toctree:: whatsnew-head.rst + whatsnew-pypy2-6.0.0.rst + whatsnew-pypy2-5.10.0.rst whatsnew-pypy2-5.10.0.rst whatsnew-pypy2-5.9.0.rst whatsnew-pypy2-5.8.0.rst diff --git a/pypy/doc/release-v6.0.0.rst b/pypy/doc/release-v6.0.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-v6.0.0.rst @@ -0,0 +1,102 @@ +====================================== +PyPy2.7 and PyPy3.5 v6.0 dual release +====================================== + +The PyPy team is proud to release both PyPy2.7 v6.0 (an interpreter supporting +Python 2.7 syntax), and a PyPy3.5 v6.0 (an interpreter supporting Python +3.5 syntax). The two releases are both based on much the same codebase, thus +the dual release. + +This release is a feature release following our previous 5.10 incremental +release in late December 2017, with many improvements in the C-API +compatability layer and other improvements in speed and CPython compatibility. +Since the changes affect the included python development header files, all +c-extension modules must be recompiled for this version. + +The Windows PyPy3.5 release is still considered beta-quality. There are issues +with unicode handling especially around system calls and c-extensions. + +The Matplotlib TkAgg backend now works with PyPy. PyGame and pygtk also now can +work with PyPy. + +As always, this release is 100% compatible with the previous one and fixed +several issues and bugs raised by the growing community of PyPy users. +We strongly recommend updating. + +The utf8 branch that changes internal representation of unicode to utf8 did not +make it into the release. We also began working on a Python3.6 implementation, +help is welcome. + +We updated the cffi module included in PyPy to version 1.11.5 + +You can download the v6.0 releases here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project. If PyPy is not quite good enough for your needs, we are available for +direct consulting work. + +We would also like to thank our contributors and encourage new people to join +the project. PyPy has many layers and we need help with all of them: `PyPy`_ +and `RPython`_ documentation improvements, tweaking popular `modules`_ to run +on pypy, or general `help`_ with making RPython's JIT even better. + +.. _`PyPy`: index.html +.. _`RPython`: https://rpython.readthedocs.org +.. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly +.. _`help`: project-ideas.html + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7 and CPython 3.5. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +The PyPy release supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html + +Changelog +========= + +* Speed up C-API method calls, and make most Py*_Check calls C macros +* Speed up C-API slot method calls +* Enable TkAgg backend support for matplotlib +* support ``hastzinfo`` and ``tzinfo`` in the C-API ``PyDateTime*`` structures +* datetime.h is now more similar to CPython +* We now support ``PyUnicode_AsUTF{16,32}String``, ``_PyLong_AsByteArray``, + ``_PyLong_AsByteArrayO``, +* PyPy3.5 on Windows is compiled with the Microsoft Visual Compiler v14, like + CPython +* Fix performance of attribute lookup when more than 80 attributes are used +* Improve performance on passing built-in types to C-API C code +* Improve the performance of datetime and timedelta by skipping the consistency + checks of the datetime values (they are correct by construction) +* Improve handling of ``bigint`` s, including fixing ``int_divmod`` +* Improve reporting of GC statistics +* Accept unicode filenames in ``dbm.open()`` +* Improve RPython support for half-floats +* Added missing attributes to C-API ``instancemethod`` on pypy3 +* Store error state in thread-local storage for C-API. +* Fix JIT bugs exposed in the sre module +* Improve speed of Python parser, improve ParseError messages slightly +* Handle JIT hooks more efficiently + +We also refactored many parts of the JIT bridge optimizations, as well as cpyext +internals, and together with new contributors fixed issues, added new +documentation, and cleaned up the codebase. diff --git a/pypy/doc/whatsnew-6.0.0.rst b/pypy/doc/whatsnew-6.0.0.rst deleted file mode 100644 --- a/pypy/doc/whatsnew-6.0.0.rst +++ /dev/null @@ -1,103 +0,0 @@ -=========================== -What's new in PyPy2.7 5.10+ -=========================== - -.. this is a revision shortly after release-pypy2.7-v5.10.0 -.. startrev: 6b024edd9d12 - -.. branch: cpyext-avoid-roundtrip - -Big refactoring of some cpyext code, which avoids a lot of nonsense when -calling C from Python and vice-versa: the result is a big speedup in -function/method calls, up to 6 times faster. - -.. branch: cpyext-datetime2 - -Support ``tzinfo`` field on C-API datetime objects, fixes latest pandas HEAD - - -.. branch: mapdict-size-limit - -Fix a corner case of mapdict: When an instance is used like a dict (using -``setattr`` and ``getattr``, or ``.__dict__``) and a lot of attributes are -added, then the performance using mapdict is linear in the number of -attributes. This is now fixed (by switching to a regular dict after 80 -attributes). - - -.. branch: cpyext-faster-arg-passing - -When using cpyext, improve the speed of passing certain objects from PyPy to C -code, most notably None, True, False, types, all instances of C-defined types. -Before, a dict lookup was needed every time such an object crossed over, now it -is just a field read. - - -.. branch: 2634_datetime_timedelta_performance - -Improve datetime + timedelta performance. - -.. branch: memory-accounting - -Improve way to describe memory - -.. branch: msvc14 - -Allow compilaiton with Visual Studio 2017 compiler suite on windows - -.. branch: winapi - -Update _winapi and internal _winbase_cffi (via _winbase_build) for python 3 - -.. branch: refactor-slots - -Refactor cpyext slots. - - -.. branch: call-loopinvariant-into-bridges - -Speed up branchy code that does a lot of function inlining by saving one call -to read the TLS in most bridges. - -.. branch: rpython-sprint - -Refactor in rpython signatures - -.. branch: cpyext-tls-operror2 - -Store error state thread-locally in executioncontext, fixes issue #2764 - -.. branch: cpyext-fast-typecheck - -Optimize `Py*_Check` for `Bool`, `Float`, `Set`. Also refactor and simplify -`W_PyCWrapperObject` which is used to call slots from the C-API, greatly -improving microbenchmarks in https://github.com/antocuni/cpyext-benchmarks - - -.. branch: fix-sre-problems - -Fix two (unrelated) JIT bugs manifesting in the re module: - -- green fields are broken and were thus disabled, plus their usage removed from - the _sre implementation - -- in rare "trace is too long" situations, the JIT could break behaviour - arbitrarily. - -.. branch: jit-hooks-can-be-disabled - -Be more efficient about JIT hooks. Make it possible for the frontend to declare -that jit hooks are currently not enabled at all. in that case, the list of ops -does not have to be created in the case of the on_abort hook (which is -expensive). - - -.. branch: pyparser-improvements - -Improve speed of Python parser, improve ParseError messages slightly. - -.. branch: ioctl-arg-size - -Work around possible bugs in upstream ioctl users, like CPython allocate at -least 1024 bytes for the arg in calls to ``ioctl(fd, request, arg)``. Fixes -issue #2776 diff --git a/pypy/doc/whatsnew-pypy2-6.0.0.rst b/pypy/doc/whatsnew-pypy2-6.0.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-pypy2-6.0.0.rst @@ -0,0 +1,103 @@ +=========================== +What's new in PyPy2.7 5.10+ +=========================== + +.. this is a revision shortly after release-pypy2.7-v5.10.0 +.. startrev: 6b024edd9d12 + +.. branch: cpyext-avoid-roundtrip + +Big refactoring of some cpyext code, which avoids a lot of nonsense when +calling C from Python and vice-versa: the result is a big speedup in +function/method calls, up to 6 times faster. + +.. branch: cpyext-datetime2 + +Support ``tzinfo`` field on C-API datetime objects, fixes latest pandas HEAD + + +.. branch: mapdict-size-limit + +Fix a corner case of mapdict: When an instance is used like a dict (using +``setattr`` and ``getattr``, or ``.__dict__``) and a lot of attributes are +added, then the performance using mapdict is linear in the number of +attributes. This is now fixed (by switching to a regular dict after 80 +attributes). + + +.. branch: cpyext-faster-arg-passing + +When using cpyext, improve the speed of passing certain objects from PyPy to C +code, most notably None, True, False, types, all instances of C-defined types. +Before, a dict lookup was needed every time such an object crossed over, now it +is just a field read. + + +.. branch: 2634_datetime_timedelta_performance + +Improve datetime + timedelta performance. + +.. branch: memory-accounting + +Improve way to describe memory + +.. branch: msvc14 + +Allow compilaiton with Visual Studio 2017 compiler suite on windows + +.. branch: winapi + +Update _winapi and internal _winbase_cffi (via _winbase_build) for python 3 + +.. branch: refactor-slots + +Refactor cpyext slots. + + +.. branch: call-loopinvariant-into-bridges + +Speed up branchy code that does a lot of function inlining by saving one call +to read the TLS in most bridges. + +.. branch: rpython-sprint + +Refactor in rpython signatures + +.. branch: cpyext-tls-operror2 + +Store error state thread-locally in executioncontext, fixes issue #2764 + +.. branch: cpyext-fast-typecheck + +Optimize `Py*_Check` for `Bool`, `Float`, `Set`. Also refactor and simplify +`W_PyCWrapperObject` which is used to call slots from the C-API, greatly +improving microbenchmarks in https://github.com/antocuni/cpyext-benchmarks + + +.. branch: fix-sre-problems + +Fix two (unrelated) JIT bugs manifesting in the re module: + +- green fields are broken and were thus disabled, plus their usage removed from + the _sre implementation + +- in rare "trace is too long" situations, the JIT could break behaviour + arbitrarily. + +.. branch: jit-hooks-can-be-disabled + +Be more efficient about JIT hooks. Make it possible for the frontend to declare +that jit hooks are currently not enabled at all. in that case, the list of ops +does not have to be created in the case of the on_abort hook (which is +expensive). + + +.. branch: pyparser-improvements + +Improve speed of Python parser, improve ParseError messages slightly. + +.. branch: ioctl-arg-size + +Work around possible bugs in upstream ioctl users, like CPython allocate at +least 1024 bytes for the arg in calls to ``ioctl(fd, request, arg)``. Fixes +issue #2776 diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh --- a/pypy/tool/release/repackage.sh +++ b/pypy/tool/release/repackage.sh @@ -1,8 +1,8 @@ # Edit these appropriately before running this script pmaj=2 # python main version pmin=7 # python minor version -maj=5 -min=8 +maj=6 +min=0 rev=0 branchname=release-pypy$pmaj.$pmin-$maj.x # ==OR== release-$maj.x # ==OR== release-$maj.$min.x tagname=release-pypy$pmaj.$pmin-v$maj.$min.$rev # ==OR== release-$maj.$min From pypy.commits at gmail.com Thu Apr 5 06:56:09 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 05 Apr 2018 03:56:09 -0700 (PDT) Subject: [pypy-commit] pypy release-pypy3.5-6.x: merge py3.5 into release Message-ID: <5ac600c9.0b371c0a.68516.35e2@mx.google.com> Author: Matti Picus Branch: release-pypy3.5-6.x Changeset: r94247:4b3a0384cf33 Date: 2018-04-05 13:55 +0300 http://bitbucket.org/pypy/pypy/changeset/4b3a0384cf33/ Log: merge py3.5 into release diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -6,36 +6,36 @@ Except when otherwise stated (look for LICENSE files in directories or information at the beginning of each file) all software and documentation in the 'rpython', 'pypy', 'ctype_configure', 'dotviewer', 'demo', 'lib_pypy', -'py', and '_pytest' directories is licensed as follows: +'py', and '_pytest' directories is licensed as follows: The MIT License - Permission is hereby granted, free of charge, to any person - obtaining a copy of this software and associated documentation - files (the "Software"), to deal in the Software without - restriction, including without limitation the rights to use, - copy, modify, merge, publish, distribute, sublicense, and/or - sell copies of the Software, and to permit persons to whom the + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation + files (the "Software"), to deal in the Software without + restriction, including without limitation the rights to use, + copy, modify, merge, publish, distribute, sublicense, and/or + sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included + The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. PyPy Copyright holders 2003-2018 ------------------------------------ +-------------------------------- Except when otherwise stated (look for LICENSE files or information at the beginning of each file) the files in the 'pypy' directory are each -copyrighted by one or more of the following people and organizations: +copyrighted by one or more of the following people and organizations: Armin Rigo Maciej Fijalkowski @@ -89,13 +89,13 @@ Niko Matsakis Alexander Hesse Ludovic Aubry + stian Jacob Hallen Jason Creighton Mark Young Alex Martelli Spenser Bauman Michal Bendowski - stian Jan de Mooij Tyler Wade Vincent Legoll @@ -123,10 +123,10 @@ Wenzhu Man Konstantin Lopuhin John Witulski + Jeremy Thurgood Greg Price Ivan Sichmann Freitas Dario Bertini - Jeremy Thurgood Mark Pearse Simon Cross Tobias Pape @@ -145,18 +145,19 @@ Adrian Kuhn tav Georg Brandl + Joannah Nanjekye Bert Freudenberg Stian Andreassen Wanja Saatkamp Mike Blume - Joannah Nanjekye Gerald Klix Oscar Nierstrasz Rami Chowdhury Stefan H. Muller + Dodan Mihai Tim Felgentreff Eugene Oden - Dodan Mihai + Colin Valliant Jeff Terrace Henry Mason Vasily Kuznetsov @@ -225,12 +226,14 @@ Vaibhav Sood Reuben Cummings Attila Gobi + Floris Bruynooghe Christopher Pope Tristan Arthur Christian Tismer Dan Stromberg Carl Meyer Florin Papa + Arianna Avanzini Jens-Uwe Mager Valentina Mukhamedzhanova Stefano Parmesan @@ -250,9 +253,11 @@ Alejandro J. Cura Vladimir Kryachko Gabriel + Thomas Hisch Mark Williams Kunal Grover Nathan Taylor + Barry Hart Travis Francis Athougies Yasir Suhail Sergey Kishchenko @@ -260,6 +265,7 @@ Lutz Paelike Ian Foote Philipp Rustemeuer + Logan Chien Catalin Gabriel Manciu Jacob Oscarson Ryan Gonzalez @@ -295,7 +301,6 @@ Akira Li Gustavo Niemeyer Rafał Gałczyński - Logan Chien Lucas Stadler roberto at goyle Matt Bogosian @@ -308,6 +313,7 @@ Anna Katrina Dominguez Kim Jin Su Amber Brown + Miro Hrončok Anthony Sottile Nate Bragg Ben Darnell @@ -315,7 +321,6 @@ Godefroid Chappelle Julian Berman Michael Hudson-Doyle - Floris Bruynooghe Stephan Busemann Dan Colish timo @@ -357,6 +362,7 @@ Michael Chermside Anna Ravencroft remarkablerocket + Pauli Virtanen Petre Vijiac Berker Peksag Christian Muirhead @@ -381,6 +387,7 @@ Graham Markall Dan Loewenherz werat + Andrew Stepanov Niclas Olofsson Chris Pressey Tobias Diaz @@ -395,14 +402,14 @@ m at funkyhat.org Stefan Marr - Heinrich-Heine University, Germany + Heinrich-Heine University, Germany Open End AB (formerly AB Strakt), Sweden - merlinux GmbH, Germany - tismerysoft GmbH, Germany - Logilab Paris, France - DFKI GmbH, Germany + merlinux GmbH, Germany + tismerysoft GmbH, Germany + Logilab Paris, France + DFKI GmbH, Germany Impara, Germany - Change Maker, Sweden + Change Maker, Sweden University of California Berkeley, USA Google Inc. King's College London @@ -410,14 +417,14 @@ The PyPy Logo as used by http://speed.pypy.org and others was created by Samuel Reis and is distributed on terms of Creative Commons Share Alike License. - -License for 'lib-python/2.7' -============================ + +License for 'lib-python/2.7, lib-python/3' +========================================== Except when otherwise stated (look for LICENSE files or copyright/license -information at the beginning of each file) the files in the 'lib-python/2.7' +information at the beginning of each file) the files in the 'lib-python' directory are all copyrighted by the Python Software Foundation and licensed -under the terms that you can find here: https://docs.python.org/2/license.html +under the terms that you can find here: https://docs.python.org/3/license.html License for 'pypy/module/unicodedata/' ====================================== @@ -441,9 +448,9 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -56,13 +56,13 @@ Niko Matsakis Alexander Hesse Ludovic Aubry + stian Jacob Hallen Jason Creighton Mark Young Alex Martelli Spenser Bauman Michal Bendowski - stian Jan de Mooij Tyler Wade Vincent Legoll @@ -90,10 +90,10 @@ Wenzhu Man Konstantin Lopuhin John Witulski + Jeremy Thurgood Greg Price Ivan Sichmann Freitas Dario Bertini - Jeremy Thurgood Mark Pearse Simon Cross Tobias Pape @@ -112,18 +112,19 @@ Adrian Kuhn tav Georg Brandl + Joannah Nanjekye Bert Freudenberg Stian Andreassen Wanja Saatkamp Mike Blume - Joannah Nanjekye Gerald Klix Oscar Nierstrasz Rami Chowdhury Stefan H. Muller + Dodan Mihai Tim Felgentreff Eugene Oden - Dodan Mihai + Colin Valliant Jeff Terrace Henry Mason Vasily Kuznetsov @@ -192,12 +193,14 @@ Vaibhav Sood Reuben Cummings Attila Gobi + Floris Bruynooghe Christopher Pope Tristan Arthur Christian Tismer Dan Stromberg Carl Meyer Florin Papa + Arianna Avanzini Jens-Uwe Mager Valentina Mukhamedzhanova Stefano Parmesan @@ -221,6 +224,7 @@ Mark Williams Kunal Grover Nathan Taylor + Barry Hart Travis Francis Athougies Yasir Suhail Sergey Kishchenko @@ -228,6 +232,7 @@ Lutz Paelike Ian Foote Philipp Rustemeuer + Logan Chien Catalin Gabriel Manciu Jacob Oscarson Ryan Gonzalez @@ -263,7 +268,6 @@ Akira Li Gustavo Niemeyer Rafał Gałczyński - Logan Chien Lucas Stadler roberto at goyle Matt Bogosian @@ -276,6 +280,7 @@ Anna Katrina Dominguez Kim Jin Su Amber Brown + Miro Hrončok Anthony Sottile Nate Bragg Ben Darnell @@ -283,7 +288,6 @@ Godefroid Chappelle Julian Berman Michael Hudson-Doyle - Floris Bruynooghe Stephan Busemann Dan Colish timo @@ -325,6 +329,7 @@ Michael Chermside Anna Ravencroft remarkablerocket + Pauli Virtanen Petre Vijiac Berker Peksag Christian Muirhead @@ -349,6 +354,7 @@ Graham Markall Dan Loewenherz werat + Andrew Stepanov Niclas Olofsson Chris Pressey Tobias Diaz diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-v6.0.0.rst release-v5.10.1.rst release-v5.10.0.rst release-v5.9.0.rst diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -7,6 +7,8 @@ .. toctree:: whatsnew-head.rst + whatsnew-pypy2-6.0.0.rst + whatsnew-pypy2-5.10.0.rst whatsnew-pypy2-5.10.0.rst whatsnew-pypy2-5.9.0.rst whatsnew-pypy2-5.8.0.rst diff --git a/pypy/doc/release-v6.0.0.rst b/pypy/doc/release-v6.0.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-v6.0.0.rst @@ -0,0 +1,102 @@ +====================================== +PyPy2.7 and PyPy3.5 v6.0 dual release +====================================== + +The PyPy team is proud to release both PyPy2.7 v6.0 (an interpreter supporting +Python 2.7 syntax), and a PyPy3.5 v6.0 (an interpreter supporting Python +3.5 syntax). The two releases are both based on much the same codebase, thus +the dual release. + +This release is a feature release following our previous 5.10 incremental +release in late December 2017, with many improvements in the C-API +compatability layer and other improvements in speed and CPython compatibility. +Since the changes affect the included python development header files, all +c-extension modules must be recompiled for this version. + +The Windows PyPy3.5 release is still considered beta-quality. There are issues +with unicode handling especially around system calls and c-extensions. + +The Matplotlib TkAgg backend now works with PyPy. PyGame and pygtk also now can +work with PyPy. + +As always, this release is 100% compatible with the previous one and fixed +several issues and bugs raised by the growing community of PyPy users. +We strongly recommend updating. + +The utf8 branch that changes internal representation of unicode to utf8 did not +make it into the release. We also began working on a Python3.6 implementation, +help is welcome. + +We updated the cffi module included in PyPy to version 1.11.5 + +You can download the v6.0 releases here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project. If PyPy is not quite good enough for your needs, we are available for +direct consulting work. + +We would also like to thank our contributors and encourage new people to join +the project. PyPy has many layers and we need help with all of them: `PyPy`_ +and `RPython`_ documentation improvements, tweaking popular `modules`_ to run +on pypy, or general `help`_ with making RPython's JIT even better. + +.. _`PyPy`: index.html +.. _`RPython`: https://rpython.readthedocs.org +.. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly +.. _`help`: project-ideas.html + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7 and CPython 3.5. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +The PyPy release supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html + +Changelog +========= + +* Speed up C-API method calls, and make most Py*_Check calls C macros +* Speed up C-API slot method calls +* Enable TkAgg backend support for matplotlib +* support ``hastzinfo`` and ``tzinfo`` in the C-API ``PyDateTime*`` structures +* datetime.h is now more similar to CPython +* We now support ``PyUnicode_AsUTF{16,32}String``, ``_PyLong_AsByteArray``, + ``_PyLong_AsByteArrayO``, +* PyPy3.5 on Windows is compiled with the Microsoft Visual Compiler v14, like + CPython +* Fix performance of attribute lookup when more than 80 attributes are used +* Improve performance on passing built-in types to C-API C code +* Improve the performance of datetime and timedelta by skipping the consistency + checks of the datetime values (they are correct by construction) +* Improve handling of ``bigint`` s, including fixing ``int_divmod`` +* Improve reporting of GC statistics +* Accept unicode filenames in ``dbm.open()`` +* Improve RPython support for half-floats +* Added missing attributes to C-API ``instancemethod`` on pypy3 +* Store error state in thread-local storage for C-API. +* Fix JIT bugs exposed in the sre module +* Improve speed of Python parser, improve ParseError messages slightly +* Handle JIT hooks more efficiently + +We also refactored many parts of the JIT bridge optimizations, as well as cpyext +internals, and together with new contributors fixed issues, added new +documentation, and cleaned up the codebase. diff --git a/pypy/doc/whatsnew-6.0.0.rst b/pypy/doc/whatsnew-pypy2-6.0.0.rst rename from pypy/doc/whatsnew-6.0.0.rst rename to pypy/doc/whatsnew-pypy2-6.0.0.rst diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py --- a/pypy/interpreter/pyparser/pyparse.py +++ b/pypy/interpreter/pyparser/pyparse.py @@ -148,12 +148,12 @@ raise if enc is not None: compile_info.encoding = enc + if explicit_encoding: + compile_info.flags |= consts.PyCF_FOUND_ENCODING return self._parse(textsrc, compile_info) def _parse(self, textsrc, compile_info): flags = compile_info.flags - if explicit_encoding: - flags |= consts.PyCF_FOUND_ENCODING # The tokenizer is very picky about how it wants its input. source_lines = textsrc.splitlines(True) diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh --- a/pypy/tool/release/repackage.sh +++ b/pypy/tool/release/repackage.sh @@ -1,8 +1,8 @@ # Edit these appropriately before running this script pmaj=2 # python main version pmin=7 # python minor version -maj=5 -min=8 +maj=6 +min=0 rev=0 branchname=release-pypy$pmaj.$pmin-$maj.x # ==OR== release-$maj.x # ==OR== release-$maj.$min.x tagname=release-pypy$pmaj.$pmin-v$maj.$min.$rev # ==OR== release-$maj.$min From pypy.commits at gmail.com Thu Apr 5 06:56:07 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 05 Apr 2018 03:56:07 -0700 (PDT) Subject: [pypy-commit] pypy release-pypy2.7-6.x: merge default into release Message-ID: <5ac600c7.8f8d1c0a.ebda.311b@mx.google.com> Author: Matti Picus Branch: release-pypy2.7-6.x Changeset: r94246:cda3e08c8c81 Date: 2018-04-05 13:54 +0300 http://bitbucket.org/pypy/pypy/changeset/cda3e08c8c81/ Log: merge default into release diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -6,36 +6,36 @@ Except when otherwise stated (look for LICENSE files in directories or information at the beginning of each file) all software and documentation in the 'rpython', 'pypy', 'ctype_configure', 'dotviewer', 'demo', 'lib_pypy', -'py', and '_pytest' directories is licensed as follows: +'py', and '_pytest' directories is licensed as follows: The MIT License - Permission is hereby granted, free of charge, to any person - obtaining a copy of this software and associated documentation - files (the "Software"), to deal in the Software without - restriction, including without limitation the rights to use, - copy, modify, merge, publish, distribute, sublicense, and/or - sell copies of the Software, and to permit persons to whom the + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation + files (the "Software"), to deal in the Software without + restriction, including without limitation the rights to use, + copy, modify, merge, publish, distribute, sublicense, and/or + sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included + The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. PyPy Copyright holders 2003-2018 ------------------------------------ +-------------------------------- Except when otherwise stated (look for LICENSE files or information at the beginning of each file) the files in the 'pypy' directory are each -copyrighted by one or more of the following people and organizations: +copyrighted by one or more of the following people and organizations: Armin Rigo Maciej Fijalkowski @@ -89,13 +89,13 @@ Niko Matsakis Alexander Hesse Ludovic Aubry + stian Jacob Hallen Jason Creighton Mark Young Alex Martelli Spenser Bauman Michal Bendowski - stian Jan de Mooij Tyler Wade Vincent Legoll @@ -123,10 +123,10 @@ Wenzhu Man Konstantin Lopuhin John Witulski + Jeremy Thurgood Greg Price Ivan Sichmann Freitas Dario Bertini - Jeremy Thurgood Mark Pearse Simon Cross Tobias Pape @@ -145,18 +145,19 @@ Adrian Kuhn tav Georg Brandl + Joannah Nanjekye Bert Freudenberg Stian Andreassen Wanja Saatkamp Mike Blume - Joannah Nanjekye Gerald Klix Oscar Nierstrasz Rami Chowdhury Stefan H. Muller + Dodan Mihai Tim Felgentreff Eugene Oden - Dodan Mihai + Colin Valliant Jeff Terrace Henry Mason Vasily Kuznetsov @@ -225,12 +226,14 @@ Vaibhav Sood Reuben Cummings Attila Gobi + Floris Bruynooghe Christopher Pope Tristan Arthur Christian Tismer Dan Stromberg Carl Meyer Florin Papa + Arianna Avanzini Jens-Uwe Mager Valentina Mukhamedzhanova Stefano Parmesan @@ -250,9 +253,11 @@ Alejandro J. Cura Vladimir Kryachko Gabriel + Thomas Hisch Mark Williams Kunal Grover Nathan Taylor + Barry Hart Travis Francis Athougies Yasir Suhail Sergey Kishchenko @@ -260,6 +265,7 @@ Lutz Paelike Ian Foote Philipp Rustemeuer + Logan Chien Catalin Gabriel Manciu Jacob Oscarson Ryan Gonzalez @@ -295,7 +301,6 @@ Akira Li Gustavo Niemeyer Rafał Gałczyński - Logan Chien Lucas Stadler roberto at goyle Matt Bogosian @@ -308,6 +313,7 @@ Anna Katrina Dominguez Kim Jin Su Amber Brown + Miro Hrončok Anthony Sottile Nate Bragg Ben Darnell @@ -315,7 +321,6 @@ Godefroid Chappelle Julian Berman Michael Hudson-Doyle - Floris Bruynooghe Stephan Busemann Dan Colish timo @@ -357,6 +362,7 @@ Michael Chermside Anna Ravencroft remarkablerocket + Pauli Virtanen Petre Vijiac Berker Peksag Christian Muirhead @@ -381,6 +387,7 @@ Graham Markall Dan Loewenherz werat + Andrew Stepanov Niclas Olofsson Chris Pressey Tobias Diaz @@ -395,14 +402,14 @@ m at funkyhat.org Stefan Marr - Heinrich-Heine University, Germany + Heinrich-Heine University, Germany Open End AB (formerly AB Strakt), Sweden - merlinux GmbH, Germany - tismerysoft GmbH, Germany - Logilab Paris, France - DFKI GmbH, Germany + merlinux GmbH, Germany + tismerysoft GmbH, Germany + Logilab Paris, France + DFKI GmbH, Germany Impara, Germany - Change Maker, Sweden + Change Maker, Sweden University of California Berkeley, USA Google Inc. King's College London @@ -410,14 +417,14 @@ The PyPy Logo as used by http://speed.pypy.org and others was created by Samuel Reis and is distributed on terms of Creative Commons Share Alike License. - -License for 'lib-python/2.7' -============================ + +License for 'lib-python/2.7, lib-python/3' +========================================== Except when otherwise stated (look for LICENSE files or copyright/license -information at the beginning of each file) the files in the 'lib-python/2.7' +information at the beginning of each file) the files in the 'lib-python' directory are all copyrighted by the Python Software Foundation and licensed -under the terms that you can find here: https://docs.python.org/2/license.html +under the terms that you can find here: https://docs.python.org/3/license.html License for 'pypy/module/unicodedata/' ====================================== @@ -441,9 +448,9 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -56,13 +56,13 @@ Niko Matsakis Alexander Hesse Ludovic Aubry + stian Jacob Hallen Jason Creighton Mark Young Alex Martelli Spenser Bauman Michal Bendowski - stian Jan de Mooij Tyler Wade Vincent Legoll @@ -90,10 +90,10 @@ Wenzhu Man Konstantin Lopuhin John Witulski + Jeremy Thurgood Greg Price Ivan Sichmann Freitas Dario Bertini - Jeremy Thurgood Mark Pearse Simon Cross Tobias Pape @@ -112,18 +112,19 @@ Adrian Kuhn tav Georg Brandl + Joannah Nanjekye Bert Freudenberg Stian Andreassen Wanja Saatkamp Mike Blume - Joannah Nanjekye Gerald Klix Oscar Nierstrasz Rami Chowdhury Stefan H. Muller + Dodan Mihai Tim Felgentreff Eugene Oden - Dodan Mihai + Colin Valliant Jeff Terrace Henry Mason Vasily Kuznetsov @@ -192,12 +193,14 @@ Vaibhav Sood Reuben Cummings Attila Gobi + Floris Bruynooghe Christopher Pope Tristan Arthur Christian Tismer Dan Stromberg Carl Meyer Florin Papa + Arianna Avanzini Jens-Uwe Mager Valentina Mukhamedzhanova Stefano Parmesan @@ -221,6 +224,7 @@ Mark Williams Kunal Grover Nathan Taylor + Barry Hart Travis Francis Athougies Yasir Suhail Sergey Kishchenko @@ -228,6 +232,7 @@ Lutz Paelike Ian Foote Philipp Rustemeuer + Logan Chien Catalin Gabriel Manciu Jacob Oscarson Ryan Gonzalez @@ -263,7 +268,6 @@ Akira Li Gustavo Niemeyer Rafał Gałczyński - Logan Chien Lucas Stadler roberto at goyle Matt Bogosian @@ -276,6 +280,7 @@ Anna Katrina Dominguez Kim Jin Su Amber Brown + Miro Hrončok Anthony Sottile Nate Bragg Ben Darnell @@ -283,7 +288,6 @@ Godefroid Chappelle Julian Berman Michael Hudson-Doyle - Floris Bruynooghe Stephan Busemann Dan Colish timo @@ -325,6 +329,7 @@ Michael Chermside Anna Ravencroft remarkablerocket + Pauli Virtanen Petre Vijiac Berker Peksag Christian Muirhead @@ -349,6 +354,7 @@ Graham Markall Dan Loewenherz werat + Andrew Stepanov Niclas Olofsson Chris Pressey Tobias Diaz diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-v6.0.0.rst release-v5.10.1.rst release-v5.10.0.rst release-v5.9.0.rst diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -7,6 +7,8 @@ .. toctree:: whatsnew-head.rst + whatsnew-pypy2-6.0.0.rst + whatsnew-pypy2-5.10.0.rst whatsnew-pypy2-5.10.0.rst whatsnew-pypy2-5.9.0.rst whatsnew-pypy2-5.8.0.rst diff --git a/pypy/doc/release-v6.0.0.rst b/pypy/doc/release-v6.0.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-v6.0.0.rst @@ -0,0 +1,102 @@ +====================================== +PyPy2.7 and PyPy3.5 v6.0 dual release +====================================== + +The PyPy team is proud to release both PyPy2.7 v6.0 (an interpreter supporting +Python 2.7 syntax), and a PyPy3.5 v6.0 (an interpreter supporting Python +3.5 syntax). The two releases are both based on much the same codebase, thus +the dual release. + +This release is a feature release following our previous 5.10 incremental +release in late December 2017, with many improvements in the C-API +compatability layer and other improvements in speed and CPython compatibility. +Since the changes affect the included python development header files, all +c-extension modules must be recompiled for this version. + +The Windows PyPy3.5 release is still considered beta-quality. There are issues +with unicode handling especially around system calls and c-extensions. + +The Matplotlib TkAgg backend now works with PyPy. PyGame and pygtk also now can +work with PyPy. + +As always, this release is 100% compatible with the previous one and fixed +several issues and bugs raised by the growing community of PyPy users. +We strongly recommend updating. + +The utf8 branch that changes internal representation of unicode to utf8 did not +make it into the release. We also began working on a Python3.6 implementation, +help is welcome. + +We updated the cffi module included in PyPy to version 1.11.5 + +You can download the v6.0 releases here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project. If PyPy is not quite good enough for your needs, we are available for +direct consulting work. + +We would also like to thank our contributors and encourage new people to join +the project. PyPy has many layers and we need help with all of them: `PyPy`_ +and `RPython`_ documentation improvements, tweaking popular `modules`_ to run +on pypy, or general `help`_ with making RPython's JIT even better. + +.. _`PyPy`: index.html +.. _`RPython`: https://rpython.readthedocs.org +.. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly +.. _`help`: project-ideas.html + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7 and CPython 3.5. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +The PyPy release supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html + +Changelog +========= + +* Speed up C-API method calls, and make most Py*_Check calls C macros +* Speed up C-API slot method calls +* Enable TkAgg backend support for matplotlib +* support ``hastzinfo`` and ``tzinfo`` in the C-API ``PyDateTime*`` structures +* datetime.h is now more similar to CPython +* We now support ``PyUnicode_AsUTF{16,32}String``, ``_PyLong_AsByteArray``, + ``_PyLong_AsByteArrayO``, +* PyPy3.5 on Windows is compiled with the Microsoft Visual Compiler v14, like + CPython +* Fix performance of attribute lookup when more than 80 attributes are used +* Improve performance on passing built-in types to C-API C code +* Improve the performance of datetime and timedelta by skipping the consistency + checks of the datetime values (they are correct by construction) +* Improve handling of ``bigint`` s, including fixing ``int_divmod`` +* Improve reporting of GC statistics +* Accept unicode filenames in ``dbm.open()`` +* Improve RPython support for half-floats +* Added missing attributes to C-API ``instancemethod`` on pypy3 +* Store error state in thread-local storage for C-API. +* Fix JIT bugs exposed in the sre module +* Improve speed of Python parser, improve ParseError messages slightly +* Handle JIT hooks more efficiently + +We also refactored many parts of the JIT bridge optimizations, as well as cpyext +internals, and together with new contributors fixed issues, added new +documentation, and cleaned up the codebase. diff --git a/pypy/doc/whatsnew-6.0.0.rst b/pypy/doc/whatsnew-pypy2-6.0.0.rst rename from pypy/doc/whatsnew-6.0.0.rst rename to pypy/doc/whatsnew-pypy2-6.0.0.rst diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh --- a/pypy/tool/release/repackage.sh +++ b/pypy/tool/release/repackage.sh @@ -1,8 +1,8 @@ # Edit these appropriately before running this script pmaj=2 # python main version pmin=7 # python minor version -maj=5 -min=8 +maj=6 +min=0 rev=0 branchname=release-pypy$pmaj.$pmin-$maj.x # ==OR== release-$maj.x # ==OR== release-$maj.$min.x tagname=release-pypy$pmaj.$pmin-v$maj.$min.$rev # ==OR== release-$maj.$min From pypy.commits at gmail.com Thu Apr 5 07:43:30 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 05 Apr 2018 04:43:30 -0700 (PDT) Subject: [pypy-commit] pypy gc-hooks: argh, this was a bad bug: make sure to clear action._next after perform(), else the next time we put it in the queue, the ._next link will point to a spurious action Message-ID: <5ac60be2.568bdf0a.bacd0.d8cf@mx.google.com> Author: Antonio Cuni Branch: gc-hooks Changeset: r94248:df2490d5d814 Date: 2018-04-05 13:41 +0200 http://bitbucket.org/pypy/pypy/changeset/df2490d5d814/ Log: argh, this was a bad bug: make sure to clear action._next after perform(), else the next time we put it in the queue, the ._next link will point to a spurious action diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -430,6 +430,7 @@ @rgc.no_collect def _fired_actions_append(self, action): + assert action._next is None if self._fired_actions_first is None: self._fired_actions_first = action self._fired_actions_last = action @@ -494,7 +495,7 @@ while action is not None: action._fired = False action.perform(ec, frame) - action = action._next + action._next, action = None, action._next self.action_dispatcher = action_dispatcher diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py --- a/pypy/interpreter/test/test_executioncontext.py +++ b/pypy/interpreter/test/test_executioncontext.py @@ -37,6 +37,37 @@ pass assert i == 9 + def test_action_queue(self): + events = [] + + class Action1(executioncontext.AsyncAction): + def perform(self, ec, frame): + events.append('one') + + class Action2(executioncontext.AsyncAction): + def perform(self, ec, frame): + events.append('two') + + space = self.space + a1 = Action1(space) + a2 = Action2(space) + a1.fire() + a2.fire() + space.appexec([], """(): + n = 5 + return n + 2 + """) + assert events == ['one', 'two'] + # + events[:] = [] + a1.fire() + space.appexec([], """(): + n = 5 + return n + 2 + """) + assert events == ['one'] + + def test_periodic_action(self): from pypy.interpreter.executioncontext import ActionFlag diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py --- a/pypy/module/gc/test/test_hook.py +++ b/pypy/module/gc/test/test_hook.py @@ -21,9 +21,16 @@ def fire_gc_collect(space, a, b, c, d, e, f): gchooks.fire_gc_collect(a, b, c, d, e, f) + @unwrap_spec(ObjSpace) + def fire_many(space): + gchooks.fire_gc_minor(0, 0) + gchooks.fire_gc_collect_step(0, 0) + gchooks.fire_gc_collect(1, 2, 3, 4, 5, 6) + cls.w_fire_gc_minor = space.wrap(interp2app(fire_gc_minor)) cls.w_fire_gc_collect_step = space.wrap(interp2app(fire_gc_collect_step)) cls.w_fire_gc_collect = space.wrap(interp2app(fire_gc_collect)) + cls.w_fire_many = space.wrap(interp2app(fire_many)) def test_on_gc_minor(self): import gc @@ -98,3 +105,19 @@ assert S.STATE_SWEEPING == 2 assert S.STATE_FINALIZING == 3 assert S.GC_STATES == ('SCANNING', 'MARKING', 'SWEEPING', 'FINALIZING') + + def test_clear_queue(self): + import gc + lst = [] + def on_gc_minor(stats): lst.append('minor') + def on_gc_collect_step(stats): lst.append('step') + def on_gc_collect(stats): lst.append('collect') + gc.set_hooks(on_gc_minor=on_gc_minor, + on_gc_collect_step=on_gc_collect_step, + on_gc_collect=on_gc_collect) + # + self.fire_many() + assert lst == ['minor', 'step', 'collect'] + lst[:] = [] + self.fire_gc_minor(0, 0) + assert lst == ['minor'] From pypy.commits at gmail.com Thu Apr 5 08:50:04 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 05 Apr 2018 05:50:04 -0700 (PDT) Subject: [pypy-commit] pypy default: emphasize cpyext speed improvements Message-ID: <5ac61b7c.138fdf0a.5495c.e154@mx.google.com> Author: Matti Picus Branch: Changeset: r94249:c52ab484f81b Date: 2018-04-05 15:49 +0300 http://bitbucket.org/pypy/pypy/changeset/c52ab484f81b/ Log: emphasize cpyext speed improvements diff --git a/pypy/doc/release-v6.0.0.rst b/pypy/doc/release-v6.0.0.rst --- a/pypy/doc/release-v6.0.0.rst +++ b/pypy/doc/release-v6.0.0.rst @@ -8,13 +8,14 @@ the dual release. This release is a feature release following our previous 5.10 incremental -release in late December 2017, with many improvements in the C-API -compatability layer and other improvements in speed and CPython compatibility. -Since the changes affect the included python development header files, all -c-extension modules must be recompiled for this version. +release in late December 2017. Our C-API compatability layer ``cpyext`` is +now much faster (see the `blog post`_) as well as more complete. We have made +many other improvements in speed and CPython compatibility. Since the changes +affect the included python development header files, all c-extension modules must +be recompiled for this version. -The Windows PyPy3.5 release is still considered beta-quality. There are issues -with unicode handling especially around system calls and c-extensions. +The Windows PyPy3.5 release is still considered beta-quality. There are open +issues with unicode handling especially around system calls and c-extensions. The Matplotlib TkAgg backend now works with PyPy. PyGame and pygtk also now can work with PyPy. @@ -23,11 +24,11 @@ several issues and bugs raised by the growing community of PyPy users. We strongly recommend updating. +We updated the cffi module included in PyPy to version 1.11.5 + The utf8 branch that changes internal representation of unicode to utf8 did not -make it into the release. We also began working on a Python3.6 implementation, -help is welcome. - -We updated the cffi module included in PyPy to version 1.11.5 +make it into the release, so there is still more goodness coming. We also +began working on a Python3.6 implementation, help is welcome. You can download the v6.0 releases here: @@ -46,6 +47,7 @@ .. _`RPython`: https://rpython.readthedocs.org .. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly .. _`help`: project-ideas.html +.. _`blog post`: https://morepypy.blogspot.it/2017/10/cape-of-good-hope-for-pypy-hello-from.html What is PyPy? ============= From pypy.commits at gmail.com Thu Apr 5 08:53:19 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 05 Apr 2018 05:53:19 -0700 (PDT) Subject: [pypy-commit] pypy gc-hooks: chage the API: instead of using the ugly set_hook (and having to implement get_on_gc_*), we expose a hooks object on which we can simply get/set the various on_gc_* fields Message-ID: <5ac61c3f.39b0df0a.1925c.f84a@mx.google.com> Author: Antonio Cuni Branch: gc-hooks Changeset: r94250:47b121de440b Date: 2018-04-05 14:28 +0200 http://bitbucket.org/pypy/pypy/changeset/47b121de440b/ Log: chage the API: instead of using the ugly set_hook (and having to implement get_on_gc_*), we expose a hooks object on which we can simply get/set the various on_gc_* fields diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py --- a/pypy/module/gc/__init__.py +++ b/pypy/module/gc/__init__.py @@ -34,7 +34,7 @@ 'get_typeids_z': 'referents.get_typeids_z', 'get_typeids_list': 'referents.get_typeids_list', 'GcRef': 'referents.W_GcRef', - 'set_hooks': 'hook.set_hooks', + 'hooks': 'space.fromcache(hook.W_AppLevelHooks)', 'GcCollectStepStats': 'hook.W_GcCollectStepStats', }) MixedModule.__init__(self, space, w_name) diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py --- a/pypy/module/gc/hook.py +++ b/pypy/module/gc/hook.py @@ -4,14 +4,14 @@ from rpython.rlib.rarithmetic import r_uint from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault from pypy.interpreter.baseobjspace import W_Root -from pypy.interpreter.typedef import TypeDef, interp_attrproperty +from pypy.interpreter.typedef import TypeDef, interp_attrproperty, GetSetProperty from pypy.interpreter.executioncontext import AsyncAction class LowLevelGcHooks(GcHooks): def __init__(self, space): self.space = space - self.hooks = space.fromcache(AppLevelHooks) + self.hooks = space.fromcache(W_AppLevelHooks) def is_gc_minor_enabled(self): return self.hooks.gc_minor_enabled @@ -47,7 +47,7 @@ action.fire() -class AppLevelHooks(object): +class W_AppLevelHooks(W_Root): def __init__(self, space): self.space = space @@ -58,26 +58,39 @@ self.gc_collect_step = GcCollectStepHookAction(space) self.gc_collect = GcCollectHookAction(space) - def set_hooks(self, space, w_on_gc_minor, w_on_gc_collect_step, - w_on_gc_collect): - self.gc_minor_enabled = not space.is_none(w_on_gc_minor) - self.gc_minor.w_callable = w_on_gc_minor + def descr_get_on_gc_minor(self, space): + return self.gc_minor.w_callable + + def descr_set_on_gc_minor(self, space, w_obj): + self.gc_minor_enabled = not space.is_none(w_obj) + self.gc_minor.w_callable = w_obj self.gc_minor.fix_annotation() - # - self.gc_collect_step_enabled = not space.is_none(w_on_gc_collect_step) - self.gc_collect_step.w_callable = w_on_gc_collect_step + + def descr_get_on_gc_collect_step(self, space): + return self.gc_collect_step.w_callable + + def descr_set_on_gc_collect_step(self, space, w_obj): + self.gc_collect_step_enabled = not space.is_none(w_obj) + self.gc_collect_step.w_callable = w_obj self.gc_collect_step.fix_annotation() - # - self.gc_collect_enabled = not space.is_none(w_on_gc_collect) - self.gc_collect.w_callable = w_on_gc_collect + + def descr_get_on_gc_collect(self, space): + return self.gc_collect.w_callable + + def descr_set_on_gc_collect(self, space, w_obj): + self.gc_collect_enabled = not space.is_none(w_obj) + self.gc_collect.w_callable = w_obj self.gc_collect.fix_annotation() class GcMinorHookAction(AsyncAction): - w_callable = None total_memory_used = 0 pinned_objects = 0 + def __init__(self, space): + AsyncAction.__init__(self, space) + self.w_callable = space.w_None + def fix_annotation(self): # the annotation of the class and its attributes must be completed # BEFORE we do the gc transform; this makes sure that everything is @@ -93,10 +106,13 @@ class GcCollectStepHookAction(AsyncAction): - w_callable = None oldstate = 0 newstate = 0 + def __init__(self, space): + AsyncAction.__init__(self, space) + self.w_callable = space.w_None + def fix_annotation(self): # the annotation of the class and its attributes must be completed # BEFORE we do the gc transform; this makes sure that everything is @@ -112,7 +128,6 @@ class GcCollectHookAction(AsyncAction): - w_callable = None count = 0 arenas_count_before = 0 arenas_count_after = 0 @@ -120,6 +135,10 @@ rawmalloc_bytes_before = 0 rawmalloc_bytes_after = 0 + def __init__(self, space): + AsyncAction.__init__(self, space) + self.w_callable = space.w_None + def fix_annotation(self): # the annotation of the class and its attributes must be completed # BEFORE we do the gc transform; this makes sure that everything is @@ -176,6 +195,22 @@ d[name] = interp_attrproperty(name, cls=cls, wrapfn="newint") return d + +W_AppLevelHooks.typedef = TypeDef( + "GcHooks", + on_gc_minor = GetSetProperty( + W_AppLevelHooks.descr_get_on_gc_minor, + W_AppLevelHooks.descr_set_on_gc_minor), + + on_gc_collect_step = GetSetProperty( + W_AppLevelHooks.descr_get_on_gc_collect_step, + W_AppLevelHooks.descr_set_on_gc_collect_step), + + on_gc_collect = GetSetProperty( + W_AppLevelHooks.descr_get_on_gc_collect, + W_AppLevelHooks.descr_set_on_gc_collect), + ) + W_GcMinorStats.typedef = TypeDef( "GcMinorStats", **wrap_many_ints(W_GcMinorStats, ( @@ -205,14 +240,3 @@ "rawmalloc_bytes_before", "rawmalloc_bytes_after")) ) - - - at unwrap_spec(w_on_gc_minor=WrappedDefault(None), - w_on_gc_collect_step=WrappedDefault(None), - w_on_gc_collect=WrappedDefault(None)) -def set_hooks(space, w_on_gc_minor=None, - w_on_gc_collect_step=None, - w_on_gc_collect=None): - hooks = space.fromcache(AppLevelHooks) - hooks.set_hooks(space, w_on_gc_minor, w_on_gc_collect_step, w_on_gc_collect) - diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py --- a/pypy/module/gc/test/test_hook.py +++ b/pypy/module/gc/test/test_hook.py @@ -32,12 +32,18 @@ cls.w_fire_gc_collect = space.wrap(interp2app(fire_gc_collect)) cls.w_fire_many = space.wrap(interp2app(fire_many)) + def test_default(self): + import gc + assert gc.hooks.on_gc_minor is None + assert gc.hooks.on_gc_collect_step is None + assert gc.hooks.on_gc_collect is None + def test_on_gc_minor(self): import gc lst = [] def on_gc_minor(stats): lst.append((stats.total_memory_used, stats.pinned_objects)) - gc.set_hooks(on_gc_minor=on_gc_minor) + gc.hooks.on_gc_minor = on_gc_minor self.fire_gc_minor(10, 20) self.fire_gc_minor(30, 40) assert lst == [ @@ -45,7 +51,7 @@ (30, 40), ] # - gc.set_hooks(on_gc_minor=None) + gc.hooks.on_gc_minor = None self.fire_gc_minor(50, 60) # won't fire because the hooks is disabled assert lst == [ (10, 20), @@ -57,7 +63,7 @@ lst = [] def on_gc_collect_step(stats): lst.append((stats.oldstate, stats.newstate)) - gc.set_hooks(on_gc_collect_step=on_gc_collect_step) + gc.hooks.on_gc_collect_step = on_gc_collect_step self.fire_gc_collect_step(10, 20) self.fire_gc_collect_step(30, 40) assert lst == [ @@ -65,7 +71,7 @@ (30, 40), ] # - gc.set_hooks(on_gc_collect_step=None) + gc.hooks.on_gc_collect_step = None self.fire_gc_collect_step(50, 60) # won't fire assert lst == [ (10, 20), @@ -82,7 +88,7 @@ stats.arenas_bytes, stats.rawmalloc_bytes_before, stats.rawmalloc_bytes_after)) - gc.set_hooks(on_gc_collect=on_gc_collect) + gc.hooks.on_gc_collect = on_gc_collect self.fire_gc_collect(1, 2, 3, 4, 5, 6) self.fire_gc_collect(7, 8, 9, 10, 11, 12) assert lst == [ @@ -90,7 +96,7 @@ (7, 8, 9, 10, 11, 12), ] # - gc.set_hooks(on_gc_collect=None) + gc.hooks.on_gc_collect = None self.fire_gc_collect(42, 42, 42, 42, 42, 42) # won't fire assert lst == [ (1, 2, 3, 4, 5, 6), @@ -112,9 +118,9 @@ def on_gc_minor(stats): lst.append('minor') def on_gc_collect_step(stats): lst.append('step') def on_gc_collect(stats): lst.append('collect') - gc.set_hooks(on_gc_minor=on_gc_minor, - on_gc_collect_step=on_gc_collect_step, - on_gc_collect=on_gc_collect) + gc.hooks.on_gc_minor = on_gc_minor + gc.hooks.on_gc_collect_step = on_gc_collect_step + gc.hooks.on_gc_collect = on_gc_collect # self.fire_many() assert lst == ['minor', 'step', 'collect'] From pypy.commits at gmail.com Thu Apr 5 08:53:21 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 05 Apr 2018 05:53:21 -0700 (PDT) Subject: [pypy-commit] pypy gc-hooks: this is a wrapped class now, better to use the w_* naming convention Message-ID: <5ac61c41.77a9df0a.6b67a.8431@mx.google.com> Author: Antonio Cuni Branch: gc-hooks Changeset: r94251:1298b7800106 Date: 2018-04-05 14:29 +0200 http://bitbucket.org/pypy/pypy/changeset/1298b7800106/ Log: this is a wrapped class now, better to use the w_* naming convention diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py --- a/pypy/module/gc/hook.py +++ b/pypy/module/gc/hook.py @@ -11,25 +11,25 @@ def __init__(self, space): self.space = space - self.hooks = space.fromcache(W_AppLevelHooks) + self.w_hooks = space.fromcache(W_AppLevelHooks) def is_gc_minor_enabled(self): - return self.hooks.gc_minor_enabled + return self.w_hooks.gc_minor_enabled def is_gc_collect_step_enabled(self): - return self.hooks.gc_collect_step_enabled + return self.w_hooks.gc_collect_step_enabled def is_gc_collect_enabled(self): - return self.hooks.gc_collect_enabled + return self.w_hooks.gc_collect_enabled def on_gc_minor(self, total_memory_used, pinned_objects): - action = self.hooks.gc_minor + action = self.w_hooks.gc_minor action.total_memory_used = total_memory_used action.pinned_objects = pinned_objects action.fire() def on_gc_collect_step(self, oldstate, newstate): - action = self.hooks.gc_collect_step + action = self.w_hooks.gc_collect_step action.oldstate = oldstate action.newstate = newstate action.fire() @@ -37,7 +37,7 @@ def on_gc_collect(self, count, arenas_count_before, arenas_count_after, arenas_bytes, rawmalloc_bytes_before, rawmalloc_bytes_after): - action = self.hooks.gc_collect + action = self.w_hooks.gc_collect action.count = count action.arenas_count_before = arenas_count_before action.arenas_count_after = arenas_count_after From pypy.commits at gmail.com Thu Apr 5 08:53:23 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 05 Apr 2018 05:53:23 -0700 (PDT) Subject: [pypy-commit] pypy gc-hooks: docstring Message-ID: <5ac61c43.5b88df0a.7536f.2711@mx.google.com> Author: Antonio Cuni Branch: gc-hooks Changeset: r94252:bbdde3a6fc01 Date: 2018-04-05 14:39 +0200 http://bitbucket.org/pypy/pypy/changeset/bbdde3a6fc01/ Log: docstring diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py --- a/pypy/module/gc/hook.py +++ b/pypy/module/gc/hook.py @@ -8,6 +8,15 @@ from pypy.interpreter.executioncontext import AsyncAction class LowLevelGcHooks(GcHooks): + """ + These are the low-level hooks which are called directly from the GC. + + They can't do much, because the base class marks the methods as + @rgc.no_collect. + + This is expected to be a singleton, created by space.fromcache, and it is + integrated with the translation by targetpypystandalone.get_gchooks + """ def __init__(self, space): self.space = space From pypy.commits at gmail.com Thu Apr 5 08:53:25 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 05 Apr 2018 05:53:25 -0700 (PDT) Subject: [pypy-commit] pypy gc-hooks: add two convenience methods: hooks.set to pass an instance which contains the appropriate on_gc_* methods, and hooks.reset() to set all the hooks to None Message-ID: <5ac61c45.09e61c0a.2b958.cf3c@mx.google.com> Author: Antonio Cuni Branch: gc-hooks Changeset: r94253:db58a930e0f9 Date: 2018-04-05 14:52 +0200 http://bitbucket.org/pypy/pypy/changeset/db58a930e0f9/ Log: add two convenience methods: hooks.set to pass an instance which contains the appropriate on_gc_* methods, and hooks.reset() to set all the hooks to None diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py --- a/pypy/module/gc/hook.py +++ b/pypy/module/gc/hook.py @@ -91,6 +91,19 @@ self.gc_collect.w_callable = w_obj self.gc_collect.fix_annotation() + def descr_set(self, space, w_obj): + w_a = space.getattr(w_obj, space.newtext('on_gc_minor')) + w_b = space.getattr(w_obj, space.newtext('on_gc_collect_step')) + w_c = space.getattr(w_obj, space.newtext('on_gc_collect')) + self.descr_set_on_gc_minor(space, w_a) + self.descr_set_on_gc_collect_step(space, w_b) + self.descr_set_on_gc_collect(space, w_c) + + def descr_reset(self, space): + self.descr_set_on_gc_minor(space, space.w_None) + self.descr_set_on_gc_collect_step(space, space.w_None) + self.descr_set_on_gc_collect(space, space.w_None) + class GcMinorHookAction(AsyncAction): total_memory_used = 0 @@ -218,6 +231,9 @@ on_gc_collect = GetSetProperty( W_AppLevelHooks.descr_get_on_gc_collect, W_AppLevelHooks.descr_set_on_gc_collect), + + set = interp2app(W_AppLevelHooks.descr_set), + reset = interp2app(W_AppLevelHooks.descr_reset), ) W_GcMinorStats.typedef = TypeDef( diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py --- a/pypy/module/gc/test/test_hook.py +++ b/pypy/module/gc/test/test_hook.py @@ -114,16 +114,28 @@ def test_clear_queue(self): import gc - lst = [] - def on_gc_minor(stats): lst.append('minor') - def on_gc_collect_step(stats): lst.append('step') - def on_gc_collect(stats): lst.append('collect') - gc.hooks.on_gc_minor = on_gc_minor - gc.hooks.on_gc_collect_step = on_gc_collect_step - gc.hooks.on_gc_collect = on_gc_collect - # + class MyHooks(object): + + def __init__(self): + self.lst = [] + + def on_gc_minor(self, stats): + self.lst.append('minor') + + def on_gc_collect_step(self, stats): + self.lst.append('step') + + def on_gc_collect(self, stats): + self.lst.append('collect') + + myhooks = MyHooks() + gc.hooks.set(myhooks) self.fire_many() - assert lst == ['minor', 'step', 'collect'] - lst[:] = [] + assert myhooks.lst == ['minor', 'step', 'collect'] + myhooks.lst[:] = [] self.fire_gc_minor(0, 0) - assert lst == ['minor'] + assert myhooks.lst == ['minor'] + gc.hooks.reset() + assert gc.hooks.on_gc_minor is None + assert gc.hooks.on_gc_collect_step is None + assert gc.hooks.on_gc_collect is None From pypy.commits at gmail.com Thu Apr 5 10:37:40 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 05 Apr 2018 07:37:40 -0700 (PDT) Subject: [pypy-commit] buildbot default: add bencher4_32 slave (thanks baroquesoftware.com) Message-ID: <5ac634b4.77a9df0a.6b67a.a162@mx.google.com> Author: Matti Picus Branch: Changeset: r1061:e72d9ed2d204 Date: 2018-04-05 17:37 +0300 http://bitbucket.org/pypy/buildbot/changeset/e72d9ed2d204/ Log: add bencher4_32 slave (thanks baroquesoftware.com) diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -418,14 +418,14 @@ 'builders': [ {"name": LINUX32OWN, - "slavenames": ["tannit32"], + "slavenames": ["tannit32", "bencher4_32"], "builddir": LINUX32OWN, "factory": pypyOwnTestFactory, "category": 'linux32', "locks": [TannitCPU.access('counting')], }, {"name": LINUX32RPYTHON, - "slavenames": ["tannit32"], + "slavenames": ["tannit32", "bencher4_32"], "builddir": LINUX32RPYTHON, "factory": pypyRPythonTestFactory, "category": 'linux32', @@ -449,7 +449,7 @@ }, {"name": APPLVLLINUX32, #"slavenames": ["allegro32"], - "slavenames": ["tannit32"], + "slavenames": ["tannit32", "bencher4_32"], "builddir": APPLVLLINUX32, "factory": pypyTranslatedAppLevelTestFactory, 'category': 'linux32', @@ -464,7 +464,7 @@ "locks": [Bencher4Lock.access('counting')], }, {"name": LIBPYTHON_LINUX32, - "slavenames": ["tannit32"], + "slavenames": ["tannit32", "bencher4_32"], #"slavenames": ["allegro32"], "builddir": LIBPYTHON_LINUX32, "factory": pypyTranslatedLibPythonTestFactory, @@ -481,7 +481,7 @@ }, {"name" : JITLINUX32, #"slavenames": ["allegro32"], - "slavenames": ["tannit32"], + "slavenames": ["tannit32", "bencher4_32"], 'builddir' : JITLINUX32, 'factory' : pypyJITTranslatedTestFactory, 'category' : 'linux32', From pypy.commits at gmail.com Thu Apr 5 13:18:22 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 05 Apr 2018 10:18:22 -0700 (PDT) Subject: [pypy-commit] pypy gc-hooks: fix this test, and add comment explaing why it was failing Message-ID: <5ac65a5e.14a1df0a.16040.a5ca@mx.google.com> Author: Antonio Cuni Branch: gc-hooks Changeset: r94255:580dd80b842f Date: 2018-04-05 19:17 +0200 http://bitbucket.org/pypy/pypy/changeset/580dd80b842f/ Log: fix this test, and add comment explaing why it was failing diff --git a/pypy/module/gc/test/test_ztranslation.py b/pypy/module/gc/test/test_ztranslation.py --- a/pypy/module/gc/test/test_ztranslation.py +++ b/pypy/module/gc/test/test_ztranslation.py @@ -1,4 +1,9 @@ from pypy.objspace.fake.checkmodule import checkmodule def test_checkmodule(): - checkmodule('gc') + # we need to ignore GcCollectStepStats, else checkmodule fails. I think + # this happens because W_GcCollectStepStats.__init__ is only called from + # GcCollectStepHookAction.perform() and the fake objspace doesn't know + # about those: so, perform() is never annotated and the annotator thinks + # W_GcCollectStepStats has no attributes + checkmodule('gc', ignore=['GcCollectStepStats']) diff --git a/pypy/objspace/fake/checkmodule.py b/pypy/objspace/fake/checkmodule.py --- a/pypy/objspace/fake/checkmodule.py +++ b/pypy/objspace/fake/checkmodule.py @@ -4,6 +4,7 @@ def checkmodule(*modnames, **kwds): translate_startup = kwds.pop('translate_startup', True) + ignore = set(kwds.pop('ignore', ())) assert not kwds config = get_pypy_config(translating=True) space = FakeObjSpace(config) @@ -17,6 +18,8 @@ module.init(space) modules.append(module) for name in module.loaders: + if name in ignore: + continue seeobj_w.append(module._load_lazily(space, name)) if hasattr(module, 'submodules'): for cls in module.submodules.itervalues(): From pypy.commits at gmail.com Thu Apr 5 13:18:19 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 05 Apr 2018 10:18:19 -0700 (PDT) Subject: [pypy-commit] pypy gc-hooks: we can't run these tests with -A, skip them Message-ID: <5ac65a5b.8b421c0a.53a73.95e4@mx.google.com> Author: Antonio Cuni Branch: gc-hooks Changeset: r94254:4f5f4f37faf3 Date: 2018-04-05 18:59 +0200 http://bitbucket.org/pypy/pypy/changeset/4f5f4f37faf3/ Log: we can't run these tests with -A, skip them diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py --- a/pypy/module/gc/test/test_hook.py +++ b/pypy/module/gc/test/test_hook.py @@ -1,3 +1,4 @@ +import pytest from rpython.rlib.rarithmetic import r_uint from pypy.module.gc.hook import LowLevelGcHooks from pypy.interpreter.baseobjspace import ObjSpace @@ -6,6 +7,8 @@ class AppTestGcHooks(object): def setup_class(cls): + if cls.runappdirect: + pytest.skip("these tests cannot work with -A") space = cls.space gchooks = space.fromcache(LowLevelGcHooks) From pypy.commits at gmail.com Thu Apr 5 20:01:09 2018 From: pypy.commits at gmail.com (rlamy) Date: Thu, 05 Apr 2018 17:01:09 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Create W_IntObject instead of W_LongObject from space.newlong_from_rbigint() Message-ID: <5ac6b8c5.89ce1c0a.45b5d.20bc@mx.google.com> Author: Ronan Lamy Branch: py3.5 Changeset: r94256:0af136eb60af Date: 2018-04-06 00:59 +0100 http://bitbucket.org/pypy/pypy/changeset/0af136eb60af/ Log: Create W_IntObject instead of W_LongObject from space.newlong_from_rbigint() whenever possible. In particular, make sure that unmarshalling an int constant that fits in 64 bits gives a W_IntObject. Fixes #2785. diff --git a/pypy/module/marshal/test/test_marshal.py b/pypy/module/marshal/test/test_marshal.py --- a/pypy/module/marshal/test/test_marshal.py +++ b/pypy/module/marshal/test/test_marshal.py @@ -229,7 +229,7 @@ BadReader(marshal.dumps(value))) - at pytest.mark.skipif('config.option.runappdirect') + at pytest.mark.skipif('config.option.runappdirect or sys.maxint > 2 ** 32') class AppTestSmallLong(AppTestMarshal): spaceconfig = AppTestMarshal.spaceconfig.copy() spaceconfig["objspace.std.withsmalllong"] = True diff --git a/pypy/module/marshal/test/test_marshalimpl.py b/pypy/module/marshal/test/test_marshalimpl.py --- a/pypy/module/marshal/test/test_marshalimpl.py +++ b/pypy/module/marshal/test/test_marshalimpl.py @@ -1,5 +1,6 @@ from pypy.module.marshal import interp_marshal from pypy.interpreter.error import OperationError +from pypy.objspace.std.intobject import W_IntObject import sys @@ -100,3 +101,15 @@ for i in range(100): _marshal_check(sign * ((1L << i) - 1L)) _marshal_check(sign * (1L << i)) + +def test_int_roundtrip(space): + a = 0xffffffff + w_a = space.newint(a) + m = interp_marshal.StringMarshaller(space, 4) + interp_marshal.marshal(space, w_a, m) + s = m.get_value() + u = interp_marshal.StringUnmarshaller(space, space.newbytes(s)) + w_res = u.load_w_obj() + + assert type(w_res) is W_IntObject + assert w_res.intval == w_a.intval == a diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py --- a/pypy/objspace/std/objspace.py +++ b/pypy/objspace/std/objspace.py @@ -312,7 +312,10 @@ return W_LongObject.fromrarith_int(val) def newlong_from_rbigint(self, val): - return newlong(self, val) + try: + return self.newint(val.toint()) + except OverflowError: + return newlong(self, val) def newtuple(self, list_w): from pypy.objspace.std.tupleobject import wraptuple From pypy.commits at gmail.com Thu Apr 5 20:01:12 2018 From: pypy.commits at gmail.com (rlamy) Date: Thu, 05 Apr 2018 17:01:12 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: merge heads Message-ID: <5ac6b8c8.ce9ddf0a.480d9.3832@mx.google.com> Author: Ronan Lamy Branch: py3.5 Changeset: r94257:4b1bc362c207 Date: 2018-04-06 01:00 +0100 http://bitbucket.org/pypy/pypy/changeset/4b1bc362c207/ Log: merge heads diff --git a/pypy/module/marshal/test/test_marshal.py b/pypy/module/marshal/test/test_marshal.py --- a/pypy/module/marshal/test/test_marshal.py +++ b/pypy/module/marshal/test/test_marshal.py @@ -229,7 +229,7 @@ BadReader(marshal.dumps(value))) - at pytest.mark.skipif('config.option.runappdirect') + at pytest.mark.skipif('config.option.runappdirect or sys.maxint > 2 ** 32') class AppTestSmallLong(AppTestMarshal): spaceconfig = AppTestMarshal.spaceconfig.copy() spaceconfig["objspace.std.withsmalllong"] = True diff --git a/pypy/module/marshal/test/test_marshalimpl.py b/pypy/module/marshal/test/test_marshalimpl.py --- a/pypy/module/marshal/test/test_marshalimpl.py +++ b/pypy/module/marshal/test/test_marshalimpl.py @@ -1,5 +1,6 @@ from pypy.module.marshal import interp_marshal from pypy.interpreter.error import OperationError +from pypy.objspace.std.intobject import W_IntObject import sys @@ -100,3 +101,15 @@ for i in range(100): _marshal_check(sign * ((1L << i) - 1L)) _marshal_check(sign * (1L << i)) + +def test_int_roundtrip(space): + a = 0xffffffff + w_a = space.newint(a) + m = interp_marshal.StringMarshaller(space, 4) + interp_marshal.marshal(space, w_a, m) + s = m.get_value() + u = interp_marshal.StringUnmarshaller(space, space.newbytes(s)) + w_res = u.load_w_obj() + + assert type(w_res) is W_IntObject + assert w_res.intval == w_a.intval == a diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py --- a/pypy/objspace/std/objspace.py +++ b/pypy/objspace/std/objspace.py @@ -312,7 +312,10 @@ return W_LongObject.fromrarith_int(val) def newlong_from_rbigint(self, val): - return newlong(self, val) + try: + return self.newint(val.toint()) + except OverflowError: + return newlong(self, val) def newtuple(self, list_w): from pypy.objspace.std.tupleobject import wraptuple From pypy.commits at gmail.com Fri Apr 6 08:31:55 2018 From: pypy.commits at gmail.com (mjacob) Date: Fri, 06 Apr 2018 05:31:55 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: Implement the logic that calls the async generator finalizer hook. Message-ID: <5ac768bb.cea61c0a.fb721.91f8@mx.google.com> Author: Manuel Jacob Branch: py3.6 Changeset: r94258:236031b989f8 Date: 2018-04-06 14:27 +0200 http://bitbucket.org/pypy/pypy/changeset/236031b989f8/ Log: Implement the logic that calls the async generator finalizer hook. diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py --- a/pypy/interpreter/generator.py +++ b/pypy/interpreter/generator.py @@ -3,7 +3,7 @@ from pypy.interpreter.pyopcode import LoopBlock, SApplicationException, Yield from pypy.interpreter.pycode import CO_YIELD_INSIDE_TRY from pypy.interpreter.astcompiler import consts -from rpython.rlib import jit, rgc +from rpython.rlib import jit, rgc, rweakref from rpython.rlib.objectmodel import specialize from rpython.rlib.rarithmetic import r_uint @@ -588,20 +588,52 @@ KIND = "async generator" KIND_U = u"async_generator" + def __init__(self, frame, name=None, qualname=None): + self.hooks_inited = False + GeneratorOrCoroutine.__init__(self, frame, name, qualname) + + def init_hooks(self): + if self.hooks_inited: + return + self.hooks_inited = True + + self.w_finalizer = self.space.appexec([], '''(): + import sys + hooks = sys.get_asyncgen_hooks() + return hooks.finalizer''') + + def _finalize_(self): + if self.frame is not None and self.frame.lastblock is not None: + if self.w_finalizer is not self.space.w_None: + # XXX: this is a hack to resurrect the weakref that was cleared + # before running _finalize_() + if self.space.config.translation.rweakref: + self.frame.f_generator_wref = rweakref.ref(self) + try: + self.space.call_function(self.w_finalizer, self) + except OperationError as e: + e.write_unraisable(self.space, "async generator finalizer") + return + GeneratorOrCoroutine._finalize_(self) + def descr__aiter__(self): """Return an asynchronous iterator.""" return self def descr__anext__(self): + self.init_hooks() return AsyncGenASend(self, self.space.w_None) def descr_asend(self, w_arg): + self.init_hooks() return AsyncGenASend(self, w_arg) def descr_athrow(self, w_type, w_val=None, w_tb=None): + self.init_hooks() return AsyncGenAThrow(self, w_type, w_val, w_tb) def descr_aclose(self): + self.init_hooks() return AsyncGenAThrow(self, None, None, None) diff --git a/pypy/interpreter/test/test_coroutine.py b/pypy/interpreter/test/test_coroutine.py --- a/pypy/interpreter/test/test_coroutine.py +++ b/pypy/interpreter/test/test_coroutine.py @@ -537,6 +537,56 @@ assert state == 2 """ + def test_async_aclose_in_finalize_hook_await_in_finally(self): """ + import gc + import sys + import types + + @types.coroutine + def coro(): + yield 'coro' + + state = 0 + async def ag(): + nonlocal state + try: + yield + finally: + state = 1 + await coro() + state = 2 + + async def run(): + a = ag() + async for i in a: + break + del a + gc.collect() + gc.collect() + gc.collect() + a = run() + + a2 = None + assert sys.get_asyncgen_hooks() == (None, None) + def _finalize(g): + nonlocal a2 + a2 = g.aclose() + sys.set_asyncgen_hooks(finalizer=_finalize) + assert state == 0 + try: + a.send(None) + except StopIteration: + pass + assert a2.send(None) == 'coro' + assert state == 1 + try: + a2.send(None) + except StopIteration: + pass + assert state == 2 + sys.set_asyncgen_hooks(None, None) + """ + def test_async_anext_close(self): """ async def ag(): yield 42 From pypy.commits at gmail.com Sat Apr 7 14:18:46 2018 From: pypy.commits at gmail.com (mattip) Date: Sat, 07 Apr 2018 11:18:46 -0700 (PDT) Subject: [pypy-commit] pypy default: pygobject, not pygtk (lazka) Message-ID: <5ac90b86.8edbdf0a.e7b7.e826@mx.google.com> Author: Matti Picus Branch: Changeset: r94259:766c77b4baf4 Date: 2018-04-07 21:17 +0300 http://bitbucket.org/pypy/pypy/changeset/766c77b4baf4/ Log: pygobject, not pygtk (lazka) diff --git a/pypy/doc/release-v6.0.0.rst b/pypy/doc/release-v6.0.0.rst --- a/pypy/doc/release-v6.0.0.rst +++ b/pypy/doc/release-v6.0.0.rst @@ -17,8 +17,7 @@ The Windows PyPy3.5 release is still considered beta-quality. There are open issues with unicode handling especially around system calls and c-extensions. -The Matplotlib TkAgg backend now works with PyPy. PyGame and pygtk also now can -work with PyPy. +The Matplotlib TkAgg backend now works with PyPy, as do pygame and pygobject. As always, this release is 100% compatible with the previous one and fixed several issues and bugs raised by the growing community of PyPy users. From pypy.commits at gmail.com Sat Apr 7 14:41:18 2018 From: pypy.commits at gmail.com (mattip) Date: Sat, 07 Apr 2018 11:41:18 -0700 (PDT) Subject: [pypy-commit] pypy default: test, fix for issue #2792 (resizing non-initialized PyTuple), also add fast-path optimization Message-ID: <5ac910ce.4ec11c0a.5973c.f33c@mx.google.com> Author: Matti Picus Branch: Changeset: r94260:5014ee1237e8 Date: 2018-04-07 21:40 +0300 http://bitbucket.org/pypy/pypy/changeset/5014ee1237e8/ Log: test, fix for issue #2792 (resizing non-initialized PyTuple), also add fast-path optimization diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py --- a/pypy/module/cpyext/test/test_tupleobject.py +++ b/pypy/module/cpyext/test/test_tupleobject.py @@ -67,6 +67,18 @@ assert space.int_w(space.getitem(w_tuple, space.wrap(i))) == 42 + i decref(space, ar[0]) + py_tuple = state.ccall("PyTuple_New", 1) + ar[0] = py_tuple + api._PyTuple_Resize(ar, 1) + assert api.PyTuple_Size(ar[0]) == 1 + decref(space, ar[0]) + + py_tuple = state.ccall("PyTuple_New", 1) + ar[0] = py_tuple + api._PyTuple_Resize(ar, 5) + assert api.PyTuple_Size(ar[0]) == 5 + decref(space, ar[0]) + lltype.free(ar, flavor='raw') def test_setitem(self, space, api): diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py --- a/pypy/module/cpyext/tupleobject.py +++ b/pypy/module/cpyext/tupleobject.py @@ -187,6 +187,8 @@ PyErr_BadInternalCall(space) oldref = rffi.cast(PyTupleObject, ref) oldsize = oldref.c_ob_size + if oldsize == newsize: + return 0 ptup = state.ccall("PyTuple_New", newsize) if not ptup: state.check_and_raise_exception(always=True) @@ -199,8 +201,9 @@ to_cp = newsize for i in range(to_cp): ob = oldref.c_ob_item[i] - incref(space, ob) - newref.c_ob_item[i] = ob + if ob: + incref(space, ob) + newref.c_ob_item[i] = ob except: decref(space, p_ref[0]) p_ref[0] = lltype.nullptr(PyObject.TO) From pypy.commits at gmail.com Sat Apr 7 17:47:05 2018 From: pypy.commits at gmail.com (mattip) Date: Sat, 07 Apr 2018 14:47:05 -0700 (PDT) Subject: [pypy-commit] pypy default: document merged branch Message-ID: <5ac93c59.6ca0df0a.2b35f.1afe@mx.google.com> Author: Matti Picus Branch: Changeset: r94263:ab000b7b9ac5 Date: 2018-04-08 00:46 +0300 http://bitbucket.org/pypy/pypy/changeset/ab000b7b9ac5/ Log: document merged branch diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -5,4 +5,8 @@ .. this is a revision shortly after release-pypy-6.0.0 .. startrev: 2e04adf1b89f +.. branch: cpyext-subclass-setattr +Fix for python-level classes that inherit from C-API types, previously the +`w_obj` was not necessarily preserved throughout the lifetime of the `pyobj` +which led to cases where instance attributes were lost. Fixes issue #2793 From pypy.commits at gmail.com Sat Apr 7 17:47:00 2018 From: pypy.commits at gmail.com (mattip) Date: Sat, 07 Apr 2018 14:47:00 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-subclass-setattr: close branch to be merged Message-ID: <5ac93c54.1ca0df0a.8fc5d.02a0@mx.google.com> Author: Matti Picus Branch: cpyext-subclass-setattr Changeset: r94261:cff9f23aad6e Date: 2018-04-08 00:40 +0300 http://bitbucket.org/pypy/pypy/changeset/cff9f23aad6e/ Log: close branch to be merged From pypy.commits at gmail.com Sat Apr 7 17:47:03 2018 From: pypy.commits at gmail.com (mattip) Date: Sat, 07 Apr 2018 14:47:03 -0700 (PDT) Subject: [pypy-commit] pypy default: merge cpyext-subclass-setattr which fixes cpyext pyobjects "losing" a w_obj Message-ID: <5ac93c57.69b9df0a.1ffc5.43c0@mx.google.com> Author: Matti Picus Branch: Changeset: r94262:be473ba66a18 Date: 2018-04-08 00:42 +0300 http://bitbucket.org/pypy/pypy/changeset/be473ba66a18/ Log: merge cpyext-subclass-setattr which fixes cpyext pyobjects "losing" a w_obj diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -640,7 +640,7 @@ 'Py_DivisionWarningFlag', 'Py_DontWriteBytecodeFlag', 'Py_NoUserSiteDirectory', '_Py_QnewFlag', 'Py_Py3kWarningFlag', 'Py_HashRandomizationFlag', '_Py_PackageContext', '_PyTraceMalloc_Track', '_PyTraceMalloc_Untrack', 'PyMem_Malloc', - 'Py_IncRef', 'Py_DecRef', 'PyObject_Free', 'PyObject_GC_Del', 'PyType_GenericAlloc', + 'PyObject_Free', 'PyObject_GC_Del', 'PyType_GenericAlloc', '_PyObject_New', '_PyObject_NewVar', '_PyObject_GC_New', '_PyObject_GC_NewVar', 'PyObject_Init', 'PyObject_InitVar', 'PyInt_FromLong', diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py --- a/pypy/module/cpyext/pyobject.py +++ b/pypy/module/cpyext/pyobject.py @@ -60,10 +60,10 @@ def _cpyext_attach_pyobj(self, space, py_obj): self._cpy_ref = py_obj - rawrefcount.create_link_pyobj(self, py_obj) + rawrefcount.create_link_pypy(self, py_obj) cls._cpyext_attach_pyobj = _cpyext_attach_pyobj -add_direct_pyobj_storage(W_BaseCPyObject) +add_direct_pyobj_storage(W_BaseCPyObject) add_direct_pyobj_storage(W_TypeObject) add_direct_pyobj_storage(W_NoneObject) add_direct_pyobj_storage(W_BoolObject) @@ -414,3 +414,14 @@ @cpython_api([rffi.VOIDP], lltype.Signed, error=CANNOT_FAIL) def _Py_HashPointer(space, ptr): return rffi.cast(lltype.Signed, ptr) + + at cpython_api([PyObject], lltype.Void) +def Py_IncRef(space, obj): + # used only ifdef PYPY_DEBUG_REFCOUNT + if obj: + incref(space, obj) + + at cpython_api([PyObject], lltype.Void) +def Py_DecRef(space, obj): + # used only ifdef PYPY_DEBUG_REFCOUNT + decref(space, obj) diff --git a/pypy/module/cpyext/src/object.c b/pypy/module/cpyext/src/object.c --- a/pypy/module/cpyext/src/object.c +++ b/pypy/module/cpyext/src/object.c @@ -5,18 +5,6 @@ extern void _PyPy_Free(void *ptr); extern void *_PyPy_Malloc(Py_ssize_t size); -void -Py_IncRef(PyObject *o) -{ - Py_XINCREF(o); -} - -void -Py_DecRef(PyObject *o) -{ - Py_XDECREF(o); -} - /* * The actual value of this variable will be the address of * pyobject.w_marker_deallocating, and will be set by diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c --- a/pypy/module/cpyext/test/array.c +++ b/pypy/module/cpyext/test/array.c @@ -2492,6 +2492,87 @@ return PyLong_FromLong(obj1->ob_type->tp_dealloc == obj2->ob_type->tp_dealloc); } +static PyObject * +subclass_with_attribute(PyObject *self, PyObject* args) { + /* what happens when we use tp_alloc to create the subclass, then + * assign to the w_obj via python, then get the GC to collect? + * The w_obj should not be collected!! + */ + PyObject * obj, *sub, *attrib, *funcname, *attribname, *collect, *res, *tup; + PyTypeObject * subtype; + int i; + if (!PyArg_ParseTuple(args, "OOOO", &obj, &funcname, &attribname, &collect)) { + return NULL; + } + if (!PyType_Check(obj)) { + PyErr_SetString(PyExc_TypeError, + "expected type object"); + return NULL; + } + subtype = (PyTypeObject*)obj; + sub = subtype->tp_alloc(subtype, 0); + if (!sub) { + return NULL; + } + attrib = PyObject_GetAttr(sub, funcname); + if (!attrib || (attrib == Py_None) ) { + PyErr_SetString(PyExc_ValueError, + "could not find function to call"); + Py_XDECREF(attrib); + Py_DECREF(sub); + return NULL; + } + tup = PyTuple_New(0); + /* + #ifdef PYPY_VERSION + printf("calling addattrib pypylink %lu \n", sub->ob_pypy_link); + #endif + */ + res = PyObject_Call(attrib, tup, NULL); + /* + #ifdef PYPY_VERSION + printf("after addattrib pypylink %lu \n", sub->ob_pypy_link); + #endif + */ + Py_DECREF(attrib); + if (res == NULL) { + Py_DECREF(tup); + Py_DECREF(sub); + return NULL; + } + Py_DECREF(res); + for(i=0; i<10; i++) { + /* + #ifdef PYPY_VERSION + printf("starting loop iteration %d refcnt %lu pypylink %lu \n", i, + sub->ob_refcnt, sub->ob_pypy_link); + #else + printf("starting loop iteration %d refcnt %lu\n", i, sub->ob_refcnt); + #endif + */ + attrib = PyObject_GetAttr(sub, attribname); + if (!attrib || (attrib == Py_None)) { + PyErr_SetString(PyExc_ValueError, + "could not find attrib on object"); + Py_XDECREF(attrib); + Py_DECREF(tup); + Py_DECREF(sub); + return NULL; + } + Py_XDECREF(attrib); + res = PyObject_Call(collect, tup, NULL); + if (res == NULL) { + Py_DECREF(tup); + Py_DECREF(sub); + return NULL; + } + Py_DECREF(res); + } + Py_DECREF(tup); + Py_DECREF(sub); + Py_RETURN_NONE; +} + /*********************** Install Module **************************/ static PyMethodDef a_methods[] = { @@ -2502,6 +2583,7 @@ {"create_and_release_buffer", (PyCFunction)create_and_release_buffer, METH_O, NULL}, {"same_dealloc", (PyCFunction)same_dealloc, METH_VARARGS, NULL}, {"getitem", (PyCFunction)getitem, METH_VARARGS, NULL}, + {"subclass_with_attribute", (PyCFunction)subclass_with_attribute, METH_VARARGS, NULL}, {NULL, NULL, 0, NULL} /* Sentinel */ }; diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py --- a/pypy/module/cpyext/test/test_arraymodule.py +++ b/pypy/module/cpyext/test/test_arraymodule.py @@ -186,3 +186,15 @@ # array_subscr does) raises(IndexError, module.getitem, a, -5) + def test_subclass_with_attribute(self): + module = self.import_module(name='array') + class Sub(module.array): + def addattrib(self): + print('called addattrib') + self.attrib = True + import gc + module.subclass_with_attribute(Sub, "addattrib", "attrib", gc.collect) + if self.runappdirect: + assert Sub.__module__ == 'pypy.module.cpyext.test.test_arraymodule' + assert str(Sub) == "" + diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -5,7 +5,7 @@ from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.api import generic_cpy_call from pypy.module.cpyext.pyobject import make_ref, from_ref, decref, as_pyobj -from pypy.module.cpyext.typeobject import PyTypeObjectPtr +from pypy.module.cpyext.typeobject import PyTypeObjectPtr, W_PyCTypeObject class AppTestTypeObject(AppTestCpythonExtensionBase): @@ -412,33 +412,42 @@ def test_type_dict(self): foo = self.import_module("foo") module = self.import_extension('test', [ - ("hack_tp_dict", "METH_O", + ("hack_tp_dict", "METH_VARARGS", ''' - PyTypeObject *type = args->ob_type; + PyTypeObject *type, *obj; PyObject *a1 = PyLong_FromLong(1); PyObject *a2 = PyLong_FromLong(2); PyObject *value; + PyObject * key; + if (!PyArg_ParseTuple(args, "OO", &obj, &key)) + return NULL; + type = obj->ob_type; - if (PyDict_SetItemString(type->tp_dict, "a", + if (PyDict_SetItem(type->tp_dict, key, a1) < 0) return NULL; Py_DECREF(a1); PyType_Modified(type); - value = PyObject_GetAttrString((PyObject *)type, "a"); + value = PyObject_GetAttr((PyObject *)type, key); Py_DECREF(value); - if (PyDict_SetItemString(type->tp_dict, "a", + if (PyDict_SetItem(type->tp_dict, key, a2) < 0) return NULL; Py_DECREF(a2); PyType_Modified(type); - value = PyObject_GetAttrString((PyObject *)type, "a"); + value = PyObject_GetAttr((PyObject *)type, key); return value; ''' ) ]) obj = foo.new() - assert module.hack_tp_dict(obj) == 2 + assert module.hack_tp_dict(obj, "a") == 2 + class Sub(foo.fooType): + pass + obj = Sub() + assert module.hack_tp_dict(obj, "b") == 2 + def test_tp_descr_get(self): module = self.import_extension('foo', [ @@ -560,6 +569,23 @@ assert w_obj is None assert api.PyErr_Occurred() is None + def test_subclass_not_PyCTypeObject(self, space, api): + pyobj = make_ref(space, api.PyLong_Type) + py_type = rffi.cast(PyTypeObjectPtr, pyobj) + w_pyclass = W_PyCTypeObject(space, py_type) + w_class = space.appexec([w_pyclass], """(base): + class Sub(base): + def addattrib(self, value): + self.attrib = value + return Sub + """) + assert w_pyclass in w_class.mro_w + assert isinstance(w_pyclass, W_PyCTypeObject) + assert not isinstance(w_class, W_PyCTypeObject) + assert w_pyclass.is_cpytype() + # XXX document the current status, not clear if this is desirable + assert w_class.is_cpytype() + class AppTestSlots(AppTestCpythonExtensionBase): def setup_class(cls): diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -1,6 +1,6 @@ import os -from rpython.rlib import jit +from rpython.rlib import jit, rawrefcount from rpython.rlib.objectmodel import specialize, we_are_translated from rpython.rtyper.lltypesystem import rffi, lltype @@ -517,6 +517,10 @@ self.w_doc = space.newtext( rffi.charp2str(cts.cast('char*', pto.c_tp_doc))) + def _cpyext_attach_pyobj(self, space, py_obj): + self._cpy_ref = py_obj + rawrefcount.create_link_pyobj(self, py_obj) + @bootstrap_function def init_typeobject(space): make_typedescr(space.w_type.layout.typedef, @@ -777,7 +781,6 @@ try: w_obj = _type_realize(space, py_obj) finally: - name = rffi.charp2str(cts.cast('char*', pto.c_tp_name)) pto.c_tp_flags &= ~Py_TPFLAGS_READYING pto.c_tp_flags |= Py_TPFLAGS_READY return w_obj @@ -884,7 +887,6 @@ base = pto.c_tp_base base_pyo = rffi.cast(PyObject, pto.c_tp_base) if base and not base.c_tp_flags & Py_TPFLAGS_READY: - name = rffi.charp2str(cts.cast('char*', base.c_tp_name)) type_realize(space, base_pyo) if base and not pto.c_ob_type: # will be filled later pto.c_ob_type = base.c_ob_type From pypy.commits at gmail.com Sun Apr 8 03:41:00 2018 From: pypy.commits at gmail.com (arigo) Date: Sun, 08 Apr 2018 00:41:00 -0700 (PDT) Subject: [pypy-commit] pypy default: Test and fix for issue #2788. Also implements "del obj.getsetprop". Message-ID: <5ac9c78c.07711c0a.97f27.62b4@mx.google.com> Author: Armin Rigo Branch: Changeset: r94264:1843dd2013b4 Date: 2018-04-08 09:34 +0200 http://bitbucket.org/pypy/pypy/changeset/1843dd2013b4/ Log: Test and fix for issue #2788. Also implements "del obj.getsetprop". diff --git a/pypy/interpreter/test/test_typedef.py b/pypy/interpreter/test/test_typedef.py --- a/pypy/interpreter/test/test_typedef.py +++ b/pypy/interpreter/test/test_typedef.py @@ -423,3 +423,10 @@ def test_get_with_none_arg(self): raises(TypeError, type.__dict__['__mro__'].__get__, None) raises(TypeError, type.__dict__['__mro__'].__get__, None, None) + + def test_builtin_readonly_property(self): + import sys + x = lambda: 5 + e = raises(TypeError, 'x.func_globals = {}') + if '__pypy__' in sys.builtin_module_names: + assert str(e.value) == "readonly attribute 'func_globals'" diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py --- a/pypy/interpreter/typedef.py +++ b/pypy/interpreter/typedef.py @@ -309,12 +309,18 @@ self.reqcls, Arguments(space, [w_obj, space.newtext(self.name)])) + def readonly_attribute(self, space): # overwritten in cpyext + if self.name == '': + raise oefmt(space.w_TypeError, "readonly attribute") + else: + raise oefmt(space.w_TypeError, "readonly attribute '%s'", self.name) + def descr_property_set(self, space, w_obj, w_value): """property.__set__(obj, value) Change the value of the property of the given obj.""" fset = self.fset if fset is None: - raise oefmt(space.w_TypeError, "readonly attribute") + raise self.readonly_attribute(space) try: fset(self, space, w_obj, w_value) except DescrMismatch: diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -1626,6 +1626,64 @@ pass C(42) # assert is not aborting + def test_getset(self): + module = self.import_extension('foo', [ + ("get_instance", "METH_NOARGS", + ''' + return PyObject_New(PyObject, &Foo_Type); + ''' + ), ("get_number", "METH_NOARGS", + ''' + return PyInt_FromLong(my_global_number); + ''' + )], prologue=''' + #if PY_MAJOR_VERSION > 2 + #define PyInt_FromLong PyLong_FromLong + #endif + static long my_global_number; + static PyTypeObject Foo_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "foo.foo", + }; + static PyObject *bar_get(PyObject *foo, void *closure) + { + return PyInt_FromLong(1000 + (long)closure); + } + static PyObject *baz_get(PyObject *foo, void *closure) + { + return PyInt_FromLong(2000 + (long)closure); + } + static int baz_set(PyObject *foo, PyObject *x, void *closure) + { + if (x != NULL) + my_global_number = 3000 + (long)closure + PyInt_AsLong(x); + else + my_global_number = 4000 + (long)closure; + return 0; + } + static PyGetSetDef foo_getset[] = { + { "bar", bar_get, NULL, "mybardoc", (void *)42 }, + { "baz", baz_get, baz_set, "mybazdoc", (void *)43 }, + { NULL } + }; + ''', more_init = ''' + Foo_Type.tp_getset = foo_getset; + Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT; + if (PyType_Ready(&Foo_Type) < 0) INITERROR; + ''') + foo = module.get_instance() + assert foo.bar == 1042 + assert foo.bar == 1042 + assert foo.baz == 2043 + foo.baz = 50000 + assert module.get_number() == 53043 + e = raises(AttributeError, "foo.bar = 0") + assert str(e.value).startswith("attribute 'bar' of '") + assert str(e.value).endswith("foo' objects is not writable") + del foo.baz + assert module.get_number() == 4043 + raises(AttributeError, "del foo.bar") + class AppTestHashable(AppTestCpythonExtensionBase): def test_unhashable(self): diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -54,19 +54,26 @@ class W_GetSetPropertyEx(GetSetProperty): def __init__(self, getset, w_type): self.getset = getset - self.name = rffi.charp2str(getset.c_name) self.w_type = w_type - doc = set = get = None + doc = fset = fget = fdel = None if doc: # XXX dead code? doc = rffi.charp2str(getset.c_doc) if getset.c_get: - get = GettersAndSetters.getter.im_func + fget = GettersAndSetters.getter.im_func if getset.c_set: - set = GettersAndSetters.setter.im_func - GetSetProperty.__init__(self, get, set, None, doc, + fset = GettersAndSetters.setter.im_func + fdel = GettersAndSetters.deleter.im_func + GetSetProperty.__init__(self, fget, fset, fdel, doc, cls=None, use_closure=True, tag="cpyext_1") + self.name = rffi.charp2str(getset.c_name) + + def readonly_attribute(self, space): # overwritten + raise oefmt(space.w_AttributeError, + "attribute '%s' of '%N' objects is not writable", + self.name, self.w_type) + def PyDescr_NewGetSet(space, getset, w_type): return W_GetSetPropertyEx(getset, w_type) @@ -454,6 +461,16 @@ state = space.fromcache(State) state.check_and_raise_exception() + def deleter(self, space, w_self): + assert isinstance(self, W_GetSetPropertyEx) + check_descr(space, w_self, self.w_type) + res = generic_cpy_call( + space, self.getset.c_set, w_self, None, + self.getset.c_closure) + if rffi.cast(lltype.Signed, res) < 0: + state = space.fromcache(State) + state.check_and_raise_exception() + def member_getter(self, space, w_self): assert isinstance(self, W_MemberDescr) check_descr(space, w_self, self.w_type) From pypy.commits at gmail.com Sun Apr 8 04:44:48 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 08 Apr 2018 01:44:48 -0700 (PDT) Subject: [pypy-commit] pypy issue2752: copy test from issue2752, fails with -A, passes with --jit off Message-ID: <5ac9d680.510c1c0a.1074b.1594@mx.google.com> Author: Matti Picus Branch: issue2752 Changeset: r94265:600bd2030f2f Date: 2018-04-08 11:42 +0300 http://bitbucket.org/pypy/pypy/changeset/600bd2030f2f/ Log: copy test from issue2752, fails with -A, passes with --jit off diff --git a/pypy/module/cpyext/test/test_bufferobject.py b/pypy/module/cpyext/test/test_bufferobject.py --- a/pypy/module/cpyext/test/test_bufferobject.py +++ b/pypy/module/cpyext/test/test_bufferobject.py @@ -2,8 +2,19 @@ from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.api import PyObject +from pypy.conftest import option class AppTestBufferObject(AppTestCpythonExtensionBase): + + def setup_class(cls): + from rpython.tool.udir import udir + AppTestCpythonExtensionBase.setup_class.im_func(cls) + if option.runappdirect: + cls.w_udir = str(udir) + else: + cls.w_udir = cls.space.wrap(str(udir)) + + def test_FromMemory(self): module = self.import_extension('foo', [ ("get_FromMemory", "METH_NOARGS", @@ -62,3 +73,72 @@ a = array.array('c', 'text') b = buffer(a) assert module.roundtrip(b) == 'text' + + + def test_issue2752(self): + if not self.runappdirect: + skip('too slow, run with -A') + module = self.import_extension('foo', [ + ("test_mod", 'METH_VARARGS', + """ + PyObject *obj, *collect, *tup; + Py_buffer bp; + char expected_i = '0'; + if (!PyArg_ParseTuple(args, "OO", &obj, &collect)) + return NULL; + + assert(obj); + + if (PyObject_GetBuffer(obj, &bp, PyBUF_SIMPLE) == -1) + return NULL; + + tup = PyTuple_New(0); /* for collect() */ + for (size_t i = 0; i < bp.len; ++i) + { + if (((unsigned char*)bp.buf)[i] == expected_i) + { + if (++expected_i >= '8') + expected_i = '0'; + } + else + { + PyErr_Format(PyExc_ValueError, + "mismatch: 0x%x [%x %x %x %x...] instead of 0x%x on pos=%d (got len=%d)", + ((unsigned char*)bp.buf)[i], + ((unsigned char*)bp.buf)[i+1], + ((unsigned char*)bp.buf)[i+2], + ((unsigned char*)bp.buf)[i+3], + ((unsigned char*)bp.buf)[i+4], + expected_i, i, bp.len); + PyBuffer_Release(&bp); + Py_DECREF(tup); + return NULL; + } + } + + PyBuffer_Release(&bp); + Py_DECREF(tup); + Py_RETURN_NONE; + """), + ]) + import io, gc + bufsize = 4096 + with io.open(self.udir + '/test.txt', 'wb') as f: + data = b'01234567' + for x in range(18): + data += data + f.write(data) + def sub(i): + fpos = 0 + with io.open(self.udir + '/test.txt', 'rb') as f: + for j, block in enumerate(iter(lambda: f.read(bufsize), b'')): + try: + module.test_mod(block, gc.collect) + except ValueError as e: + print("%s at top it=%d, read it=%d, fpos=%d" + % (e, i, j, fpos)) + assert False + fpos = f.tell() + + for x in map(sub, range(100)): + pass From pypy.commits at gmail.com Sun Apr 8 04:54:44 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 08 Apr 2018 01:54:44 -0700 (PDT) Subject: [pypy-commit] pypy issue2752: simplify test a bit, fails after exactly 1045 iterations Message-ID: <5ac9d8d4.832b1c0a.c8ea2.bb17@mx.google.com> Author: Matti Picus Branch: issue2752 Changeset: r94266:b6f2617d0c19 Date: 2018-04-08 11:54 +0300 http://bitbucket.org/pypy/pypy/changeset/b6f2617d0c19/ Log: simplify test a bit, fails after exactly 1045 iterations diff --git a/pypy/module/cpyext/test/test_bufferobject.py b/pypy/module/cpyext/test/test_bufferobject.py --- a/pypy/module/cpyext/test/test_bufferobject.py +++ b/pypy/module/cpyext/test/test_bufferobject.py @@ -2,19 +2,9 @@ from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.api import PyObject -from pypy.conftest import option class AppTestBufferObject(AppTestCpythonExtensionBase): - def setup_class(cls): - from rpython.tool.udir import udir - AppTestCpythonExtensionBase.setup_class.im_func(cls) - if option.runappdirect: - cls.w_udir = str(udir) - else: - cls.w_udir = cls.space.wrap(str(udir)) - - def test_FromMemory(self): module = self.import_extension('foo', [ ("get_FromMemory", "METH_NOARGS", @@ -123,22 +113,18 @@ ]) import io, gc bufsize = 4096 - with io.open(self.udir + '/test.txt', 'wb') as f: + def getdata(bufsize): data = b'01234567' for x in range(18): data += data - f.write(data) - def sub(i): - fpos = 0 - with io.open(self.udir + '/test.txt', 'rb') as f: - for j, block in enumerate(iter(lambda: f.read(bufsize), b'')): - try: - module.test_mod(block, gc.collect) - except ValueError as e: - print("%s at top it=%d, read it=%d, fpos=%d" - % (e, i, j, fpos)) - assert False - fpos = f.tell() - - for x in map(sub, range(100)): - pass + if len(data) >= bufsize: + break + return data + for j, block in enumerate(iter(lambda: getdata(bufsize), b'')): + try: + module.test_mod(block, gc.collect) + except ValueError as e: + print("%s at it=%d" % (e, j)) + assert False + if j > 2000: + break From pypy.commits at gmail.com Sun Apr 8 05:04:48 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 08 Apr 2018 02:04:48 -0700 (PDT) Subject: [pypy-commit] pypy issue2752: simplify more Message-ID: <5ac9db30.85c9df0a.19ea5.80e2@mx.google.com> Author: Matti Picus Branch: issue2752 Changeset: r94267:0c1b091fb92b Date: 2018-04-08 12:03 +0300 http://bitbucket.org/pypy/pypy/changeset/0c1b091fb92b/ Log: simplify more diff --git a/pypy/module/cpyext/test/test_bufferobject.py b/pypy/module/cpyext/test/test_bufferobject.py --- a/pypy/module/cpyext/test/test_bufferobject.py +++ b/pypy/module/cpyext/test/test_bufferobject.py @@ -73,7 +73,6 @@ """ PyObject *obj, *collect, *tup; Py_buffer bp; - char expected_i = '0'; if (!PyArg_ParseTuple(args, "OO", &obj, &collect)) return NULL; @@ -83,27 +82,18 @@ return NULL; tup = PyTuple_New(0); /* for collect() */ - for (size_t i = 0; i < bp.len; ++i) - { - if (((unsigned char*)bp.buf)[i] == expected_i) - { - if (++expected_i >= '8') - expected_i = '0'; - } - else - { - PyErr_Format(PyExc_ValueError, - "mismatch: 0x%x [%x %x %x %x...] instead of 0x%x on pos=%d (got len=%d)", - ((unsigned char*)bp.buf)[i], - ((unsigned char*)bp.buf)[i+1], - ((unsigned char*)bp.buf)[i+2], - ((unsigned char*)bp.buf)[i+3], - ((unsigned char*)bp.buf)[i+4], - expected_i, i, bp.len); - PyBuffer_Release(&bp); - Py_DECREF(tup); - return NULL; - } + if (((unsigned char*)bp.buf)[0] != '0') { + PyErr_Format(PyExc_ValueError, + "mismatch: 0x%x [%x %x %x %x...] instead of '0' (got len=%d)", + ((unsigned char*)bp.buf)[0], + ((unsigned char*)bp.buf)[1], + ((unsigned char*)bp.buf)[2], + ((unsigned char*)bp.buf)[3], + ((unsigned char*)bp.buf)[4], + bp.len); + PyBuffer_Release(&bp); + Py_DECREF(tup); + return NULL; } PyBuffer_Release(&bp); @@ -120,11 +110,11 @@ if len(data) >= bufsize: break return data - for j, block in enumerate(iter(lambda: getdata(bufsize), b'')): + for j in range(2000): + block = getdata(bufsize) + assert block[:8] == '01234567' try: module.test_mod(block, gc.collect) except ValueError as e: print("%s at it=%d" % (e, j)) assert False - if j > 2000: - break From pypy.commits at gmail.com Sun Apr 8 05:18:55 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 08 Apr 2018 02:18:55 -0700 (PDT) Subject: [pypy-commit] pypy issue2752: more simplification, crashes untranslated (at teardown?) Message-ID: <5ac9de7f.02b6df0a.ebb2e.f177@mx.google.com> Author: Matti Picus Branch: issue2752 Changeset: r94268:eb2ca00ab61a Date: 2018-04-08 12:17 +0300 http://bitbucket.org/pypy/pypy/changeset/eb2ca00ab61a/ Log: more simplification, crashes untranslated (at teardown?) diff --git a/pypy/module/cpyext/test/test_bufferobject.py b/pypy/module/cpyext/test/test_bufferobject.py --- a/pypy/module/cpyext/test/test_bufferobject.py +++ b/pypy/module/cpyext/test/test_bufferobject.py @@ -66,22 +66,20 @@ def test_issue2752(self): - if not self.runappdirect: - skip('too slow, run with -A') + iterations = 10 + if self.runappdirect: + iterations = 2000 module = self.import_extension('foo', [ ("test_mod", 'METH_VARARGS', """ - PyObject *obj, *collect, *tup; + PyObject *obj; Py_buffer bp; - if (!PyArg_ParseTuple(args, "OO", &obj, &collect)) + if (!PyArg_ParseTuple(args, "O", &obj)) return NULL; - assert(obj); - if (PyObject_GetBuffer(obj, &bp, PyBUF_SIMPLE) == -1) return NULL; - tup = PyTuple_New(0); /* for collect() */ if (((unsigned char*)bp.buf)[0] != '0') { PyErr_Format(PyExc_ValueError, "mismatch: 0x%x [%x %x %x %x...] instead of '0' (got len=%d)", @@ -92,16 +90,13 @@ ((unsigned char*)bp.buf)[4], bp.len); PyBuffer_Release(&bp); - Py_DECREF(tup); return NULL; } PyBuffer_Release(&bp); - Py_DECREF(tup); Py_RETURN_NONE; """), ]) - import io, gc bufsize = 4096 def getdata(bufsize): data = b'01234567' @@ -110,11 +105,11 @@ if len(data) >= bufsize: break return data - for j in range(2000): + for j in range(iterations): block = getdata(bufsize) assert block[:8] == '01234567' try: - module.test_mod(block, gc.collect) + module.test_mod(block) except ValueError as e: print("%s at it=%d" % (e, j)) assert False From pypy.commits at gmail.com Sun Apr 8 06:18:13 2018 From: pypy.commits at gmail.com (cfbolz) Date: Sun, 08 Apr 2018 03:18:13 -0700 (PDT) Subject: [pypy-commit] pypy pyparser-improvements-2: make it possible to pass diferent files to the parsing target Message-ID: <5ac9ec65.2281df0a.1c317.49ac@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: pyparser-improvements-2 Changeset: r94269:0066fc716a62 Date: 2018-04-02 07:53 +0200 http://bitbucket.org/pypy/pypy/changeset/0066fc716a62/ Log: make it possible to pass diferent files to the parsing target diff --git a/pypy/interpreter/pyparser/test/targetparse.py b/pypy/interpreter/pyparser/test/targetparse.py --- a/pypy/interpreter/pyparser/test/targetparse.py +++ b/pypy/interpreter/pyparser/test/targetparse.py @@ -8,25 +8,36 @@ -with file("../../../rpython/rlib/unicodedata/unicodedb_5_2_0.py") as f: - s = f.read() - class FakeSpace(object): pass fakespace = FakeSpace() -def bench(title): +def bench(fn, s): a = time.clock() info = pyparse.CompileInfo("", "exec") parser = pyparse.PythonParser(fakespace) tree = parser._parse(s, info) b = time.clock() - print title, (b-a) + print fn, (b-a) def entry_point(argv): - bench("foo") + if len(argv) == 2: + fn = argv[1] + else: + fn = "../../../../rpython/rlib/unicodedata/unicodedb_5_2_0.py" + fd = os.open(fn, os.O_RDONLY, 0777) + res = [] + while True: + s = os.read(fd, 4096) + if not s: + break + res.append(s) + os.close(fd) + s = "".join(res) + print len(s) + bench(fn, s) return 0 From pypy.commits at gmail.com Sun Apr 8 06:18:15 2018 From: pypy.commits at gmail.com (cfbolz) Date: Sun, 08 Apr 2018 03:18:15 -0700 (PDT) Subject: [pypy-commit] pypy pyparser-improvements-2: probably doesn't matter because code isn't indentend super deep, but don't copy Message-ID: <5ac9ec67.ea86df0a.c5b43.26f3@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: pyparser-improvements-2 Changeset: r94270:7fb340b040e0 Date: 2018-04-07 13:30 +0200 http://bitbucket.org/pypy/pypy/changeset/7fb340b040e0/ Log: probably doesn't matter because code isn't indentend super deep, but don't copy the indentation list on dedent. diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py --- a/pypy/interpreter/pyparser/pytokenizer.py +++ b/pypy/interpreter/pyparser/pytokenizer.py @@ -143,7 +143,7 @@ token_list.append((tokens.INDENT, line[:pos], lnum, 0, line)) last_comment = '' while column < indents[-1]: - indents = indents[:-1] + indents.pop() token_list.append((tokens.DEDENT, '', lnum, pos, line)) last_comment = '' if column != indents[-1]: From pypy.commits at gmail.com Sun Apr 8 06:18:21 2018 From: pypy.commits at gmail.com (cfbolz) Date: Sun, 08 Apr 2018 03:18:21 -0700 (PDT) Subject: [pypy-commit] pypy pyparser-improvements-2: track matching parenthesis for better errors Message-ID: <5ac9ec6d.c2091c0a.4711f.906f@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: pyparser-improvements-2 Changeset: r94273:7eaa7ec2554a Date: 2018-04-07 14:18 +0200 http://bitbucket.org/pypy/pypy/changeset/7eaa7ec2554a/ Log: track matching parenthesis for better errors diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py --- a/pypy/interpreter/pyparser/pytokenizer.py +++ b/pypy/interpreter/pyparser/pytokenizer.py @@ -73,14 +73,14 @@ logical line; continuation lines are included. """ token_list = [] - lnum = parenlev = continued = 0 + lnum = continued = 0 namechars = NAMECHARS numchars = NUMCHARS contstr, needcont = '', 0 contline = None indents = [0] last_comment = '' - parenlevstart = (0, 0, "") + parenstack = [] # make the annotator happy endDFA = DUMMY_DFA @@ -123,7 +123,7 @@ contline = contline + line continue - elif parenlev == 0 and not continued: # new statement + elif not parenstack and not continued: # new statement if not line: break column = 0 while pos < max: # measure leading whitespace @@ -152,10 +152,10 @@ else: # continued statement if not line: - if parenlev > 0: - lnum1, start1, line1 = parenlevstart + if parenstack: + _, lnum1, start1, line1 = parenstack[0] raise TokenError("parenthesis is never closed", line1, - lnum1, start1 + 1, token_list, lnum) + lnum1, start1, token_list, lnum) raise TokenError("EOF in multi-line statement", line, lnum, 0, token_list) continued = 0 @@ -180,7 +180,7 @@ token_list.append((tokens.NUMBER, token, lnum, start, line)) last_comment = '' elif initial in '\r\n': - if parenlev <= 0: + if not parenstack: tok = (tokens.NEWLINE, last_comment, lnum, start, line) token_list.append(tok) last_comment = '' @@ -222,14 +222,20 @@ continued = 1 else: if initial in '([{': - if parenlev == 0: - parenlevstart = (lnum, start, line) - parenlev = parenlev + 1 + parenstack.append((initial, lnum, start, line)) elif initial in ')]}': - parenlev = parenlev - 1 - if parenlev < 0: + if not parenstack: raise TokenError("unmatched '%s'" % initial, line, - lnum, start + 1, token_list) + lnum, start, token_list) + opening, lnum1, start1, line1 = parenstack.pop() + if not ((opening == "(" and initial == ")") or + (opening == "[" and initial == "]") or + (opening == "{" and initial == "}")): + raise TokenError( + "parenthesis '%s' and '%s' don't match" % ( + opening, initial), + line1, lnum1, start1, token_list, + lastlineno=lnum) if token in python_opmap: punct = python_opmap[token] else: diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py --- a/pypy/interpreter/pyparser/test/test_pyparse.py +++ b/pypy/interpreter/pyparser/test/test_pyparse.py @@ -92,12 +92,12 @@ exc = py.test.raises(SyntaxError, parse, "x = (\n\n(),\n(),").value assert exc.msg == "parenthesis is never closed" assert exc.lineno == 1 - assert exc.offset == 5 + assert exc.offset == 4 assert exc.lastlineno == 5 exc = py.test.raises(SyntaxError, parse, "abc)").value assert exc.msg == "unmatched ')'" assert exc.lineno == 1 - assert exc.offset == 4 + assert exc.offset == 3 def test_is(self): self.parse("x is y") diff --git a/pypy/interpreter/pyparser/test/test_pytokenizer.py b/pypy/interpreter/pyparser/test/test_pytokenizer.py --- a/pypy/interpreter/pyparser/test/test_pytokenizer.py +++ b/pypy/interpreter/pyparser/test/test_pytokenizer.py @@ -6,9 +6,11 @@ def tokenize(s): return pytokenizer.generate_tokens(s.splitlines(True) + ["\n"], 0) -def check_token_error(s, msg): +def check_token_error(s, msg, pos=-1): error = pytest.raises(TokenError, tokenize, s) assert error.value.msg == msg + if pos != -1: + assert error.value.offset == pos class TestTokenizer(object): @@ -28,10 +30,22 @@ def test_error_parenthesis(self): for paren in "([{": check_token_error(paren + "1 + 2", - "parenthesis is never closed") + "parenthesis is never closed", + 0) for paren in ")]}": check_token_error("1 + 2" + paren, - "unmatched '%s'" % (paren, )) + "unmatched '%s'" % (paren, ), + 5) + for i, opening in enumerate("([{"): + for j, closing in enumerate(")]}"): + if i == j: + continue + error = pytest.raises(TokenError, tokenize, opening + "1\n" + closing) + assert error.value.msg == \ + "parenthesis '%s' and '%s' don't match" % (opening, closing) + assert error.value.offset == 0 + assert error.value.lineno == 1 + assert error.value.lastlineno == 2 From pypy.commits at gmail.com Sun Apr 8 06:18:22 2018 From: pypy.commits at gmail.com (cfbolz) Date: Sun, 08 Apr 2018 03:18:22 -0700 (PDT) Subject: [pypy-commit] pypy pyparser-improvements-2: fix position of error Message-ID: <5ac9ec6e.464a1c0a.d3346.40c3@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: pyparser-improvements-2 Changeset: r94274:0682e48513a0 Date: 2018-04-07 14:19 +0200 http://bitbucket.org/pypy/pypy/changeset/0682e48513a0/ Log: fix position of error diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py --- a/pypy/interpreter/pyparser/pytokenizer.py +++ b/pypy/interpreter/pyparser/pytokenizer.py @@ -171,7 +171,7 @@ if start == end: raise TokenError("Unknown character", line, - lnum, start + 1, token_list) + lnum, start, token_list) pos = end token, initial = line[start:end], line[start] diff --git a/pypy/interpreter/pyparser/test/test_pytokenizer.py b/pypy/interpreter/pyparser/test/test_pytokenizer.py --- a/pypy/interpreter/pyparser/test/test_pytokenizer.py +++ b/pypy/interpreter/pyparser/test/test_pytokenizer.py @@ -49,3 +49,6 @@ assert error.value.lineno == 1 assert error.value.lastlineno == 2 + + def test_unknown_char(self): + check_token_error("?", "Unknown character", 0) From pypy.commits at gmail.com Sun Apr 8 06:18:17 2018 From: pypy.commits at gmail.com (cfbolz) Date: Sun, 08 Apr 2018 03:18:17 -0700 (PDT) Subject: [pypy-commit] pypy pyparser-improvements-2: add at least a very simple test for the tokenizer Message-ID: <5ac9ec69.e7361c0a.cbddb.27ed@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: pyparser-improvements-2 Changeset: r94271:61275f7cf5ea Date: 2018-04-07 13:44 +0200 http://bitbucket.org/pypy/pypy/changeset/61275f7cf5ea/ Log: add at least a very simple test for the tokenizer diff --git a/pypy/interpreter/pyparser/test/test_pytokenizer.py b/pypy/interpreter/pyparser/test/test_pytokenizer.py new file mode 100644 --- /dev/null +++ b/pypy/interpreter/pyparser/test/test_pytokenizer.py @@ -0,0 +1,19 @@ +from pypy.interpreter.pyparser import pytokenizer +from pypy.interpreter.pyparser.pygram import tokens + +def tokenize(s): + return pytokenizer.generate_tokens(s.splitlines(True) + ["\n"], 0) + +class TestTokenizer(object): + + def test_simple(self): + line = "a+1" + tks = tokenize(line) + assert tks == [ + (tokens.NAME, 'a', 1, 0, line), + (tokens.PLUS, '+', 1, 1, line), + (tokens.NUMBER, '1', 1, 2, line), + (tokens.NEWLINE, '', 2, 0, '\n'), + (tokens.NEWLINE, '', 2, 0, '\n'), + (tokens.ENDMARKER, '', 2, 0, ''), + ] From pypy.commits at gmail.com Sun Apr 8 06:18:24 2018 From: pypy.commits at gmail.com (cfbolz) Date: Sun, 08 Apr 2018 03:18:24 -0700 (PDT) Subject: [pypy-commit] pypy pyparser-improvements-2: fix more offsets Message-ID: <5ac9ec70.902e1c0a.d94b2.d976@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: pyparser-improvements-2 Changeset: r94275:52f6bfe8f3cd Date: 2018-04-07 14:25 +0200 http://bitbucket.org/pypy/pypy/changeset/52f6bfe8f3cd/ Log: fix more offsets diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py --- a/pypy/interpreter/pyparser/pytokenizer.py +++ b/pypy/interpreter/pyparser/pytokenizer.py @@ -98,7 +98,7 @@ if not line: raise TokenError( "EOF while scanning triple-quoted string literal", - strstart[2], strstart[0], strstart[1]+1, + strstart[2], strstart[0], strstart[1], token_list, lnum-1) endmatch = endDFA.recognize(line) if endmatch >= 0: @@ -248,7 +248,7 @@ start = pos if start Author: Carl Friedrich Bolz-Tereick Branch: pyparser-improvements-2 Changeset: r94276:692d3ce9de27 Date: 2018-04-07 14:38 +0200 http://bitbucket.org/pypy/pypy/changeset/692d3ce9de27/ Log: tweak error message and position diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py --- a/pypy/interpreter/pyparser/pytokenizer.py +++ b/pypy/interpreter/pyparser/pytokenizer.py @@ -231,11 +231,13 @@ if not ((opening == "(" and initial == ")") or (opening == "[" and initial == "]") or (opening == "{" and initial == "}")): + msg = "closing parenthesis '%s' does not match opening parenthesis '%s'" % ( + initial, opening) + + if lnum1 != lnum: + msg += " on line " + str(lnum1) raise TokenError( - "parenthesis '%s' and '%s' don't match" % ( - opening, initial), - line1, lnum1, start1, token_list, - lastlineno=lnum) + msg, line, lnum, start, token_list) if token in python_opmap: punct = python_opmap[token] else: diff --git a/pypy/interpreter/pyparser/test/test_pytokenizer.py b/pypy/interpreter/pyparser/test/test_pytokenizer.py --- a/pypy/interpreter/pyparser/test/test_pytokenizer.py +++ b/pypy/interpreter/pyparser/test/test_pytokenizer.py @@ -45,12 +45,12 @@ for j, closing in enumerate(")]}"): if i == j: continue - error = pytest.raises(TokenError, tokenize, opening + "1\n" + closing) - assert error.value.msg == \ - "parenthesis '%s' and '%s' don't match" % (opening, closing) - assert error.value.offset == 0 - assert error.value.lineno == 1 - assert error.value.lastlineno == 2 + check_token_error(opening + "1\n" + closing, + "closing parenthesis '%s' does not match opening parenthesis '%s' on line 1" % (closing, opening), + pos=0, line=2) + check_token_error(opening + "1" + closing, + "closing parenthesis '%s' does not match opening parenthesis '%s'" % (closing, opening), + pos=2, line=1) def test_unknown_char(self): From pypy.commits at gmail.com Sun Apr 8 06:18:28 2018 From: pypy.commits at gmail.com (cfbolz) Date: Sun, 08 Apr 2018 03:18:28 -0700 (PDT) Subject: [pypy-commit] pypy pyparser-improvements-2: expand these abbreviations Message-ID: <5ac9ec74.4a8e1c0a.36a0d.3fb0@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: pyparser-improvements-2 Changeset: r94277:2a295eea4b53 Date: 2018-04-07 14:39 +0200 http://bitbucket.org/pypy/pypy/changeset/2a295eea4b53/ Log: expand these abbreviations diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py --- a/pypy/interpreter/pyparser/pytokenizer.py +++ b/pypy/interpreter/pyparser/pytokenizer.py @@ -97,7 +97,7 @@ if contstr: if not line: raise TokenError( - "EOF while scanning triple-quoted string literal", + "end of file (EOF) while scanning triple-quoted string literal", strstart[2], strstart[0], strstart[1], token_list, lnum-1) endmatch = endDFA.recognize(line) @@ -156,7 +156,7 @@ _, lnum1, start1, line1 = parenstack[0] raise TokenError("parenthesis is never closed", line1, lnum1, start1, token_list, lnum) - raise TokenError("EOF in multi-line statement", line, + raise TokenError("end of file (EOF) in multi-line statement", line, lnum, 0, token_list) continued = 0 @@ -249,7 +249,7 @@ if start < 0: start = pos if start Author: Carl Friedrich Bolz-Tereick Branch: pyparser-improvements-2 Changeset: r94272:8b6d076bb5b7 Date: 2018-04-07 13:50 +0200 http://bitbucket.org/pypy/pypy/changeset/8b6d076bb5b7/ Log: a test for parenthesis error msgs diff --git a/pypy/interpreter/pyparser/test/test_pytokenizer.py b/pypy/interpreter/pyparser/test/test_pytokenizer.py --- a/pypy/interpreter/pyparser/test/test_pytokenizer.py +++ b/pypy/interpreter/pyparser/test/test_pytokenizer.py @@ -1,9 +1,16 @@ +import pytest from pypy.interpreter.pyparser import pytokenizer from pypy.interpreter.pyparser.pygram import tokens +from pypy.interpreter.pyparser.error import TokenError def tokenize(s): return pytokenizer.generate_tokens(s.splitlines(True) + ["\n"], 0) +def check_token_error(s, msg): + error = pytest.raises(TokenError, tokenize, s) + assert error.value.msg == msg + + class TestTokenizer(object): def test_simple(self): @@ -17,3 +24,14 @@ (tokens.NEWLINE, '', 2, 0, '\n'), (tokens.ENDMARKER, '', 2, 0, ''), ] + + def test_error_parenthesis(self): + for paren in "([{": + check_token_error(paren + "1 + 2", + "parenthesis is never closed") + + for paren in ")]}": + check_token_error("1 + 2" + paren, + "unmatched '%s'" % (paren, )) + + From pypy.commits at gmail.com Sun Apr 8 09:44:55 2018 From: pypy.commits at gmail.com (cfbolz) Date: Sun, 08 Apr 2018 06:44:55 -0700 (PDT) Subject: [pypy-commit] pypy pyparser-improvements-2: fix SyntaxError offsets Message-ID: <5aca1cd7.89571c0a.8f5b9.e75e@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: pyparser-improvements-2 Changeset: r94278:3405c95f9e61 Date: 2018-04-08 15:44 +0200 http://bitbucket.org/pypy/pypy/changeset/3405c95f9e61/ Log: fix SyntaxError offsets (so far, the errors that came from the parser had an off-by-one error, which was the source of me thinking that SyntaxError.offset is 0-based. In reality, the parser is wrong and it should be 1-based). diff --git a/pypy/interpreter/pyparser/error.py b/pypy/interpreter/pyparser/error.py --- a/pypy/interpreter/pyparser/error.py +++ b/pypy/interpreter/pyparser/error.py @@ -6,6 +6,7 @@ lastlineno=0): self.msg = msg self.lineno = lineno + # NB: offset is a 1-based index! self.offset = offset self.text = text self.filename = filename diff --git a/pypy/interpreter/pyparser/parser.py b/pypy/interpreter/pyparser/parser.py --- a/pypy/interpreter/pyparser/parser.py +++ b/pypy/interpreter/pyparser/parser.py @@ -199,6 +199,7 @@ self.token_type = token_type self.value = value self.lineno = lineno + # this is a 0-based index self.column = column self.line = line self.expected = expected diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py --- a/pypy/interpreter/pyparser/pyparse.py +++ b/pypy/interpreter/pyparser/pyparse.py @@ -188,7 +188,9 @@ if e.expected_str is not None: msg += " (expected '%s')" % e.expected_str - raise new_err(msg, e.lineno, e.column, e.line, + # parser.ParseError(...).column is 0-based, but the offsets in the + # exceptions in the error module are 1-based, hence the '+ 1' + raise new_err(msg, e.lineno, e.column + 1, e.line, compile_info.filename) else: tree = self.root diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py --- a/pypy/interpreter/pyparser/pytokenizer.py +++ b/pypy/interpreter/pyparser/pytokenizer.py @@ -98,7 +98,7 @@ if not line: raise TokenError( "end of file (EOF) while scanning triple-quoted string literal", - strstart[2], strstart[0], strstart[1], + strstart[2], strstart[0], strstart[1]+1, token_list, lnum-1) endmatch = endDFA.recognize(line) if endmatch >= 0: @@ -148,16 +148,16 @@ last_comment = '' if column != indents[-1]: err = "unindent does not match any outer indentation level" - raise TokenIndentationError(err, line, lnum, 0, token_list) + raise TokenIndentationError(err, line, lnum, column+1, token_list) else: # continued statement if not line: if parenstack: _, lnum1, start1, line1 = parenstack[0] raise TokenError("parenthesis is never closed", line1, - lnum1, start1, token_list, lnum) + lnum1, start1 + 1, token_list, lnum) raise TokenError("end of file (EOF) in multi-line statement", line, - lnum, 0, token_list) + lnum, 0, token_list) # XXX why is the offset 0 here? continued = 0 while pos < max: @@ -171,7 +171,7 @@ if start == end: raise TokenError("Unknown character", line, - lnum, start, token_list) + lnum, start + 1, token_list) pos = end token, initial = line[start:end], line[start] @@ -226,7 +226,7 @@ elif initial in ')]}': if not parenstack: raise TokenError("unmatched '%s'" % initial, line, - lnum, start, token_list) + lnum, start + 1, token_list) opening, lnum1, start1, line1 = parenstack.pop() if not ((opening == "(" and initial == ")") or (opening == "[" and initial == "]") or @@ -237,7 +237,7 @@ if lnum1 != lnum: msg += " on line " + str(lnum1) raise TokenError( - msg, line, lnum, start, token_list) + msg, line, lnum, start + 1, token_list) if token in python_opmap: punct = python_opmap[token] else: @@ -250,7 +250,7 @@ start = pos if start Author: Ronan Lamy Branch: py3.5 Changeset: r94279:ab173e2698f9 Date: 2018-04-08 17:26 +0100 http://bitbucket.org/pypy/pypy/changeset/ab173e2698f9/ Log: Refactor W_TextIOWrapper.read_w and .readline_w to ensure that the expensive calls to ._check_closed() get jitted (issue #2790). diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py --- a/pypy/module/_io/interp_textio.py +++ b/pypy/module/_io/interp_textio.py @@ -715,15 +715,20 @@ self._writeflush(space) if size < 0: - # Read everything - w_bytes = space.call_method(self.w_buffer, "read") - w_decoded = space.call_method(self.w_decoder, "decode", w_bytes, space.w_True) - check_decoded(space, w_decoded) - w_result = space.newunicode(self.decoded.get_chars(-1)) - w_final = space.add(w_result, w_decoded) - self.snapshot = None - return w_final + return self._read_all(space) + else: + return self._read(space, size) + def _read_all(self, space): + w_bytes = space.call_method(self.w_buffer, "read") + w_decoded = space.call_method(self.w_decoder, "decode", w_bytes, space.w_True) + check_decoded(space, w_decoded) + w_result = space.newunicode(self.decoded.get_chars(-1)) + w_final = space.add(w_result, w_decoded) + self.snapshot = None + return w_final + + def _read(self, space, size): remaining = size builder = UnicodeBuilder(size) @@ -737,6 +742,7 @@ return space.newunicode(builder.build()) + def _scan_line_ending(self, limit): if self.readuniversal: return self.decoded.find_newline_universal(limit) @@ -756,8 +762,11 @@ self._check_attached(space) self._check_closed(space) self._writeflush(space) + limit = convert_size(space, w_limit) + return space.newunicode(self._readline(space, limit)) - limit = convert_size(space, w_limit) + def _readline(self, space, limit): + # This is a separate function so that readline_w() can be jitted. remnant = None builder = UnicodeBuilder() while True: @@ -805,8 +814,7 @@ # We have consumed the buffer self.decoded.reset() - result = builder.build() - return space.newunicode(result) + return builder.build() # _____________________________________________________________ # write methods From pypy.commits at gmail.com Mon Apr 9 03:31:59 2018 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 09 Apr 2018 00:31:59 -0700 (PDT) Subject: [pypy-commit] pypy pyparser-improvements-2: fix fallout from position fixes Message-ID: <5acb16ef.32c3df0a.3e8ea.ba7a@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: pyparser-improvements-2 Changeset: r94280:fefc40951b6f Date: 2018-04-09 09:27 +0200 http://bitbucket.org/pypy/pypy/changeset/fefc40951b6f/ Log: fix fallout from position fixes diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py --- a/pypy/interpreter/test/test_compiler.py +++ b/pypy/interpreter/test/test_compiler.py @@ -77,7 +77,7 @@ """) assert self.space.unwrap(w_args) == ( 'unindent does not match any outer indentation level', - ('', 3, 0, ' y\n')) + ('', 3, 2, ' y\n')) def test_getcodeflags(self): code = self.compiler.compile('from __future__ import division\n', diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py --- a/pypy/interpreter/test/test_syntax.py +++ b/pypy/interpreter/test/test_syntax.py @@ -750,7 +750,7 @@ except SyntaxError as e: assert e.lineno == 4 assert e.text.endswith('a b c d e\n') - assert e.offset == e.text.index('b') + assert e.offset == e.text.index('b') + 1 # offset is 1-based else: raise Exception("no SyntaxError??") From pypy.commits at gmail.com Mon Apr 9 03:32:03 2018 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 09 Apr 2018 00:32:03 -0700 (PDT) Subject: [pypy-commit] pypy pyparser-improvements-2: document branch Message-ID: <5acb16f3.0999df0a.b07e3.f0b3@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: pyparser-improvements-2 Changeset: r94282:8988c01a0750 Date: 2018-04-09 09:31 +0200 http://bitbucket.org/pypy/pypy/changeset/8988c01a0750/ Log: document branch diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -6,3 +6,7 @@ .. startrev: 2e04adf1b89f +.. branch: pyparser-improvements-2 + +Improve line offsets that are reported by SyntaxError. Improve error messages +for a few situations, including mismatched parenthesis. From pypy.commits at gmail.com Mon Apr 9 03:32:01 2018 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 09 Apr 2018 00:32:01 -0700 (PDT) Subject: [pypy-commit] pypy pyparser-improvements-2: fix CPython tests that are affected (in one, we are like CPython now again) Message-ID: <5acb16f1.84b2df0a.beca6.dcc7@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: pyparser-improvements-2 Changeset: r94281:65640d1556ff Date: 2018-04-09 09:29 +0200 http://bitbucket.org/pypy/pypy/changeset/65640d1556ff/ Log: fix CPython tests that are affected (in one, we are like CPython now again) diff --git a/lib-python/2.7/test/test_eof.py b/lib-python/2.7/test/test_eof.py --- a/lib-python/2.7/test/test_eof.py +++ b/lib-python/2.7/test/test_eof.py @@ -5,7 +5,7 @@ class EOFTestCase(unittest.TestCase): def test_EOFC(self): - expect = "EOL while scanning string literal (, line 1)" + expect = "end of line (EOL) while scanning string literal (, line 1)" try: eval("""'this is a test\ """) @@ -15,7 +15,7 @@ raise test_support.TestFailed def test_EOFS(self): - expect = ("EOF while scanning triple-quoted string literal " + expect = ("end of file (EOF) while scanning triple-quoted string literal " "(, line 1)") try: eval("""'''this is a test""") diff --git a/lib-python/2.7/test/test_traceback.py b/lib-python/2.7/test/test_traceback.py --- a/lib-python/2.7/test/test_traceback.py +++ b/lib-python/2.7/test/test_traceback.py @@ -123,10 +123,7 @@ self.assertEqual(len(err), 4) self.assertEqual(err[1].strip(), "print(2)") self.assertIn("^", err[2]) - if check_impl_detail(): - self.assertEqual(err[1].find("p"), err[2].find("^")) - if check_impl_detail(pypy=True): - self.assertEqual(err[1].find("2)") + 1, err[2].find("^")) + self.assertEqual(err[1].find("p"), err[2].find("^")) def test_base_exception(self): # Test that exceptions derived from BaseException are formatted right From pypy.commits at gmail.com Mon Apr 9 03:37:31 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 09 Apr 2018 00:37:31 -0700 (PDT) Subject: [pypy-commit] pypy issue2752: after PyObject_GetBuffer fails, a second call also fails, PyString_AsString succeeds Message-ID: <5acb183b.ab87df0a.213da.9ea9@mx.google.com> Author: Matti Picus Branch: issue2752 Changeset: r94283:882bbee00812 Date: 2018-04-09 10:32 +0300 http://bitbucket.org/pypy/pypy/changeset/882bbee00812/ Log: after PyObject_GetBuffer fails, a second call also fails, PyString_AsString succeeds diff --git a/pypy/module/cpyext/test/test_bufferobject.py b/pypy/module/cpyext/test/test_bufferobject.py --- a/pypy/module/cpyext/test/test_bufferobject.py +++ b/pypy/module/cpyext/test/test_bufferobject.py @@ -81,14 +81,22 @@ return NULL; if (((unsigned char*)bp.buf)[0] != '0') { + void * buf = (void*)bp.buf; + unsigned char val[4]; + unsigned char * s = PyString_AsString(obj); + memcpy(val, bp.buf, 4); + PyBuffer_Release(&bp); + if (PyObject_GetBuffer(obj, &bp, PyBUF_SIMPLE) == -1) + return NULL; PyErr_Format(PyExc_ValueError, - "mismatch: 0x%x [%x %x %x %x...] instead of '0' (got len=%d)", + "mismatch: %p [%x %x %x %x...] now %p [%x %x %x %x...] as str '%s'", + buf, val[0], val[1], val[2], val[3], + (void *)bp.buf, ((unsigned char*)bp.buf)[0], ((unsigned char*)bp.buf)[1], ((unsigned char*)bp.buf)[2], ((unsigned char*)bp.buf)[3], - ((unsigned char*)bp.buf)[4], - bp.len); + s); PyBuffer_Release(&bp); return NULL; } From pypy.commits at gmail.com Mon Apr 9 04:32:03 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 09 Apr 2018 01:32:03 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: test, fix for issue 2787 (now passes on -A --python=python3) Message-ID: <5acb2503.89571c0a.126e7.10dd@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r94284:c84d22857147 Date: 2018-04-09 11:31 +0300 http://bitbucket.org/pypy/pypy/changeset/c84d22857147/ Log: test, fix for issue 2787 (now passes on -A --python=python3) diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -33,7 +33,7 @@ space.newint(addr.get_protocol()), space.newint(addr.get_pkttype()), space.newint(addr.get_hatype()), - space.newtext(addr.get_haddr())]) + space.newbytes(addr.get_haddr())]) elif rsocket.HAS_AF_UNIX and isinstance(addr, rsocket.UNIXAddress): path = addr.get_path() if _c.linux and len(path) > 0 and path[0] == '\x00': diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -768,13 +768,14 @@ def test_convert_between_tuple_and_sockaddr_ll(self): import _socket s = _socket.socket(_socket.AF_PACKET, _socket.SOCK_RAW) - assert s.getsockname() == ('', 0, 0, 0, '') + assert s.getsockname() == ('', 0, 0, 0, b''), 's.getsockname %s' % str(s.getsockname()) s.bind(('lo', 123)) a, b, c, d, e = s.getsockname() assert (a, b, c) == ('lo', 123, 0) assert isinstance(d, int) - assert isinstance(e, str) + assert isinstance(e, bytes) assert 0 <= len(e) <= 8 + s.close() class AppTestSocketTCP: From pypy.commits at gmail.com Mon Apr 9 07:25:34 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 09 Apr 2018 04:25:34 -0700 (PDT) Subject: [pypy-commit] extradoc extradoc: add historical info from sprint Message-ID: <5acb4dae.55a5df0a.22ea6.1333@mx.google.com> Author: Matti Picus Branch: extradoc Changeset: r5886:3b5b787ccbcf Date: 2018-04-09 14:25 +0200 http://bitbucket.org/pypy/extradoc/changeset/3b5b787ccbcf/ Log: add historical info from sprint diff --git a/sprintinfo/leysin-winter-2018/planning.txt b/sprintinfo/leysin-winter-2018/planning.txt new file mode 100644 --- /dev/null +++ b/sprintinfo/leysin-winter-2018/planning.txt @@ -0,0 +1,100 @@ +What we were working on - Sunday +====================== + +- finish unicode-utf8 - need to finish merge to default +- document the different rpython decorators like enforceargs, signature, and then interp-level unwrap_spec - arianna +- move more things to extra-tests, especially lib_pypy tests +- py3 test runners are too complicated - flub +- see what's missing to get PyGTK running (see also PyGObject and https://github.com/pygobject/pycairo/issues/90 ) - mjacob PAUSED +- revdb: bring it up to date, improve usability, other improvements (xoraxax) +- more cffi tutorial/overview rewrite - jdb + + +Plans for Monday +============= + +- cffi tutorial/overview rewrite - jdb + - document the different rpython decorators like signature, and then interp-level unwrap_spec - arianna DROPPED +- JIT topics: guard-compatible, and the subsequent research project to save and reuse traces across processes (remi) + -> https://docs.google.com/spreadsheets/d/1bIx8NwHOo5DYTOwnenOncN0YKsV2nknvzaHWJazqPnY/edit?usp=sharing + +- revdb: bring it up to date, improve usability, other improvements (xoraxax, armin around, mjacob) +- finish unicode-utf8 - merge of default. Gave up with merge of py3.5 into a new branch off py3.5 ``unicode-utf8-py3.5`` +- cpyext performance (anto, jdb shadowing) +- cpyext cycle detection discussion (Stefan, Armin, anto) +- py3 test runners are too complicated - (flub, ronan) + +Tues +==== + +- clif and cppyy https://gist.github.com/mattip/76be19a4cffab6991ee957059814d95a NOBODY KNOWS +- sqlite3 failure in pypy3.6 branch: http://buildbot.pypy.org/summary/longrepr?testname=unmodified&builder=pypy-c-jit-linux-x86-64&build=5319&mod=lib-python.3.test.test_sqlite - arianna w/manuel SOME PROGRESS, waiting for PR +- continue revdb merge - hunting ll_assert bug (xoraxax, armin around, mjacob around) +- looking for cpyext improvements (anton, mattip) SPED UP TYPECHECKS, more to do + +- JIT topics: understand the performance characteristics of guard-compatible (remi) + - found cases where guard_compatible produces more bridges than it should + - attempted to use vtune, perf, valgrind to look at assembly, but that seems impossible +- presentation about how cffi works (arigo, jdb, flub, etc.) DONE +- cpyext cycle detection implementation (stefan) +- py3 test runners refactoring (ronan, flub) PROGRESS +- support for positional-only arguments? (mjacob) NO OPINION +- fix py3.6 lib-python tests - start with test_asyncgen (mjacob) IN-PROGRESS +- pygame windows binary wheel, appveyor pypy (Ren?) PROGRESS + +Wed - rest day +=========== + +- skiing (arigo, flub, antonio) +- hiking (remi) +- guard_compatible: found that our fix from yesterday was incomplete (remi) +- being sick, laying around (Ren?). +- cpyext: manage to call tp_traverse from the GC, find alternative approaches to collect cycles (stefan) + - use boehm for cpyext + - trace heaps individually: might need several cycles to collect everything (like now), trace at once: only need one cycle +- make new tests less slow + +Thursday +======== + +- guard_compatible (remi) - VTune integration working; new microbenchmarks; found a segfault +- pygame windows binary wheel, appveyor pypy, report differences to cpython in issues.(Ren?) almost done. - error reported, nightly compiled different to release compiled (release user, can not use nightly generated wheel) +- try gc.collect(1) near display.flip(), benchmark to test dropped frames. jit.incremental? import pypyjit... turn on/off (Ren?) - lots of graphs, found GC deallocating all at once, maybe fixed? +- test runner refactoring (ronan) IN PROGRESS +- cpyext: optimize tp_as_mapping slots (anto, ronan around, armin around even if he doesn't want to) MOSTLY DONE, optimized all the slots; more optimizations possible +- cpyext cycle detection: clean up code, continue with implementation (stefan) +- fix test_async DEBUGGING + +Friday +====== + +- guard_compatible debugging (remi) +- vtune support: finish and merge the branch, missing would be giving some better name to the loops than "rpy_loop1" +- Make minimal 'pygame.examples.framedrop' benchmark. check rect benchmark later on.(Ren?) +- finish windows pygame wheel (Ren?) +- cpyext cycle detection: remove generic_cpy_call and directly call tp_traverse, do further cleanup (stefan) +- try to finish and possibly merge the cpyext branch about slots (anto) +- investigate rawrefcount-free-early (anto, armin) +- test runner refactoring (ronan) + +All tasks +======= + +- make win32 builds green +- make packaging more like cpython/portable builds +- get CI builders for PyPy into mainstream projects (Numpy, Scipy, lxml, uwsgi) +- get more of scientific stack working (tensorflow?) +- cpyext performance improvements +- General 3.5 and 3.6 improvements +- update www.pypy.org, speed.pypy.org (web devs needed) +- go over projects at https://bitbucket.org/pypy, delete or document dead projects +- look at google's clif https://github.com/google/clif and try to make a backend for PyPy, how is it different from cppyy +- have a look at https://github.com/aguinet/dragonffi as well +- use cffi instead of ctypes in rpython +- pygame test failures +- what about vmprof ? +- separate cffi versioning from cpyext versioning? + + + + From pypy.commits at gmail.com Mon Apr 9 10:46:47 2018 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 09 Apr 2018 07:46:47 -0700 (PDT) Subject: [pypy-commit] pypy.org extradoc: capitalize PyPy Message-ID: <5acb7cd7.c2b61c0a.10abc.8b6d@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: extradoc Changeset: r921:7a8d6cacd50d Date: 2018-04-09 16:46 +0200 http://bitbucket.org/pypy/pypy.org/changeset/7a8d6cacd50d/ Log: capitalize PyPy diff --git a/don2.html b/don2.html --- a/don2.html +++ b/don2.html @@ -2,7 +2,7 @@
  • - Donate towards general pypy progress
    + Donate towards general PyPy progress
  • From pypy.commits at gmail.com Mon Apr 9 10:46:44 2018 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 09 Apr 2018 07:46:44 -0700 (PDT) Subject: [pypy-commit] pypy.org extradoc: - remove flattr link Message-ID: <5acb7cd4.54d91c0a.5f3ac.7a79@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: extradoc Changeset: r920:272637cfd909 Date: 2018-04-09 16:45 +0200 http://bitbucket.org/pypy/pypy.org/changeset/272637cfd909/ Log: - remove flattr link - make "Donate towards general pypy progress" not a link (since it does nothing when you click on it) diff --git a/archive.html b/archive.html --- a/archive.html +++ b/archive.html @@ -33,8 +33,6 @@
    diff --git a/compat.html b/compat.html --- a/compat.html +++ b/compat.html @@ -33,8 +33,6 @@
    diff --git a/contact.html b/contact.html --- a/contact.html +++ b/contact.html @@ -33,8 +33,6 @@
    diff --git a/don2.html b/don2.html --- a/don2.html +++ b/don2.html @@ -2,7 +2,7 @@
  • - Donate towards general pypy progress
    + Donate towards general pypy progress
  • diff --git a/download.html b/download.html --- a/download.html +++ b/download.html @@ -33,8 +33,6 @@
    diff --git a/features.html b/features.html --- a/features.html +++ b/features.html @@ -33,8 +33,6 @@
    diff --git a/howtohelp.html b/howtohelp.html --- a/howtohelp.html +++ b/howtohelp.html @@ -33,8 +33,6 @@
    diff --git a/index.html b/index.html --- a/index.html +++ b/index.html @@ -33,8 +33,6 @@
    diff --git a/numpydonate.html b/numpydonate.html --- a/numpydonate.html +++ b/numpydonate.html @@ -33,8 +33,6 @@
    diff --git a/people.html b/people.html --- a/people.html +++ b/people.html @@ -33,8 +33,6 @@
    diff --git a/performance.html b/performance.html --- a/performance.html +++ b/performance.html @@ -33,8 +33,6 @@
    diff --git a/py3donate.html b/py3donate.html --- a/py3donate.html +++ b/py3donate.html @@ -33,8 +33,6 @@
    diff --git a/source/_layouts/site.genshi b/source/_layouts/site.genshi --- a/source/_layouts/site.genshi +++ b/source/_layouts/site.genshi @@ -63,8 +63,6 @@ diff --git a/sponsor.html b/sponsor.html --- a/sponsor.html +++ b/sponsor.html @@ -33,8 +33,6 @@
    diff --git a/success.html b/success.html --- a/success.html +++ b/success.html @@ -33,8 +33,6 @@
    diff --git a/tmdonate.html b/tmdonate.html --- a/tmdonate.html +++ b/tmdonate.html @@ -33,8 +33,6 @@
    diff --git a/tmdonate2.html b/tmdonate2.html --- a/tmdonate2.html +++ b/tmdonate2.html @@ -33,8 +33,6 @@
    From pypy.commits at gmail.com Mon Apr 9 17:31:22 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 09 Apr 2018 14:31:22 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: merge default into py3.5 Message-ID: <5acbdbaa.8d6d1c0a.52ff0.2598@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r94285:e3384cb52177 Date: 2018-04-10 00:30 +0300 http://bitbucket.org/pypy/pypy/changeset/e3384cb52177/ Log: merge default into py3.5 diff --git a/pypy/doc/release-v6.0.0.rst b/pypy/doc/release-v6.0.0.rst --- a/pypy/doc/release-v6.0.0.rst +++ b/pypy/doc/release-v6.0.0.rst @@ -8,26 +8,26 @@ the dual release. This release is a feature release following our previous 5.10 incremental -release in late December 2017, with many improvements in the C-API -compatability layer and other improvements in speed and CPython compatibility. -Since the changes affect the included python development header files, all -c-extension modules must be recompiled for this version. +release in late December 2017. Our C-API compatability layer ``cpyext`` is +now much faster (see the `blog post`_) as well as more complete. We have made +many other improvements in speed and CPython compatibility. Since the changes +affect the included python development header files, all c-extension modules must +be recompiled for this version. -The Windows PyPy3.5 release is still considered beta-quality. There are issues -with unicode handling especially around system calls and c-extensions. +The Windows PyPy3.5 release is still considered beta-quality. There are open +issues with unicode handling especially around system calls and c-extensions. -The Matplotlib TkAgg backend now works with PyPy. PyGame and pygtk also now can -work with PyPy. +The Matplotlib TkAgg backend now works with PyPy, as do pygame and pygobject. As always, this release is 100% compatible with the previous one and fixed several issues and bugs raised by the growing community of PyPy users. We strongly recommend updating. +We updated the cffi module included in PyPy to version 1.11.5 + The utf8 branch that changes internal representation of unicode to utf8 did not -make it into the release. We also began working on a Python3.6 implementation, -help is welcome. - -We updated the cffi module included in PyPy to version 1.11.5 +make it into the release, so there is still more goodness coming. We also +began working on a Python3.6 implementation, help is welcome. You can download the v6.0 releases here: @@ -46,6 +46,7 @@ .. _`RPython`: https://rpython.readthedocs.org .. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly .. _`help`: project-ideas.html +.. _`blog post`: https://morepypy.blogspot.it/2017/10/cape-of-good-hope-for-pypy-hello-from.html What is PyPy? ============= diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -5,4 +5,8 @@ .. this is a revision shortly after release-pypy-6.0.0 .. startrev: 2e04adf1b89f +.. branch: cpyext-subclass-setattr +Fix for python-level classes that inherit from C-API types, previously the +`w_obj` was not necessarily preserved throughout the lifetime of the `pyobj` +which led to cases where instance attributes were lost. Fixes issue #2793 diff --git a/pypy/interpreter/test/test_typedef.py b/pypy/interpreter/test/test_typedef.py --- a/pypy/interpreter/test/test_typedef.py +++ b/pypy/interpreter/test/test_typedef.py @@ -423,3 +423,10 @@ def test_get_with_none_arg(self): raises(TypeError, type.__dict__['__mro__'].__get__, None) raises(TypeError, type.__dict__['__mro__'].__get__, None, None) + + def test_builtin_readonly_property(self): + import sys + x = lambda: 5 + e = raises(AttributeError, 'x.func_globals = {}') + if '__pypy__' in sys.builtin_module_names: + assert str(e.value) == "readonly attribute 'func_globals'" diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py --- a/pypy/interpreter/typedef.py +++ b/pypy/interpreter/typedef.py @@ -313,12 +313,18 @@ self.reqcls, Arguments(space, [w_obj, space.newtext(self.name)])) + def readonly_attribute(self, space): # overwritten in cpyext + if self.name == '': + raise oefmt(space.w_AttributeError, "readonly attribute") + else: + raise oefmt(space.w_AttributeError, "readonly attribute '%s'", self.name) + def descr_property_set(self, space, w_obj, w_value): """property.__set__(obj, value) Change the value of the property of the given obj.""" fset = self.fset if fset is None: - raise oefmt(space.w_AttributeError, "readonly attribute") + raise self.readonly_attribute(space) try: fset(self, space, w_obj, w_value) except DescrMismatch: diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py --- a/pypy/module/cpyext/pyobject.py +++ b/pypy/module/cpyext/pyobject.py @@ -60,10 +60,10 @@ def _cpyext_attach_pyobj(self, space, py_obj): self._cpy_ref = py_obj - rawrefcount.create_link_pyobj(self, py_obj) + rawrefcount.create_link_pypy(self, py_obj) cls._cpyext_attach_pyobj = _cpyext_attach_pyobj -add_direct_pyobj_storage(W_BaseCPyObject) +add_direct_pyobj_storage(W_BaseCPyObject) add_direct_pyobj_storage(W_TypeObject) add_direct_pyobj_storage(W_NoneObject) add_direct_pyobj_storage(W_BoolObject) @@ -415,3 +415,14 @@ @cpython_api([rffi.VOIDP], lltype.Signed, error=CANNOT_FAIL) def _Py_HashPointer(space, ptr): return rffi.cast(lltype.Signed, ptr) + + at cpython_api([PyObject], lltype.Void) +def Py_IncRef(space, obj): + # used only ifdef PYPY_DEBUG_REFCOUNT + if obj: + incref(space, obj) + + at cpython_api([PyObject], lltype.Void) +def Py_DecRef(space, obj): + # used only ifdef PYPY_DEBUG_REFCOUNT + decref(space, obj) diff --git a/pypy/module/cpyext/src/object.c b/pypy/module/cpyext/src/object.c --- a/pypy/module/cpyext/src/object.c +++ b/pypy/module/cpyext/src/object.c @@ -5,18 +5,6 @@ extern void _PyPy_Free(void *ptr); extern void *_PyPy_Malloc(Py_ssize_t size); -void -Py_IncRef(PyObject *o) -{ - Py_XINCREF(o); -} - -void -Py_DecRef(PyObject *o) -{ - Py_XDECREF(o); -} - /* * The actual value of this variable will be the address of * pyobject.w_marker_deallocating, and will be set by diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c --- a/pypy/module/cpyext/test/array.c +++ b/pypy/module/cpyext/test/array.c @@ -2936,6 +2936,87 @@ return PyLong_FromLong(obj1->ob_type->tp_dealloc == obj2->ob_type->tp_dealloc); } +static PyObject * +subclass_with_attribute(PyObject *self, PyObject* args) { + /* what happens when we use tp_alloc to create the subclass, then + * assign to the w_obj via python, then get the GC to collect? + * The w_obj should not be collected!! + */ + PyObject * obj, *sub, *attrib, *funcname, *attribname, *collect, *res, *tup; + PyTypeObject * subtype; + int i; + if (!PyArg_ParseTuple(args, "OOOO", &obj, &funcname, &attribname, &collect)) { + return NULL; + } + if (!PyType_Check(obj)) { + PyErr_SetString(PyExc_TypeError, + "expected type object"); + return NULL; + } + subtype = (PyTypeObject*)obj; + sub = subtype->tp_alloc(subtype, 0); + if (!sub) { + return NULL; + } + attrib = PyObject_GetAttr(sub, funcname); + if (!attrib || (attrib == Py_None) ) { + PyErr_SetString(PyExc_ValueError, + "could not find function to call"); + Py_XDECREF(attrib); + Py_DECREF(sub); + return NULL; + } + tup = PyTuple_New(0); + /* + #ifdef PYPY_VERSION + printf("calling addattrib pypylink %lu \n", sub->ob_pypy_link); + #endif + */ + res = PyObject_Call(attrib, tup, NULL); + /* + #ifdef PYPY_VERSION + printf("after addattrib pypylink %lu \n", sub->ob_pypy_link); + #endif + */ + Py_DECREF(attrib); + if (res == NULL) { + Py_DECREF(tup); + Py_DECREF(sub); + return NULL; + } + Py_DECREF(res); + for(i=0; i<10; i++) { + /* + #ifdef PYPY_VERSION + printf("starting loop iteration %d refcnt %lu pypylink %lu \n", i, + sub->ob_refcnt, sub->ob_pypy_link); + #else + printf("starting loop iteration %d refcnt %lu\n", i, sub->ob_refcnt); + #endif + */ + attrib = PyObject_GetAttr(sub, attribname); + if (!attrib || (attrib == Py_None)) { + PyErr_SetString(PyExc_ValueError, + "could not find attrib on object"); + Py_XDECREF(attrib); + Py_DECREF(tup); + Py_DECREF(sub); + return NULL; + } + Py_XDECREF(attrib); + res = PyObject_Call(collect, tup, NULL); + if (res == NULL) { + Py_DECREF(tup); + Py_DECREF(sub); + return NULL; + } + Py_DECREF(res); + } + Py_DECREF(tup); + Py_DECREF(sub); + Py_RETURN_NONE; +} + /*********************** Install Module **************************/ static PyMethodDef a_methods[] = { @@ -2948,6 +3029,7 @@ {"write_buffer_len", write_buffer_len, METH_O, NULL}, {"same_dealloc", (PyCFunction)same_dealloc, METH_VARARGS, NULL}, {"getitem", (PyCFunction)getitem, METH_VARARGS, NULL}, + {"subclass_with_attribute", (PyCFunction)subclass_with_attribute, METH_VARARGS, NULL}, {NULL, NULL, 0, NULL} /* Sentinel */ }; diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py --- a/pypy/module/cpyext/test/test_arraymodule.py +++ b/pypy/module/cpyext/test/test_arraymodule.py @@ -179,3 +179,15 @@ # array_subscr does) raises(IndexError, module.getitem, a, -5) + def test_subclass_with_attribute(self): + module = self.import_module(name='array') + class Sub(module.array): + def addattrib(self): + print('called addattrib') + self.attrib = True + import gc + module.subclass_with_attribute(Sub, "addattrib", "attrib", gc.collect) + if self.runappdirect: + assert Sub.__module__ == 'pypy.module.cpyext.test.test_arraymodule' + assert str(Sub) == "" + diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py --- a/pypy/module/cpyext/test/test_tupleobject.py +++ b/pypy/module/cpyext/test/test_tupleobject.py @@ -67,6 +67,18 @@ assert space.int_w(space.getitem(w_tuple, space.wrap(i))) == 42 + i decref(space, ar[0]) + py_tuple = state.ccall("PyTuple_New", 1) + ar[0] = py_tuple + api._PyTuple_Resize(ar, 1) + assert api.PyTuple_Size(ar[0]) == 1 + decref(space, ar[0]) + + py_tuple = state.ccall("PyTuple_New", 1) + ar[0] = py_tuple + api._PyTuple_Resize(ar, 5) + assert api.PyTuple_Size(ar[0]) == 5 + decref(space, ar[0]) + lltype.free(ar, flavor='raw') def test_setitem(self, space, api): diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -5,7 +5,7 @@ from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.api import generic_cpy_call from pypy.module.cpyext.pyobject import make_ref, from_ref, decref, as_pyobj -from pypy.module.cpyext.typeobject import cts, PyTypeObjectPtr +from pypy.module.cpyext.typeobject import cts, PyTypeObjectPtr, W_PyCTypeObject @@ -410,33 +410,42 @@ def test_type_dict(self): foo = self.import_module("foo") module = self.import_extension('test', [ - ("hack_tp_dict", "METH_O", + ("hack_tp_dict", "METH_VARARGS", ''' - PyTypeObject *type = args->ob_type; + PyTypeObject *type, *obj; PyObject *a1 = PyLong_FromLong(1); PyObject *a2 = PyLong_FromLong(2); PyObject *value; + PyObject * key; + if (!PyArg_ParseTuple(args, "OO", &obj, &key)) + return NULL; + type = obj->ob_type; - if (PyDict_SetItemString(type->tp_dict, "a", + if (PyDict_SetItem(type->tp_dict, key, a1) < 0) return NULL; Py_DECREF(a1); PyType_Modified(type); - value = PyObject_GetAttrString((PyObject *)type, "a"); + value = PyObject_GetAttr((PyObject *)type, key); Py_DECREF(value); - if (PyDict_SetItemString(type->tp_dict, "a", + if (PyDict_SetItem(type->tp_dict, key, a2) < 0) return NULL; Py_DECREF(a2); PyType_Modified(type); - value = PyObject_GetAttrString((PyObject *)type, "a"); + value = PyObject_GetAttr((PyObject *)type, key); return value; ''' ) ]) obj = foo.new() - assert module.hack_tp_dict(obj) == 2 + assert module.hack_tp_dict(obj, "a") == 2 + class Sub(foo.fooType): + pass + obj = Sub() + assert module.hack_tp_dict(obj, "b") == 2 + def test_tp_descr_get(self): module = self.import_extension('foo', [ @@ -572,6 +581,23 @@ def test_typeslots(self, space): assert cts.macros['Py_tp_doc'] == 56 + def test_subclass_not_PyCTypeObject(self, space, api): + pyobj = make_ref(space, api.PyLong_Type) + py_type = rffi.cast(PyTypeObjectPtr, pyobj) + w_pyclass = W_PyCTypeObject(space, py_type) + w_class = space.appexec([w_pyclass], """(base): + class Sub(base): + def addattrib(self, value): + self.attrib = value + return Sub + """) + assert w_pyclass in w_class.mro_w + assert isinstance(w_pyclass, W_PyCTypeObject) + assert not isinstance(w_class, W_PyCTypeObject) + assert w_pyclass.is_cpytype() + # XXX document the current status, not clear if this is desirable + assert w_class.is_cpytype() + class AppTestSlots(AppTestCpythonExtensionBase): def setup_class(cls): @@ -1558,6 +1584,64 @@ pass C(42) # assert is not aborting + def test_getset(self): + module = self.import_extension('foo', [ + ("get_instance", "METH_NOARGS", + ''' + return PyObject_New(PyObject, &Foo_Type); + ''' + ), ("get_number", "METH_NOARGS", + ''' + return PyInt_FromLong(my_global_number); + ''' + )], prologue=''' + #if PY_MAJOR_VERSION > 2 + #define PyInt_FromLong PyLong_FromLong + #endif + static long my_global_number; + static PyTypeObject Foo_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "foo.foo", + }; + static PyObject *bar_get(PyObject *foo, void *closure) + { + return PyInt_FromLong(1000 + (long)closure); + } + static PyObject *baz_get(PyObject *foo, void *closure) + { + return PyInt_FromLong(2000 + (long)closure); + } + static int baz_set(PyObject *foo, PyObject *x, void *closure) + { + if (x != NULL) + my_global_number = 3000 + (long)closure + PyInt_AsLong(x); + else + my_global_number = 4000 + (long)closure; + return 0; + } + static PyGetSetDef foo_getset[] = { + { "bar", bar_get, NULL, "mybardoc", (void *)42 }, + { "baz", baz_get, baz_set, "mybazdoc", (void *)43 }, + { NULL } + }; + ''', more_init = ''' + Foo_Type.tp_getset = foo_getset; + Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT; + if (PyType_Ready(&Foo_Type) < 0) INITERROR; + ''') + foo = module.get_instance() + assert foo.bar == 1042 + assert foo.bar == 1042 + assert foo.baz == 2043 + foo.baz = 50000 + assert module.get_number() == 53043 + e = raises(AttributeError, "foo.bar = 0") + assert str(e.value).startswith("attribute 'bar' of '") + assert str(e.value).endswith("foo' objects is not writable") + del foo.baz + assert module.get_number() == 4043 + raises(AttributeError, "del foo.bar") + class AppTestHashable(AppTestCpythonExtensionBase): def test_unhashable(self): diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py --- a/pypy/module/cpyext/tupleobject.py +++ b/pypy/module/cpyext/tupleobject.py @@ -187,6 +187,8 @@ PyErr_BadInternalCall(space) oldref = rffi.cast(PyTupleObject, ref) oldsize = oldref.c_ob_size + if oldsize == newsize: + return 0 ptup = state.ccall("PyTuple_New", newsize) if not ptup: state.check_and_raise_exception(always=True) @@ -199,8 +201,9 @@ to_cp = newsize for i in range(to_cp): ob = oldref.c_ob_item[i] - incref(space, ob) - newref.c_ob_item[i] = ob + if ob: + incref(space, ob) + newref.c_ob_item[i] = ob except: decref(space, p_ref[0]) p_ref[0] = lltype.nullptr(PyObject.TO) diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -1,5 +1,5 @@ from rpython.rlib.unroll import unrolling_iterable -from rpython.rlib import jit +from rpython.rlib import jit, rawrefcount from rpython.rlib.objectmodel import specialize, we_are_translated from rpython.rtyper.lltypesystem import rffi, lltype @@ -57,19 +57,26 @@ class W_GetSetPropertyEx(GetSetProperty): def __init__(self, getset, w_type): self.getset = getset - self.name = rffi.charp2str(getset.c_name) self.w_type = w_type - doc = set = get = None + doc = fset = fget = fdel = None if doc: # XXX dead code? doc = rffi.charp2str(getset.c_doc) if getset.c_get: - get = GettersAndSetters.getter.im_func + fget = GettersAndSetters.getter.im_func if getset.c_set: - set = GettersAndSetters.setter.im_func - GetSetProperty.__init__(self, get, set, None, doc, + fset = GettersAndSetters.setter.im_func + fdel = GettersAndSetters.deleter.im_func + GetSetProperty.__init__(self, fget, fset, fdel, doc, cls=None, use_closure=True, tag="cpyext_1") + self.name = rffi.charp2str(getset.c_name) + + def readonly_attribute(self, space): # overwritten + raise oefmt(space.w_AttributeError, + "attribute '%s' of '%N' objects is not writable", + self.name, self.w_type) + def PyDescr_NewGetSet(space, getset, w_type): return W_GetSetPropertyEx(getset, w_type) @@ -455,6 +462,16 @@ state = space.fromcache(State) state.check_and_raise_exception() + def deleter(self, space, w_self): + assert isinstance(self, W_GetSetPropertyEx) + check_descr(space, w_self, self.w_type) + res = generic_cpy_call( + space, self.getset.c_set, w_self, None, + self.getset.c_closure) + if rffi.cast(lltype.Signed, res) < 0: + state = space.fromcache(State) + state.check_and_raise_exception() + def member_getter(self, space, w_self): assert isinstance(self, W_MemberDescr) check_descr(space, w_self, self.w_type) @@ -518,6 +535,10 @@ self.w_doc = space.newtext_or_none(extract_doc(rawdoc, name)) self.text_signature = extract_txtsig(rawdoc, name) + def _cpyext_attach_pyobj(self, space, py_obj): + self._cpy_ref = py_obj + rawrefcount.create_link_pyobj(self, py_obj) + @bootstrap_function def init_typeobject(space): make_typedescr(space.w_type.layout.typedef, @@ -667,7 +688,6 @@ try: w_obj = _type_realize(space, py_obj) finally: - name = rffi.charp2str(cts.cast('char*', pto.c_tp_name)) pto.c_tp_flags &= ~Py_TPFLAGS_READYING pto.c_tp_flags |= Py_TPFLAGS_READY return w_obj @@ -762,7 +782,6 @@ base = pto.c_tp_base base_pyo = rffi.cast(PyObject, pto.c_tp_base) if base and not base.c_tp_flags & Py_TPFLAGS_READY: - name = rffi.charp2str(cts.cast('char*', base.c_tp_name)) type_realize(space, base_pyo) if base and not pto.c_ob_type: # will be filled later pto.c_ob_type = base.c_ob_type From pypy.commits at gmail.com Tue Apr 10 04:09:14 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 10 Apr 2018 01:09:14 -0700 (PDT) Subject: [pypy-commit] pypy default: ignore incompatible exception message in CPython doctest Message-ID: <5acc712a.55a81c0a.323c4.5871@mx.google.com> Author: Matti Picus Branch: Changeset: r94286:4b8b957e45a4 Date: 2018-04-10 10:07 +0300 http://bitbucket.org/pypy/pypy/changeset/4b8b957e45a4/ Log: ignore incompatible exception message in CPython doctest diff --git a/lib-python/2.7/test/test_generators.py b/lib-python/2.7/test/test_generators.py --- a/lib-python/2.7/test/test_generators.py +++ b/lib-python/2.7/test/test_generators.py @@ -398,7 +398,10 @@ 0 >>> type(i.gi_frame) ->>> i.gi_running = 42 + +PyPy prints "readonly attribute 'gi_running'" so ignore the exception detail + +>>> i.gi_running = 42 # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ... TypeError: readonly attribute From pypy.commits at gmail.com Tue Apr 10 04:44:20 2018 From: pypy.commits at gmail.com (cfbolz) Date: Tue, 10 Apr 2018 01:44:20 -0700 (PDT) Subject: [pypy-commit] pypy default: merge pyparser-improvements-2 Message-ID: <5acc7964.cae0df0a.6738a.b0b5@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r94294:e66f24650daf Date: 2018-04-10 09:41 +0200 http://bitbucket.org/pypy/pypy/changeset/e66f24650daf/ Log: merge pyparser-improvements-2 - fixes .offset values of SyntaxError, which is 1-based (but the raising code sometimes assumed it was 0-based) - expand some abbreviations - better error messages for non-matching parenthesis diff --git a/lib-python/2.7/test/test_eof.py b/lib-python/2.7/test/test_eof.py --- a/lib-python/2.7/test/test_eof.py +++ b/lib-python/2.7/test/test_eof.py @@ -5,7 +5,7 @@ class EOFTestCase(unittest.TestCase): def test_EOFC(self): - expect = "EOL while scanning string literal (, line 1)" + expect = "end of line (EOL) while scanning string literal (, line 1)" try: eval("""'this is a test\ """) @@ -15,7 +15,7 @@ raise test_support.TestFailed def test_EOFS(self): - expect = ("EOF while scanning triple-quoted string literal " + expect = ("end of file (EOF) while scanning triple-quoted string literal " "(, line 1)") try: eval("""'''this is a test""") diff --git a/lib-python/2.7/test/test_traceback.py b/lib-python/2.7/test/test_traceback.py --- a/lib-python/2.7/test/test_traceback.py +++ b/lib-python/2.7/test/test_traceback.py @@ -123,10 +123,7 @@ self.assertEqual(len(err), 4) self.assertEqual(err[1].strip(), "print(2)") self.assertIn("^", err[2]) - if check_impl_detail(): - self.assertEqual(err[1].find("p"), err[2].find("^")) - if check_impl_detail(pypy=True): - self.assertEqual(err[1].find("2)") + 1, err[2].find("^")) + self.assertEqual(err[1].find("p"), err[2].find("^")) def test_base_exception(self): # Test that exceptions derived from BaseException are formatted right diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -10,3 +10,9 @@ Fix for python-level classes that inherit from C-API types, previously the `w_obj` was not necessarily preserved throughout the lifetime of the `pyobj` which led to cases where instance attributes were lost. Fixes issue #2793 + + +.. branch: pyparser-improvements-2 + +Improve line offsets that are reported by SyntaxError. Improve error messages +for a few situations, including mismatched parenthesis. diff --git a/pypy/interpreter/pyparser/error.py b/pypy/interpreter/pyparser/error.py --- a/pypy/interpreter/pyparser/error.py +++ b/pypy/interpreter/pyparser/error.py @@ -6,6 +6,7 @@ lastlineno=0): self.msg = msg self.lineno = lineno + # NB: offset is a 1-based index! self.offset = offset self.text = text self.filename = filename diff --git a/pypy/interpreter/pyparser/parser.py b/pypy/interpreter/pyparser/parser.py --- a/pypy/interpreter/pyparser/parser.py +++ b/pypy/interpreter/pyparser/parser.py @@ -199,6 +199,7 @@ self.token_type = token_type self.value = value self.lineno = lineno + # this is a 0-based index self.column = column self.line = line self.expected = expected diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py --- a/pypy/interpreter/pyparser/pyparse.py +++ b/pypy/interpreter/pyparser/pyparse.py @@ -188,7 +188,9 @@ if e.expected_str is not None: msg += " (expected '%s')" % e.expected_str - raise new_err(msg, e.lineno, e.column, e.line, + # parser.ParseError(...).column is 0-based, but the offsets in the + # exceptions in the error module are 1-based, hence the '+ 1' + raise new_err(msg, e.lineno, e.column + 1, e.line, compile_info.filename) else: tree = self.root diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py --- a/pypy/interpreter/pyparser/pytokenizer.py +++ b/pypy/interpreter/pyparser/pytokenizer.py @@ -73,14 +73,14 @@ logical line; continuation lines are included. """ token_list = [] - lnum = parenlev = continued = 0 + lnum = continued = 0 namechars = NAMECHARS numchars = NUMCHARS contstr, needcont = '', 0 contline = None indents = [0] last_comment = '' - parenlevstart = (0, 0, "") + parenstack = [] # make the annotator happy endDFA = DUMMY_DFA @@ -97,7 +97,7 @@ if contstr: if not line: raise TokenError( - "EOF while scanning triple-quoted string literal", + "end of file (EOF) while scanning triple-quoted string literal", strstart[2], strstart[0], strstart[1]+1, token_list, lnum-1) endmatch = endDFA.recognize(line) @@ -123,7 +123,7 @@ contline = contline + line continue - elif parenlev == 0 and not continued: # new statement + elif not parenstack and not continued: # new statement if not line: break column = 0 while pos < max: # measure leading whitespace @@ -143,21 +143,21 @@ token_list.append((tokens.INDENT, line[:pos], lnum, 0, line)) last_comment = '' while column < indents[-1]: - indents = indents[:-1] + indents.pop() token_list.append((tokens.DEDENT, '', lnum, pos, line)) last_comment = '' if column != indents[-1]: err = "unindent does not match any outer indentation level" - raise TokenIndentationError(err, line, lnum, 0, token_list) + raise TokenIndentationError(err, line, lnum, column+1, token_list) else: # continued statement if not line: - if parenlev > 0: - lnum1, start1, line1 = parenlevstart + if parenstack: + _, lnum1, start1, line1 = parenstack[0] raise TokenError("parenthesis is never closed", line1, lnum1, start1 + 1, token_list, lnum) - raise TokenError("EOF in multi-line statement", line, - lnum, 0, token_list) + raise TokenError("end of file (EOF) in multi-line statement", line, + lnum, 0, token_list) # XXX why is the offset 0 here? continued = 0 while pos < max: @@ -180,7 +180,7 @@ token_list.append((tokens.NUMBER, token, lnum, start, line)) last_comment = '' elif initial in '\r\n': - if parenlev <= 0: + if not parenstack: tok = (tokens.NEWLINE, last_comment, lnum, start, line) token_list.append(tok) last_comment = '' @@ -222,14 +222,22 @@ continued = 1 else: if initial in '([{': - if parenlev == 0: - parenlevstart = (lnum, start, line) - parenlev = parenlev + 1 + parenstack.append((initial, lnum, start, line)) elif initial in ')]}': - parenlev = parenlev - 1 - if parenlev < 0: + if not parenstack: raise TokenError("unmatched '%s'" % initial, line, lnum, start + 1, token_list) + opening, lnum1, start1, line1 = parenstack.pop() + if not ((opening == "(" and initial == ")") or + (opening == "[" and initial == "]") or + (opening == "{" and initial == "}")): + msg = "closing parenthesis '%s' does not match opening parenthesis '%s'" % ( + initial, opening) + + if lnum1 != lnum: + msg += " on line " + str(lnum1) + raise TokenError( + msg, line, lnum, start + 1, token_list) if token in python_opmap: punct = python_opmap[token] else: @@ -241,7 +249,7 @@ if start < 0: start = pos if start", "exec") parser = pyparse.PythonParser(fakespace) tree = parser._parse(s, info) b = time.clock() - print title, (b-a) + print fn, (b-a) def entry_point(argv): - bench("foo") + if len(argv) == 2: + fn = argv[1] + else: + fn = "../../../../rpython/rlib/unicodedata/unicodedb_5_2_0.py" + fd = os.open(fn, os.O_RDONLY, 0777) + res = [] + while True: + s = os.read(fd, 4096) + if not s: + break + res.append(s) + os.close(fd) + s = "".join(res) + print len(s) + bench(fn, s) return 0 diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py --- a/pypy/interpreter/pyparser/test/test_pyparse.py +++ b/pypy/interpreter/pyparser/test/test_pyparse.py @@ -76,14 +76,14 @@ exc = py.test.raises(SyntaxError, parse, "name another for").value assert exc.msg == "invalid syntax" assert exc.lineno == 1 - assert exc.offset == 5 + assert exc.offset == 6 assert exc.text.startswith("name another for") exc = py.test.raises(SyntaxError, parse, "x = \"blah\n\n\n").value - assert exc.msg == "EOL while scanning string literal" + assert exc.msg == "end of line (EOL) while scanning string literal" assert exc.lineno == 1 assert exc.offset == 5 exc = py.test.raises(SyntaxError, parse, "x = '''\n\n\n").value - assert exc.msg == "EOF while scanning triple-quoted string literal" + assert exc.msg == "end of file (EOF) while scanning triple-quoted string literal" assert exc.lineno == 1 assert exc.offset == 5 assert exc.lastlineno == 3 @@ -112,7 +112,7 @@ assert exc.msg == "expected an indented block" assert exc.lineno == 3 assert exc.text.startswith("pass") - assert exc.offset == 0 + assert exc.offset == 1 input = "hi\n indented" exc = py.test.raises(IndentationError, parse, input).value assert exc.msg == "unexpected indent" @@ -120,6 +120,7 @@ exc = py.test.raises(IndentationError, parse, input).value assert exc.msg == "unindent does not match any outer indentation level" assert exc.lineno == 3 + assert exc.offset == 3 def test_mac_newline(self): self.parse("this_is\ra_mac\rfile") diff --git a/pypy/interpreter/pyparser/test/test_pytokenizer.py b/pypy/interpreter/pyparser/test/test_pytokenizer.py new file mode 100644 --- /dev/null +++ b/pypy/interpreter/pyparser/test/test_pytokenizer.py @@ -0,0 +1,66 @@ +import pytest +from pypy.interpreter.pyparser import pytokenizer +from pypy.interpreter.pyparser.pygram import tokens +from pypy.interpreter.pyparser.error import TokenError + +def tokenize(s): + return pytokenizer.generate_tokens(s.splitlines(True) + ["\n"], 0) + +def check_token_error(s, msg=None, pos=-1, line=-1): + error = pytest.raises(TokenError, tokenize, s) + if msg is not None: + assert error.value.msg == msg + if pos != -1: + assert error.value.offset == pos + if line != -1: + assert error.value.lineno == line + + +class TestTokenizer(object): + + def test_simple(self): + line = "a+1" + tks = tokenize(line) + assert tks == [ + (tokens.NAME, 'a', 1, 0, line), + (tokens.PLUS, '+', 1, 1, line), + (tokens.NUMBER, '1', 1, 2, line), + (tokens.NEWLINE, '', 2, 0, '\n'), + (tokens.NEWLINE, '', 2, 0, '\n'), + (tokens.ENDMARKER, '', 2, 0, ''), + ] + + def test_error_parenthesis(self): + for paren in "([{": + check_token_error(paren + "1 + 2", + "parenthesis is never closed", + 1) + + for paren in ")]}": + check_token_error("1 + 2" + paren, + "unmatched '%s'" % (paren, ), + 6) + + for i, opening in enumerate("([{"): + for j, closing in enumerate(")]}"): + if i == j: + continue + check_token_error(opening + "1\n" + closing, + "closing parenthesis '%s' does not match opening parenthesis '%s' on line 1" % (closing, opening), + pos=1, line=2) + check_token_error(opening + "1" + closing, + "closing parenthesis '%s' does not match opening parenthesis '%s'" % (closing, opening), + pos=3, line=1) + check_token_error(opening + closing, + "closing parenthesis '%s' does not match opening parenthesis '%s'" % (closing, opening), + pos=2, line=1) + + + def test_unknown_char(self): + check_token_error("?", "Unknown character", 1) + + def test_eol_string(self): + check_token_error("x = 'a", pos=5, line=1) + + def test_eof_triple_quoted(self): + check_token_error("'''", pos=1, line=1) diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py --- a/pypy/interpreter/test/test_compiler.py +++ b/pypy/interpreter/test/test_compiler.py @@ -77,7 +77,7 @@ """) assert self.space.unwrap(w_args) == ( 'unindent does not match any outer indentation level', - ('', 3, 0, ' y\n')) + ('', 3, 2, ' y\n')) def test_getcodeflags(self): code = self.compiler.compile('from __future__ import division\n', diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py --- a/pypy/interpreter/test/test_syntax.py +++ b/pypy/interpreter/test/test_syntax.py @@ -750,7 +750,7 @@ except SyntaxError as e: assert e.lineno == 4 assert e.text.endswith('a b c d e\n') - assert e.offset == e.text.index('b') + assert e.offset == e.text.index('b') + 1 # offset is 1-based else: raise Exception("no SyntaxError??") From pypy.commits at gmail.com Tue Apr 10 04:44:22 2018 From: pypy.commits at gmail.com (cfbolz) Date: Tue, 10 Apr 2018 01:44:22 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: merge default Message-ID: <5acc7966.438b1c0a.b1165.6dcf@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: py3.5 Changeset: r94295:3d2434de5907 Date: 2018-04-10 10:36 +0200 http://bitbucket.org/pypy/pypy/changeset/3d2434de5907/ Log: merge default diff --git a/lib-python/2.7/test/test_eof.py b/lib-python/2.7/test/test_eof.py --- a/lib-python/2.7/test/test_eof.py +++ b/lib-python/2.7/test/test_eof.py @@ -5,7 +5,7 @@ class EOFTestCase(unittest.TestCase): def test_EOFC(self): - expect = "EOL while scanning string literal (, line 1)" + expect = "end of line (EOL) while scanning string literal (, line 1)" try: eval("""'this is a test\ """) @@ -15,7 +15,7 @@ raise test_support.TestFailed def test_EOFS(self): - expect = ("EOF while scanning triple-quoted string literal " + expect = ("end of file (EOF) while scanning triple-quoted string literal " "(, line 1)") try: eval("""'''this is a test""") diff --git a/lib-python/2.7/test/test_traceback.py b/lib-python/2.7/test/test_traceback.py --- a/lib-python/2.7/test/test_traceback.py +++ b/lib-python/2.7/test/test_traceback.py @@ -123,10 +123,7 @@ self.assertEqual(len(err), 4) self.assertEqual(err[1].strip(), "print(2)") self.assertIn("^", err[2]) - if check_impl_detail(): - self.assertEqual(err[1].find("p"), err[2].find("^")) - if check_impl_detail(pypy=True): - self.assertEqual(err[1].find("2)") + 1, err[2].find("^")) + self.assertEqual(err[1].find("p"), err[2].find("^")) def test_base_exception(self): # Test that exceptions derived from BaseException are formatted right diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -10,3 +10,9 @@ Fix for python-level classes that inherit from C-API types, previously the `w_obj` was not necessarily preserved throughout the lifetime of the `pyobj` which led to cases where instance attributes were lost. Fixes issue #2793 + + +.. branch: pyparser-improvements-2 + +Improve line offsets that are reported by SyntaxError. Improve error messages +for a few situations, including mismatched parenthesis. diff --git a/pypy/interpreter/pyparser/error.py b/pypy/interpreter/pyparser/error.py --- a/pypy/interpreter/pyparser/error.py +++ b/pypy/interpreter/pyparser/error.py @@ -6,6 +6,7 @@ lastlineno=0): self.msg = msg self.lineno = lineno + # NB: offset is a 1-based index! self.offset = offset self.text = text self.filename = filename diff --git a/pypy/interpreter/pyparser/parser.py b/pypy/interpreter/pyparser/parser.py --- a/pypy/interpreter/pyparser/parser.py +++ b/pypy/interpreter/pyparser/parser.py @@ -199,6 +199,7 @@ self.token_type = token_type self.value = value self.lineno = lineno + # this is a 0-based index self.column = column self.line = line self.expected = expected diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py --- a/pypy/interpreter/pyparser/pyparse.py +++ b/pypy/interpreter/pyparser/pyparse.py @@ -232,7 +232,9 @@ if e.expected_str is not None: msg += " (expected '%s')" % e.expected_str - raise new_err(msg, e.lineno, e.column, e.line, + # parser.ParseError(...).column is 0-based, but the offsets in the + # exceptions in the error module are 1-based, hence the '+ 1' + raise new_err(msg, e.lineno, e.column + 1, e.line, compile_info.filename) else: tree = self.root diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py --- a/pypy/interpreter/pyparser/pytokenizer.py +++ b/pypy/interpreter/pyparser/pytokenizer.py @@ -110,7 +110,7 @@ logical line; continuation lines are included. """ token_list = [] - lnum = parenlev = continued = 0 + lnum = continued = 0 namechars = NAMECHARS numchars = NUMCHARS contstr, needcont = '', 0 @@ -118,7 +118,7 @@ indents = [0] altindents = [0] last_comment = '' - parenlevstart = (0, 0, "") + parenstack = [] async_def = False async_def_nl = False async_def_indent = 0 @@ -138,7 +138,7 @@ if contstr: if not line: raise TokenError( - "EOF while scanning triple-quoted string literal", + "end of file (EOF) while scanning triple-quoted string literal", strstart[2], strstart[0], strstart[1]+1, token_list, lnum-1) endmatch = endDFA.recognize(line) @@ -164,7 +164,7 @@ contline = contline + line continue - elif parenlev == 0 and not continued: # new statement + elif not parenstack and not continued: # new statement if not line: break column = 0 altcolumn = 0 @@ -204,13 +204,13 @@ last_comment = '' else: while column < indents[-1]: - indents = indents[:-1] - altindents = altindents[:-1] + indents.pop() + altindents.pop() token_list.append((tokens.DEDENT, '', lnum, pos, line)) last_comment = '' if column != indents[-1]: err = "unindent does not match any outer indentation level" - raise TokenIndentationError(err, line, lnum, 0, token_list) + raise TokenIndentationError(err, line, lnum, column+1, token_list) if altcolumn != altindents[-1]: raise TabError(lnum, pos, line) if async_def_nl and async_def_indent >= indents[-1]: @@ -220,12 +220,12 @@ else: # continued statement if not line: - if parenlev > 0: - lnum1, start1, line1 = parenlevstart + if parenstack: + _, lnum1, start1, line1 = parenstack[0] raise TokenError("parenthesis is never closed", line1, lnum1, start1 + 1, token_list, lnum) - raise TokenError("EOF in multi-line statement", line, - lnum, 0, token_list) + raise TokenError("end of file (EOF) in multi-line statement", line, + lnum, 0, token_list) # XXX why is the offset 0 here? continued = 0 while pos < max: @@ -249,7 +249,7 @@ token_list.append((tokens.NUMBER, token, lnum, start, line)) last_comment = '' elif initial in '\r\n': - if parenlev <= 0: + if not parenstack: if async_def: async_def_nl = True tok = (tokens.NEWLINE, last_comment, lnum, start, line) @@ -331,14 +331,22 @@ continued = 1 else: if initial in '([{': - if parenlev == 0: - parenlevstart = (lnum, start, line) - parenlev = parenlev + 1 + parenstack.append((initial, lnum, start, line)) elif initial in ')]}': - parenlev = parenlev - 1 - if parenlev < 0: + if not parenstack: raise TokenError("unmatched '%s'" % initial, line, lnum, start + 1, token_list) + opening, lnum1, start1, line1 = parenstack.pop() + if not ((opening == "(" and initial == ")") or + (opening == "[" and initial == "]") or + (opening == "{" and initial == "}")): + msg = "closing parenthesis '%s' does not match opening parenthesis '%s'" % ( + initial, opening) + + if lnum1 != lnum: + msg += " on line " + str(lnum1) + raise TokenError( + msg, line, lnum, start + 1, token_list) if token in python_opmap: punct = python_opmap[token] else: @@ -350,7 +358,7 @@ if start < 0: start = pos if start", "exec") parser = pyparse.PythonParser(fakespace) tree = parser._parse(s, info) b = time.clock() - print title, (b-a) + print fn, (b-a) def entry_point(argv): - bench("foo") + if len(argv) == 2: + fn = argv[1] + else: + fn = "../../../../rpython/rlib/unicodedata/unicodedb_5_2_0.py" + fd = os.open(fn, os.O_RDONLY, 0777) + res = [] + while True: + s = os.read(fd, 4096) + if not s: + break + res.append(s) + os.close(fd) + s = "".join(res) + print len(s) + bench(fn, s) return 0 diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py --- a/pypy/interpreter/pyparser/test/test_pyparse.py +++ b/pypy/interpreter/pyparser/test/test_pyparse.py @@ -45,14 +45,14 @@ exc = py.test.raises(SyntaxError, parse, "name another for").value assert exc.msg == "invalid syntax" assert exc.lineno == 1 - assert exc.offset == 5 + assert exc.offset == 6 assert exc.text.startswith("name another for") exc = py.test.raises(SyntaxError, parse, "x = \"blah\n\n\n").value - assert exc.msg == "EOL while scanning string literal" + assert exc.msg == "end of line (EOL) while scanning string literal" assert exc.lineno == 1 assert exc.offset == 5 exc = py.test.raises(SyntaxError, parse, "x = '''\n\n\n").value - assert exc.msg == "EOF while scanning triple-quoted string literal" + assert exc.msg == "end of file (EOF) while scanning triple-quoted string literal" assert exc.lineno == 1 assert exc.offset == 5 assert exc.lastlineno == 3 @@ -81,7 +81,7 @@ assert exc.msg == "expected an indented block" assert exc.lineno == 3 assert exc.text.startswith("pass") - assert exc.offset == 0 + assert exc.offset == 1 input = "hi\n indented" exc = py.test.raises(IndentationError, parse, input).value assert exc.msg == "unexpected indent" @@ -89,6 +89,7 @@ exc = py.test.raises(IndentationError, parse, input).value assert exc.msg == "unindent does not match any outer indentation level" assert exc.lineno == 3 + assert exc.offset == 3 def test_taberror(self): src = """ diff --git a/pypy/interpreter/pyparser/test/test_pytokenizer.py b/pypy/interpreter/pyparser/test/test_pytokenizer.py new file mode 100644 --- /dev/null +++ b/pypy/interpreter/pyparser/test/test_pytokenizer.py @@ -0,0 +1,66 @@ +import pytest +from pypy.interpreter.pyparser import pytokenizer +from pypy.interpreter.pyparser.pygram import tokens +from pypy.interpreter.pyparser.error import TokenError + +def tokenize(s): + return pytokenizer.generate_tokens(s.splitlines(True) + ["\n"], 0) + +def check_token_error(s, msg=None, pos=-1, line=-1): + error = pytest.raises(TokenError, tokenize, s) + if msg is not None: + assert error.value.msg == msg + if pos != -1: + assert error.value.offset == pos + if line != -1: + assert error.value.lineno == line + + +class TestTokenizer(object): + + def test_simple(self): + line = "a+1" + tks = tokenize(line) + assert tks == [ + (tokens.NAME, 'a', 1, 0, line), + (tokens.PLUS, '+', 1, 1, line), + (tokens.NUMBER, '1', 1, 2, line), + (tokens.NEWLINE, '', 2, 0, '\n'), + (tokens.NEWLINE, '', 2, 0, '\n'), + (tokens.ENDMARKER, '', 2, 0, ''), + ] + + def test_error_parenthesis(self): + for paren in "([{": + check_token_error(paren + "1 + 2", + "parenthesis is never closed", + 1) + + for paren in ")]}": + check_token_error("1 + 2" + paren, + "unmatched '%s'" % (paren, ), + 6) + + for i, opening in enumerate("([{"): + for j, closing in enumerate(")]}"): + if i == j: + continue + check_token_error(opening + "1\n" + closing, + "closing parenthesis '%s' does not match opening parenthesis '%s' on line 1" % (closing, opening), + pos=1, line=2) + check_token_error(opening + "1" + closing, + "closing parenthesis '%s' does not match opening parenthesis '%s'" % (closing, opening), + pos=3, line=1) + check_token_error(opening + closing, + "closing parenthesis '%s' does not match opening parenthesis '%s'" % (closing, opening), + pos=2, line=1) + + + def test_unknown_char(self): + check_token_error("?", "Unknown character", 1) + + def test_eol_string(self): + check_token_error("x = 'a", pos=5, line=1) + + def test_eof_triple_quoted(self): + check_token_error("'''", pos=1, line=1) diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py --- a/pypy/interpreter/test/test_compiler.py +++ b/pypy/interpreter/test/test_compiler.py @@ -77,7 +77,7 @@ """) assert self.space.unwrap(w_args) == ( 'unindent does not match any outer indentation level', - ('', 3, 0, ' y\n')) + ('', 3, 2, ' y\n')) def test_getcodeflags(self): code = self.compiler.compile('from __future__ import division\n', diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py --- a/pypy/interpreter/test/test_syntax.py +++ b/pypy/interpreter/test/test_syntax.py @@ -715,8 +715,7 @@ except SyntaxError as e: assert e.lineno == 4 assert e.text.endswith('a b c d e\n') - b_pos = e.text.index('b') - assert e.offset in (b_pos, b_pos+1) # b_pos in pypy, b_pos+1 in CPython. + assert e.offset == e.text.index('b') + 1 # offset is 1-based else: raise Exception("no SyntaxError??") From pypy.commits at gmail.com Tue Apr 10 04:44:24 2018 From: pypy.commits at gmail.com (cfbolz) Date: Tue, 10 Apr 2018 01:44:24 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: fix doctests Message-ID: <5acc7968.96e81c0a.10c2.e493@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: py3.5 Changeset: r94296:d9e2802b64a3 Date: 2018-04-10 10:39 +0200 http://bitbucket.org/pypy/pypy/changeset/d9e2802b64a3/ Log: fix doctests diff --git a/lib-python/3/test/test_syntax.py b/lib-python/3/test/test_syntax.py --- a/lib-python/3/test/test_syntax.py +++ b/lib-python/3/test/test_syntax.py @@ -110,12 +110,12 @@ >>> def f(x, None): ... pass Traceback (most recent call last): -SyntaxError: invalid syntax +SyntaxError: invalid syntax (expected ')') >>> def f(*None): ... pass Traceback (most recent call last): -SyntaxError: invalid syntax +SyntaxError: invalid syntax (expected ')') >>> def f(**None): ... pass From pypy.commits at gmail.com Tue Apr 10 04:46:26 2018 From: pypy.commits at gmail.com (cfbolz) Date: Tue, 10 Apr 2018 01:46:26 -0700 (PDT) Subject: [pypy-commit] extradoc extradoc: draft post Message-ID: <5acc79e2.b6aedf0a.c4ee0.a492@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: extradoc Changeset: r5887:8a5918b257db Date: 2018-04-10 10:46 +0200 http://bitbucket.org/pypy/extradoc/changeset/8a5918b257db/ Log: draft post diff --git a/blog/draft/2018-04-better-syntaxerrors.rst b/blog/draft/2018-04-better-syntaxerrors.rst new file mode 100644 --- /dev/null +++ b/blog/draft/2018-04-better-syntaxerrors.rst @@ -0,0 +1,152 @@ +Improving SyntaxError in PyPy +============================== + +Since about a year, my halftime job at the uni has been to teach non-CS students +to program in Python. While doing that, I have been trying to see what common +stumbling blocks are, for programmers that are just starting out. There are many +things that could be said here, but one theme that emerges are +hard-to-understand error messages. One source of such error messages, +particularly when starting out, are ``SyntaxErrors``. + + +PyPy's parser (mostly following the architecture of CPython) uses a +regular-expression-based tokenizer with some cleverness to deal with +indentation, and a simple LR(1) parser. Both of these components obviously +produce errors for invalid syntax, but the messages are not that great. Often, +the message is just "invalid syntax", without any hint of what exactly is wrong. +In the last couple of weeks I have invested a little bit of effor to make them a +tiny bit better. Here are some examples. + +Missing Characters +++++++++++++++++++++ + +The first class of errors is when a token is missing, but there is only one +valid token that the parser expects. This happens most commonly by leaving out +the ':' after control flow statements (which is the syntax error I personally +still make at least a few times a day). In such situations, the parser will now +tell you which character it expected: + +.. sourcecode:: pycon + + >>>> # before + >>>> if 1 + File "", line 1 + if 1 + ^ + SyntaxError: invalid syntax + >>>> + + >>>> # after + >>>> if 1 + File "", line 1 + if 1 + ^ + SyntaxError: invalid syntax (expected ':') + >>>> + +Another example of this feature: + +.. sourcecode:: pycon + + >>>> # before + >>>> def f: + File "", line 1 + def f: + ^ + SyntaxError: invalid syntax + >>>> + + >>>> # after + >>>> def f: + File "", line 1 + def f: + ^ + SyntaxError: invalid syntax (expected '(') + >>>> + + +Parenthesis +++++++++++++++++++++++ + +Another source of errors are unmatched parenthesis. Here, PyPy has always had +slightly better error messages than CPython: + +.. sourcecode:: pycon + + >>> # CPython + >>> ) + File "", line 1 + ) + ^ + SyntaxError: invalid syntax + >>> + + >>>> # PyPy + >>> ) + File "", line 1 + ) + ^ + SyntaxError: unmatched ')' + >>>> + +The same is true for parenthesis that are never closed (the call to ``eval`` is +needed to get the error, otherwise the repl will just wait for more input): + +.. sourcecode:: pycon + + >>> # CPython + >>> eval('(') + Traceback (most recent call last): + File "", line 1, in + File "", line 1 + ( + ^ + SyntaxError: unexpected EOF while parsing + >>> + + >>>> # PyPy + >>>> eval('(') + Traceback (most recent call last): + File "", line 1, in + File "", line 1 + ( + ^ + SyntaxError: parenthesis is never closed + >>>> + + +What I have now improved is the case of parenthesis that are matched wrongly: + +.. sourcecode:: pycon + + >>>> # before + >>>> (1, + .... 2, + .... ] + File "", line 3 + ] + ^ + SyntaxError: invalid syntax + >>>> + + >>>> # after + >>>> (1, + .... 2, + .... ] + File "", line 3 + ] + ^ + SyntaxError: closing parenthesis ']' does not match opening parenthesis '(' on line 1 + >>>> + + +Conclusion +++++++++++++++ + +Obviously these are just some very simple cases, and there is still a lot of +room for improvement (one huge problem is that only a simple ``SyntaxError`` is +ever shown per parse attempt, but fixing that is rather hard). + +If you have a favorite unhelpful SyntaxError message you love to hate, please +tell us in the comments and we might try to improve it. Other kinds of bad error +messages are also always welcome! From pypy.commits at gmail.com Tue Apr 10 04:47:24 2018 From: pypy.commits at gmail.com (antocuni) Date: Tue, 10 Apr 2018 01:47:24 -0700 (PDT) Subject: [pypy-commit] pypy gc-hooks: WIP: compute the duration of gc-minor and gc-collect-step, and pass it to the GC hooks; for now we simply pass the difference between two read_timestamp(), but ideally we need a way to get a reliable way to convert them to nanoseconds or so Message-ID: <5acc7a1c.cba6df0a.36356.64a2@mx.google.com> Author: Antonio Cuni Branch: gc-hooks Changeset: r94299:98f9de10e075 Date: 2018-04-10 00:10 +0200 http://bitbucket.org/pypy/pypy/changeset/98f9de10e075/ Log: WIP: compute the duration of gc-minor and gc-collect-step, and pass it to the GC hooks; for now we simply pass the difference between two read_timestamp(), but ideally we need a way to get a reliable way to convert them to nanoseconds or so diff --git a/rpython/memory/gc/hook.py b/rpython/memory/gc/hook.py --- a/rpython/memory/gc/hook.py +++ b/rpython/memory/gc/hook.py @@ -21,12 +21,12 @@ def is_gc_collect_enabled(self): return False - def on_gc_minor(self, total_memory_used, pinned_objects): + def on_gc_minor(self, duration, total_memory_used, pinned_objects): """ Called after a minor collection """ - def on_gc_collect_step(self, oldstate, newstate): + def on_gc_collect_step(self, duration, oldstate, newstate): """ Called after each individual step of a major collection, in case the GC is incremental. @@ -48,14 +48,14 @@ # overridden @rgc.no_collect - def fire_gc_minor(self, total_memory_used, pinned_objects): + def fire_gc_minor(self, duration, total_memory_used, pinned_objects): if self.is_gc_minor_enabled(): - self.on_gc_minor(total_memory_used, pinned_objects) + self.on_gc_minor(duration, total_memory_used, pinned_objects) @rgc.no_collect - def fire_gc_collect_step(self, oldstate, newstate): + def fire_gc_collect_step(self, duration, oldstate, newstate): if self.is_gc_collect_step_enabled(): - self.on_gc_collect_step(oldstate, newstate) + self.on_gc_collect_step(duration, oldstate, newstate) @rgc.no_collect def fire_gc_collect(self, count, arenas_count_before, arenas_count_after, diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -73,6 +73,7 @@ from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop from rpython.rlib.objectmodel import specialize from rpython.rlib import rgc +from rpython.rlib.rtimer import read_timestamp from rpython.memory.gc.minimarkpage import out_of_memory # @@ -1641,6 +1642,7 @@ """Perform a minor collection: find the objects from the nursery that remain alive and move them out.""" # + start = read_timestamp() debug_start("gc-minor") # # All nursery barriers are invalid from this point on. They @@ -1838,8 +1840,11 @@ self.root_walker.finished_minor_collection() # debug_stop("gc-minor") - self.hooks.fire_gc_minor(total_memory_used=total_memory_used, - pinned_objects=self.pinned_objects_in_nursery) + duration = read_timestamp() - start + self.hooks.fire_gc_minor( + duration=duration, + total_memory_used=total_memory_used, + pinned_objects=self.pinned_objects_in_nursery) def _reset_flag_old_objects_pointing_to_pinned(self, obj, ignore): ll_assert(self.header(obj).tid & GCFLAG_PINNED_OBJECT_PARENT_KNOWN != 0, @@ -2242,6 +2247,7 @@ # Note - minor collections seem fast enough so that one # is done before every major collection step def major_collection_step(self, reserving_size=0): + start = read_timestamp() debug_start("gc-collect-step") oldstate = self.gc_state debug_print("starting gc state: ", GC_STATES[self.gc_state]) @@ -2481,8 +2487,11 @@ debug_print("stopping, now in gc state: ", GC_STATES[self.gc_state]) debug_stop("gc-collect-step") - self.hooks.fire_gc_collect_step(oldstate=oldstate, - newstate=self.gc_state) + duration = read_timestamp() - start + self.hooks.fire_gc_collect_step( + duration=duration, + oldstate=oldstate, + newstate=self.gc_state) def _sweep_old_objects_pointing_to_pinned(self, obj, new_list): if self.header(obj).tid & GCFLAG_VISITED: diff --git a/rpython/memory/gc/test/test_hook.py b/rpython/memory/gc/test/test_hook.py --- a/rpython/memory/gc/test/test_hook.py +++ b/rpython/memory/gc/test/test_hook.py @@ -25,13 +25,16 @@ self.minors = [] self.steps = [] self.collects = [] + self.durations = [] - def on_gc_minor(self, total_memory_used, pinned_objects): + def on_gc_minor(self, duration, total_memory_used, pinned_objects): + self.durations.append(duration) self.minors.append({ 'total_memory_used': total_memory_used, 'pinned_objects': pinned_objects}) - def on_gc_collect_step(self, oldstate, newstate): + def on_gc_collect_step(self, duration, oldstate, newstate): + self.durations.append(duration) self.steps.append({ 'oldstate': oldstate, 'newstate': newstate}) @@ -66,6 +69,7 @@ assert self.gc.hooks.minors == [ {'total_memory_used': 0, 'pinned_objects': 0} ] + assert self.gc.hooks.durations[0] > 0 self.gc.hooks.reset() # # these objects survive, so the total_memory_used is > 0 @@ -96,6 +100,9 @@ 'rawmalloc_bytes_after': 0, 'rawmalloc_bytes_before': 0} ] + assert len(self.gc.hooks.durations) == 4 # 4 steps + for d in self.gc.hooks.durations: + assert d > 0 self.gc.hooks.reset() # self.stackroots.append(self.malloc(S)) diff --git a/rpython/memory/test/test_transformed_gc.py b/rpython/memory/test/test_transformed_gc.py --- a/rpython/memory/test/test_transformed_gc.py +++ b/rpython/memory/test/test_transformed_gc.py @@ -1421,10 +1421,10 @@ def is_gc_collect_enabled(self): return True - def on_gc_minor(self, total_memory_used, pinned_objects): + def on_gc_minor(self, duration, total_memory_used, pinned_objects): self.stats.minors += 1 - def on_gc_collect_step(self, oldstate, newstate): + def on_gc_collect_step(self, duration, oldstate, newstate): self.stats.steps += 1 def on_gc_collect(self, count, arenas_count_before, arenas_count_after, From pypy.commits at gmail.com Tue Apr 10 04:47:28 2018 From: pypy.commits at gmail.com (antocuni) Date: Tue, 10 Apr 2018 01:47:28 -0700 (PDT) Subject: [pypy-commit] pypy gc-hooks: fix the app-level hooks to use the new 'duration' param Message-ID: <5acc7a20.e8361c0a.7ce9b.c37d@mx.google.com> Author: Antonio Cuni Branch: gc-hooks Changeset: r94300:b41028f24d1e Date: 2018-04-10 00:18 +0200 http://bitbucket.org/pypy/pypy/changeset/b41028f24d1e/ Log: fix the app-level hooks to use the new 'duration' param diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py --- a/pypy/module/gc/hook.py +++ b/pypy/module/gc/hook.py @@ -31,14 +31,16 @@ def is_gc_collect_enabled(self): return self.w_hooks.gc_collect_enabled - def on_gc_minor(self, total_memory_used, pinned_objects): + def on_gc_minor(self, duration, total_memory_used, pinned_objects): action = self.w_hooks.gc_minor + action.duration = duration action.total_memory_used = total_memory_used action.pinned_objects = pinned_objects action.fire() - def on_gc_collect_step(self, oldstate, newstate): + def on_gc_collect_step(self, duration, oldstate, newstate): action = self.w_hooks.gc_collect_step + action.duration = duration action.oldstate = oldstate action.newstate = newstate action.fire() @@ -106,6 +108,7 @@ class GcMinorHookAction(AsyncAction): + duration = 0 total_memory_used = 0 pinned_objects = 0 @@ -118,16 +121,21 @@ # BEFORE we do the gc transform; this makes sure that everything is # annotated with the correct types if NonConstant(False): + self.duration = NonConstant(-42) self.total_memory_used = NonConstant(r_uint(42)) self.pinned_objects = NonConstant(-42) self.fire() def perform(self, ec, frame): - w_stats = W_GcMinorStats(self.total_memory_used, self.pinned_objects) + w_stats = W_GcMinorStats( + self.duration, + self.total_memory_used, + self.pinned_objects) self.space.call_function(self.w_callable, w_stats) class GcCollectStepHookAction(AsyncAction): + duration = 0 oldstate = 0 newstate = 0 @@ -140,12 +148,16 @@ # BEFORE we do the gc transform; this makes sure that everything is # annotated with the correct types if NonConstant(False): + self.duration = NonConstant(-42) self.oldstate = NonConstant(-42) self.newstate = NonConstant(-42) self.fire() def perform(self, ec, frame): - w_stats = W_GcCollectStepStats(self.oldstate, self.newstate) + w_stats = W_GcCollectStepStats( + self.duration, + self.oldstate, + self.newstate) self.space.call_function(self.w_callable, w_stats) @@ -186,14 +198,16 @@ class W_GcMinorStats(W_Root): - def __init__(self, total_memory_used, pinned_objects): + def __init__(self, duration, total_memory_used, pinned_objects): + self.duration = duration self.total_memory_used = total_memory_used self.pinned_objects = pinned_objects class W_GcCollectStepStats(W_Root): - def __init__(self, oldstate, newstate): + def __init__(self, duration, oldstate, newstate): + self.duration = duration self.oldstate = oldstate self.newstate = newstate @@ -239,6 +253,7 @@ W_GcMinorStats.typedef = TypeDef( "GcMinorStats", **wrap_many_ints(W_GcMinorStats, ( + "duration", "total_memory_used", "pinned_objects")) ) @@ -251,6 +266,7 @@ STATE_FINALIZING = incminimark.STATE_FINALIZING, GC_STATES = tuple(incminimark.GC_STATES), **wrap_many_ints(W_GcCollectStepStats, ( + "duration", "oldstate", "newstate")) ) diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py --- a/pypy/module/gc/test/test_hook.py +++ b/pypy/module/gc/test/test_hook.py @@ -12,13 +12,13 @@ space = cls.space gchooks = space.fromcache(LowLevelGcHooks) - @unwrap_spec(ObjSpace, r_uint, int) - def fire_gc_minor(space, total_memory_used, pinned_objects): - gchooks.fire_gc_minor(total_memory_used, pinned_objects) + @unwrap_spec(ObjSpace, int, r_uint, int) + def fire_gc_minor(space, duration, total_memory_used, pinned_objects): + gchooks.fire_gc_minor(duration, total_memory_used, pinned_objects) - @unwrap_spec(ObjSpace, int, int) - def fire_gc_collect_step(space, oldstate, newstate): - gchooks.fire_gc_collect_step(oldstate, newstate) + @unwrap_spec(ObjSpace, int, int, int) + def fire_gc_collect_step(space, duration, oldstate, newstate): + gchooks.fire_gc_collect_step(duration, oldstate, newstate) @unwrap_spec(ObjSpace, int, int, int, r_uint, r_uint, r_uint) def fire_gc_collect(space, a, b, c, d, e, f): @@ -26,8 +26,8 @@ @unwrap_spec(ObjSpace) def fire_many(space): - gchooks.fire_gc_minor(0, 0) - gchooks.fire_gc_collect_step(0, 0) + gchooks.fire_gc_minor(0, 0, 0) + gchooks.fire_gc_collect_step(0, 0, 0) gchooks.fire_gc_collect(1, 2, 3, 4, 5, 6) cls.w_fire_gc_minor = space.wrap(interp2app(fire_gc_minor)) @@ -45,40 +45,42 @@ import gc lst = [] def on_gc_minor(stats): - lst.append((stats.total_memory_used, stats.pinned_objects)) + tup = (stats.duration, stats.total_memory_used, stats.pinned_objects) + lst.append(tup) gc.hooks.on_gc_minor = on_gc_minor - self.fire_gc_minor(10, 20) - self.fire_gc_minor(30, 40) + self.fire_gc_minor(10, 20, 30) + self.fire_gc_minor(40, 50, 60) assert lst == [ - (10, 20), - (30, 40), + (10, 20, 30), + (40, 50, 60), ] # gc.hooks.on_gc_minor = None - self.fire_gc_minor(50, 60) # won't fire because the hooks is disabled + self.fire_gc_minor(70, 80, 90) # won't fire because the hooks is disabled assert lst == [ - (10, 20), - (30, 40), + (10, 20, 30), + (40, 50, 60), ] def test_on_gc_collect_step(self): import gc lst = [] def on_gc_collect_step(stats): - lst.append((stats.oldstate, stats.newstate)) + tup = (stats.duration, stats.oldstate, stats.newstate) + lst.append(tup) gc.hooks.on_gc_collect_step = on_gc_collect_step - self.fire_gc_collect_step(10, 20) - self.fire_gc_collect_step(30, 40) + self.fire_gc_collect_step(10, 20, 30) + self.fire_gc_collect_step(40, 50, 60) assert lst == [ - (10, 20), - (30, 40), + (10, 20, 30), + (40, 50, 60), ] # gc.hooks.on_gc_collect_step = None - self.fire_gc_collect_step(50, 60) # won't fire + self.fire_gc_collect_step(70, 80, 90) # won't fire assert lst == [ - (10, 20), - (30, 40), + (10, 20, 30), + (40, 50, 60), ] def test_on_gc_collect(self): @@ -136,7 +138,7 @@ self.fire_many() assert myhooks.lst == ['minor', 'step', 'collect'] myhooks.lst[:] = [] - self.fire_gc_minor(0, 0) + self.fire_gc_minor(0, 0, 0) assert myhooks.lst == ['minor'] gc.hooks.reset() assert gc.hooks.on_gc_minor is None From pypy.commits at gmail.com Tue Apr 10 04:47:30 2018 From: pypy.commits at gmail.com (antocuni) Date: Tue, 10 Apr 2018 01:47:30 -0700 (PDT) Subject: [pypy-commit] pypy gc-hooks: rename the 'count' argument of on_gc_collect to match the name used in the source code Message-ID: <5acc7a22.07711c0a.97f27.3d3c@mx.google.com> Author: Antonio Cuni Branch: gc-hooks Changeset: r94301:09cb3ab59fa7 Date: 2018-04-10 00:25 +0200 http://bitbucket.org/pypy/pypy/changeset/09cb3ab59fa7/ Log: rename the 'count' argument of on_gc_collect to match the name used in the source code diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py --- a/pypy/module/gc/hook.py +++ b/pypy/module/gc/hook.py @@ -45,11 +45,12 @@ action.newstate = newstate action.fire() - def on_gc_collect(self, count, arenas_count_before, arenas_count_after, + def on_gc_collect(self, num_major_collects, + arenas_count_before, arenas_count_after, arenas_bytes, rawmalloc_bytes_before, rawmalloc_bytes_after): action = self.w_hooks.gc_collect - action.count = count + action.num_major_collects = num_major_collects action.arenas_count_before = arenas_count_before action.arenas_count_after = arenas_count_after action.arenas_bytes = arenas_bytes @@ -162,7 +163,7 @@ class GcCollectHookAction(AsyncAction): - count = 0 + num_major_collects = 0 arenas_count_before = 0 arenas_count_after = 0 arenas_bytes = 0 @@ -178,7 +179,7 @@ # BEFORE we do the gc transform; this makes sure that everything is # annotated with the correct types if NonConstant(False): - self.count = NonConstant(-42) + self.num_major_collects = NonConstant(-42) self.arenas_count_before = NonConstant(-42) self.arenas_count_after = NonConstant(-42) self.arenas_bytes = NonConstant(r_uint(42)) @@ -187,7 +188,7 @@ self.fire() def perform(self, ec, frame): - w_stats = W_GcCollectStats(self.count, + w_stats = W_GcCollectStats(self.num_major_collects, self.arenas_count_before, self.arenas_count_after, self.arenas_bytes, @@ -213,10 +214,11 @@ class W_GcCollectStats(W_Root): - def __init__(self, count, arenas_count_before, arenas_count_after, + def __init__(self, num_major_collects, + arenas_count_before, arenas_count_after, arenas_bytes, rawmalloc_bytes_before, rawmalloc_bytes_after): - self.count = count + self.num_major_collects = num_major_collects self.arenas_count_before = arenas_count_before self.arenas_count_after = arenas_count_after self.arenas_bytes = arenas_bytes @@ -274,7 +276,7 @@ W_GcCollectStats.typedef = TypeDef( "GcCollectStats", **wrap_many_ints(W_GcCollectStats, ( - "count", + "num_major_collects", "arenas_count_before", "arenas_count_after", "arenas_bytes", diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py --- a/pypy/module/gc/test/test_hook.py +++ b/pypy/module/gc/test/test_hook.py @@ -87,7 +87,7 @@ import gc lst = [] def on_gc_collect(stats): - lst.append((stats.count, + lst.append((stats.num_major_collects, stats.arenas_count_before, stats.arenas_count_after, stats.arenas_bytes, diff --git a/rpython/memory/gc/hook.py b/rpython/memory/gc/hook.py --- a/rpython/memory/gc/hook.py +++ b/rpython/memory/gc/hook.py @@ -37,7 +37,8 @@ """ - def on_gc_collect(self, count, arenas_count_before, arenas_count_after, + def on_gc_collect(self, num_major_collects, + arenas_count_before, arenas_count_after, arenas_bytes, rawmalloc_bytes_before, rawmalloc_bytes_after): """ @@ -58,10 +59,12 @@ self.on_gc_collect_step(duration, oldstate, newstate) @rgc.no_collect - def fire_gc_collect(self, count, arenas_count_before, arenas_count_after, + def fire_gc_collect(self, num_major_collects, + arenas_count_before, arenas_count_after, arenas_bytes, rawmalloc_bytes_before, rawmalloc_bytes_after): if self.is_gc_collect_enabled(): - self.on_gc_collect(count, arenas_count_before, arenas_count_after, + self.on_gc_collect(num_major_collects, + arenas_count_before, arenas_count_after, arenas_bytes, rawmalloc_bytes_before, rawmalloc_bytes_after) diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -2430,7 +2430,7 @@ self.rawmalloced_total_size) debug_stop("gc-collect-done") self.hooks.fire_gc_collect( - count=self.num_major_collects, + num_major_collects=self.num_major_collects, arenas_count_before=self.stat_ac_arenas_count, arenas_count_after=self.ac.arenas_count, arenas_bytes=self.ac.total_memory_used, diff --git a/rpython/memory/gc/test/test_hook.py b/rpython/memory/gc/test/test_hook.py --- a/rpython/memory/gc/test/test_hook.py +++ b/rpython/memory/gc/test/test_hook.py @@ -39,11 +39,12 @@ 'oldstate': oldstate, 'newstate': newstate}) - def on_gc_collect(self, count, arenas_count_before, arenas_count_after, + def on_gc_collect(self, num_major_collects, + arenas_count_before, arenas_count_after, arenas_bytes, rawmalloc_bytes_before, rawmalloc_bytes_after): self.collects.append({ - 'count': count, + 'num_major_collects': num_major_collects, 'arenas_count_before': arenas_count_before, 'arenas_count_after': arenas_count_after, 'arenas_bytes': arenas_bytes, @@ -93,7 +94,7 @@ {'oldstate': m.STATE_FINALIZING, 'newstate': m.STATE_SCANNING} ] assert self.gc.hooks.collects == [ - {'count': 1, + {'num_major_collects': 1, 'arenas_count_before': 0, 'arenas_count_after': 0, 'arenas_bytes': 0, @@ -108,7 +109,7 @@ self.stackroots.append(self.malloc(S)) self.gc.collect() assert self.gc.hooks.collects == [ - {'count': 2, + {'num_major_collects': 2, 'arenas_count_before': 1, 'arenas_count_after': 1, 'arenas_bytes': self.size_of_S, diff --git a/rpython/memory/test/test_transformed_gc.py b/rpython/memory/test/test_transformed_gc.py --- a/rpython/memory/test/test_transformed_gc.py +++ b/rpython/memory/test/test_transformed_gc.py @@ -1427,7 +1427,8 @@ def on_gc_collect_step(self, duration, oldstate, newstate): self.stats.steps += 1 - def on_gc_collect(self, count, arenas_count_before, arenas_count_after, + def on_gc_collect(self, num_major_collects, + arenas_count_before, arenas_count_after, arenas_bytes, rawmalloc_bytes_before, rawmalloc_bytes_after): self.stats.collects += 1 From pypy.commits at gmail.com Tue Apr 10 04:47:32 2018 From: pypy.commits at gmail.com (antocuni) Date: Tue, 10 Apr 2018 01:47:32 -0700 (PDT) Subject: [pypy-commit] pypy gc-hooks: add a count stat which counts how many events have been fired since the last time the hook was called; also, make duration cumulative, so that we can know the total time spent in each event since the last hook was called Message-ID: <5acc7a24.c3b0df0a.c9b14.2b1a@mx.google.com> Author: Antonio Cuni Branch: gc-hooks Changeset: r94302:ada614ebe930 Date: 2018-04-10 10:43 +0200 http://bitbucket.org/pypy/pypy/changeset/ada614ebe930/ Log: add a count stat which counts how many events have been fired since the last time the hook was called; also, make duration cumulative, so that we can know the total time spent in each event since the last hook was called diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py --- a/pypy/module/gc/hook.py +++ b/pypy/module/gc/hook.py @@ -33,14 +33,16 @@ def on_gc_minor(self, duration, total_memory_used, pinned_objects): action = self.w_hooks.gc_minor - action.duration = duration + action.count += 1 + action.duration += duration action.total_memory_used = total_memory_used action.pinned_objects = pinned_objects action.fire() def on_gc_collect_step(self, duration, oldstate, newstate): action = self.w_hooks.gc_collect_step - action.duration = duration + action.count += 1 + action.duration += duration action.oldstate = oldstate action.newstate = newstate action.fire() @@ -50,6 +52,7 @@ arenas_bytes, rawmalloc_bytes_before, rawmalloc_bytes_after): action = self.w_hooks.gc_collect + action.count += 1 action.num_major_collects = num_major_collects action.arenas_count_before = arenas_count_before action.arenas_count_after = arenas_count_after @@ -109,6 +112,7 @@ class GcMinorHookAction(AsyncAction): + count = 0 duration = 0 total_memory_used = 0 pinned_objects = 0 @@ -117,11 +121,16 @@ AsyncAction.__init__(self, space) self.w_callable = space.w_None + def reset(self): + self.count = 0 + self.duration = 0 + def fix_annotation(self): # the annotation of the class and its attributes must be completed # BEFORE we do the gc transform; this makes sure that everything is # annotated with the correct types if NonConstant(False): + self.count = NonConstant(-42) self.duration = NonConstant(-42) self.total_memory_used = NonConstant(r_uint(42)) self.pinned_objects = NonConstant(-42) @@ -129,13 +138,16 @@ def perform(self, ec, frame): w_stats = W_GcMinorStats( + self.count, self.duration, self.total_memory_used, self.pinned_objects) + self.reset() self.space.call_function(self.w_callable, w_stats) class GcCollectStepHookAction(AsyncAction): + count = 0 duration = 0 oldstate = 0 newstate = 0 @@ -144,11 +156,16 @@ AsyncAction.__init__(self, space) self.w_callable = space.w_None + def reset(self): + self.count = 0 + self.duration = 0 + def fix_annotation(self): # the annotation of the class and its attributes must be completed # BEFORE we do the gc transform; this makes sure that everything is # annotated with the correct types if NonConstant(False): + self.count = NonConstant(-42) self.duration = NonConstant(-42) self.oldstate = NonConstant(-42) self.newstate = NonConstant(-42) @@ -156,13 +173,16 @@ def perform(self, ec, frame): w_stats = W_GcCollectStepStats( + self.count, self.duration, self.oldstate, self.newstate) + self.reset() self.space.call_function(self.w_callable, w_stats) class GcCollectHookAction(AsyncAction): + count = 0 num_major_collects = 0 arenas_count_before = 0 arenas_count_after = 0 @@ -174,11 +194,16 @@ AsyncAction.__init__(self, space) self.w_callable = space.w_None + def reset(self): + self.count = 0 + self.duration = 0 + def fix_annotation(self): # the annotation of the class and its attributes must be completed # BEFORE we do the gc transform; this makes sure that everything is # annotated with the correct types if NonConstant(False): + self.count = NonConstant(-42) self.num_major_collects = NonConstant(-42) self.arenas_count_before = NonConstant(-42) self.arenas_count_after = NonConstant(-42) @@ -188,18 +213,21 @@ self.fire() def perform(self, ec, frame): - w_stats = W_GcCollectStats(self.num_major_collects, + w_stats = W_GcCollectStats(self.count, + self.num_major_collects, self.arenas_count_before, self.arenas_count_after, self.arenas_bytes, self.rawmalloc_bytes_before, self.rawmalloc_bytes_after) + self.reset() self.space.call_function(self.w_callable, w_stats) class W_GcMinorStats(W_Root): - def __init__(self, duration, total_memory_used, pinned_objects): + def __init__(self, count, duration, total_memory_used, pinned_objects): + self.count = count self.duration = duration self.total_memory_used = total_memory_used self.pinned_objects = pinned_objects @@ -207,17 +235,19 @@ class W_GcCollectStepStats(W_Root): - def __init__(self, duration, oldstate, newstate): + def __init__(self, count, duration, oldstate, newstate): + self.count = count self.duration = duration self.oldstate = oldstate self.newstate = newstate class W_GcCollectStats(W_Root): - def __init__(self, num_major_collects, + def __init__(self, count, num_major_collects, arenas_count_before, arenas_count_after, arenas_bytes, rawmalloc_bytes_before, rawmalloc_bytes_after): + self.count = count self.num_major_collects = num_major_collects self.arenas_count_before = arenas_count_before self.arenas_count_after = arenas_count_after @@ -255,6 +285,7 @@ W_GcMinorStats.typedef = TypeDef( "GcMinorStats", **wrap_many_ints(W_GcMinorStats, ( + "count", "duration", "total_memory_used", "pinned_objects")) @@ -268,6 +299,7 @@ STATE_FINALIZING = incminimark.STATE_FINALIZING, GC_STATES = tuple(incminimark.GC_STATES), **wrap_many_ints(W_GcCollectStepStats, ( + "count", "duration", "oldstate", "newstate")) @@ -276,6 +308,7 @@ W_GcCollectStats.typedef = TypeDef( "GcCollectStats", **wrap_many_ints(W_GcCollectStats, ( + "count", "num_major_collects", "arenas_count_before", "arenas_count_after", diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py --- a/pypy/module/gc/test/test_hook.py +++ b/pypy/module/gc/test/test_hook.py @@ -26,8 +26,11 @@ @unwrap_spec(ObjSpace) def fire_many(space): - gchooks.fire_gc_minor(0, 0, 0) - gchooks.fire_gc_collect_step(0, 0, 0) + gchooks.fire_gc_minor(5, 0, 0) + gchooks.fire_gc_minor(7, 0, 0) + gchooks.fire_gc_collect_step(5, 0, 0) + gchooks.fire_gc_collect_step(15, 0, 0) + gchooks.fire_gc_collect_step(22, 0, 0) gchooks.fire_gc_collect(1, 2, 3, 4, 5, 6) cls.w_fire_gc_minor = space.wrap(interp2app(fire_gc_minor)) @@ -45,49 +48,54 @@ import gc lst = [] def on_gc_minor(stats): - tup = (stats.duration, stats.total_memory_used, stats.pinned_objects) - lst.append(tup) + lst.append((stats.count, + stats.duration, + stats.total_memory_used, + stats.pinned_objects)) gc.hooks.on_gc_minor = on_gc_minor self.fire_gc_minor(10, 20, 30) self.fire_gc_minor(40, 50, 60) assert lst == [ - (10, 20, 30), - (40, 50, 60), + (1, 10, 20, 30), + (1, 40, 50, 60), ] # gc.hooks.on_gc_minor = None self.fire_gc_minor(70, 80, 90) # won't fire because the hooks is disabled assert lst == [ - (10, 20, 30), - (40, 50, 60), + (1, 10, 20, 30), + (1, 40, 50, 60), ] def test_on_gc_collect_step(self): import gc lst = [] def on_gc_collect_step(stats): - tup = (stats.duration, stats.oldstate, stats.newstate) - lst.append(tup) + lst.append((stats.count, + stats.duration, + stats.oldstate, + stats.newstate)) gc.hooks.on_gc_collect_step = on_gc_collect_step self.fire_gc_collect_step(10, 20, 30) self.fire_gc_collect_step(40, 50, 60) assert lst == [ - (10, 20, 30), - (40, 50, 60), + (1, 10, 20, 30), + (1, 40, 50, 60), ] # gc.hooks.on_gc_collect_step = None self.fire_gc_collect_step(70, 80, 90) # won't fire assert lst == [ - (10, 20, 30), - (40, 50, 60), + (1, 10, 20, 30), + (1, 40, 50, 60), ] def test_on_gc_collect(self): import gc lst = [] def on_gc_collect(stats): - lst.append((stats.num_major_collects, + lst.append((stats.count, + stats.num_major_collects, stats.arenas_count_before, stats.arenas_count_after, stats.arenas_bytes, @@ -97,15 +105,15 @@ self.fire_gc_collect(1, 2, 3, 4, 5, 6) self.fire_gc_collect(7, 8, 9, 10, 11, 12) assert lst == [ - (1, 2, 3, 4, 5, 6), - (7, 8, 9, 10, 11, 12), + (1, 1, 2, 3, 4, 5, 6), + (1, 7, 8, 9, 10, 11, 12), ] # gc.hooks.on_gc_collect = None self.fire_gc_collect(42, 42, 42, 42, 42, 42) # won't fire assert lst == [ - (1, 2, 3, 4, 5, 6), - (7, 8, 9, 10, 11, 12), + (1, 1, 2, 3, 4, 5, 6), + (1, 7, 8, 9, 10, 11, 12), ] def test_consts(self): @@ -117,6 +125,28 @@ assert S.STATE_FINALIZING == 3 assert S.GC_STATES == ('SCANNING', 'MARKING', 'SWEEPING', 'FINALIZING') + def test_cumulative(self): + import gc + class MyHooks(object): + + def __init__(self): + self.minors = [] + self.steps = [] + + def on_gc_minor(self, stats): + self.minors.append((stats.count, stats.duration)) + + def on_gc_collect_step(self, stats): + self.steps.append((stats.count, stats.duration)) + + on_gc_collect = None + + myhooks = MyHooks() + gc.hooks.set(myhooks) + self.fire_many() + assert myhooks.minors == [(2, 12)] + assert myhooks.steps == [(3, 42)] + def test_clear_queue(self): import gc class MyHooks(object): From pypy.commits at gmail.com Tue Apr 10 05:00:26 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 10 Apr 2018 02:00:26 -0700 (PDT) Subject: [pypy-commit] extradoc extradoc: editing Message-ID: <5acc7d2a.43b8df0a.374e5.c301@mx.google.com> Author: Matti Picus Branch: extradoc Changeset: r5888:fc88795012eb Date: 2018-04-10 11:59 +0300 http://bitbucket.org/pypy/extradoc/changeset/fc88795012eb/ Log: editing diff --git a/blog/draft/2018-04-better-syntaxerrors.rst b/blog/draft/2018-04-better-syntaxerrors.rst --- a/blog/draft/2018-04-better-syntaxerrors.rst +++ b/blog/draft/2018-04-better-syntaxerrors.rst @@ -1,26 +1,25 @@ Improving SyntaxError in PyPy ============================== -Since about a year, my halftime job at the uni has been to teach non-CS students +For the last year, my halftime job has been to teach non-CS uni students to program in Python. While doing that, I have been trying to see what common -stumbling blocks are, for programmers that are just starting out. There are many -things that could be said here, but one theme that emerges are +stumbling blocks exist for novice programmers. There are many +things that could be said here, but a common theme that emerges is hard-to-understand error messages. One source of such error messages, -particularly when starting out, are ``SyntaxErrors``. - +particularly when starting out, is ``SyntaxErrors``. PyPy's parser (mostly following the architecture of CPython) uses a regular-expression-based tokenizer with some cleverness to deal with indentation, and a simple LR(1) parser. Both of these components obviously -produce errors for invalid syntax, but the messages are not that great. Often, +produce errors for invalid syntax, but the messages are not very helpful. Often, the message is just "invalid syntax", without any hint of what exactly is wrong. -In the last couple of weeks I have invested a little bit of effor to make them a +In the last couple of weeks I have invested a little bit of effort to make them a tiny bit better. Here are some examples. Missing Characters -++++++++++++++++++++ +++++++++++++++++++ -The first class of errors is when a token is missing, but there is only one +The first class of errors occurs when a token is missing, often there is only one valid token that the parser expects. This happens most commonly by leaving out the ':' after control flow statements (which is the syntax error I personally still make at least a few times a day). In such situations, the parser will now @@ -65,10 +64,10 @@ >>>> -Parenthesis -++++++++++++++++++++++ +Parentheses ++++++++++++ -Another source of errors are unmatched parenthesis. Here, PyPy has always had +Another source of errors are unmatched parentheses. Here, PyPy has always had slightly better error messages than CPython: .. sourcecode:: pycon @@ -89,7 +88,7 @@ SyntaxError: unmatched ')' >>>> -The same is true for parenthesis that are never closed (the call to ``eval`` is +The same is true for parentheses that are never closed (the call to ``eval`` is needed to get the error, otherwise the repl will just wait for more input): .. sourcecode:: pycon @@ -141,12 +140,12 @@ Conclusion -++++++++++++++ +++++++++++ Obviously these are just some very simple cases, and there is still a lot of room for improvement (one huge problem is that only a simple ``SyntaxError`` is ever shown per parse attempt, but fixing that is rather hard). If you have a favorite unhelpful SyntaxError message you love to hate, please -tell us in the comments and we might try to improve it. Other kinds of bad error -messages are also always welcome! +tell us in the comments and we might try to improve it. Other kinds of +non-informative error messages are also always welcome! From pypy.commits at gmail.com Tue Apr 10 22:56:20 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 10 Apr 2018 19:56:20 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: typo Message-ID: <5acd7954.3d86df0a.136a3.2704@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r94305:695dfb0f493b Date: 2018-04-11 05:55 +0300 http://bitbucket.org/pypy/pypy/changeset/695dfb0f493b/ Log: typo diff --git a/lib-python/3/test/test_genexps.py b/lib-python/3/test/test_genexps.py --- a/lib-python/3/test/test_genexps.py +++ b/lib-python/3/test/test_genexps.py @@ -85,7 +85,7 @@ Verify that parenthesis are required when used as a keyword argument value PyPy has extended sytax error messages, ignore the detail for compatibility - >>> dict(a = i for i in range(10)) # doctest: +IGNORE_EXCPTION_DETAIL + >>> dict(a = i for i in range(10)) # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ... SyntaxError: invalid syntax diff --git a/lib-python/3/test/test_syntax.py b/lib-python/3/test/test_syntax.py --- a/lib-python/3/test/test_syntax.py +++ b/lib-python/3/test/test_syntax.py @@ -109,12 +109,12 @@ PyPy has extended sytax error messages, ignore the detail for compatibility ->>> def f(x, None): # doctest: +IGNORE_EXCPTION_DETAIL +>>> def f(x, None): # doctest: +IGNORE_EXCEPTION_DETAIL ... pass Traceback (most recent call last): SyntaxError: invalid syntax (expected ')') ->>> def f(*None): # doctest: +IGNORE_EXCPTION_DETAIL +>>> def f(*None): # doctest: +IGNORE_EXCEPTION_DETAIL ... pass Traceback (most recent call last): SyntaxError: invalid syntax (expected ')') diff --git a/lib-python/3/test/test_unpack_ex.py b/lib-python/3/test/test_unpack_ex.py --- a/lib-python/3/test/test_unpack_ex.py +++ b/lib-python/3/test/test_unpack_ex.py @@ -161,14 +161,14 @@ Generator expression in function arguments PyPy has extended sytax error messages, ignore the detail for compatibility - >>> list(*x for x in (range(5) for i in range(3))) # doctest: +IGNORE_EXCPTION_DETAIL + >>> list(*x for x in (range(5) for i in range(3))) # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ... list(*x for x in (range(5) for i in range(3))) ^ SyntaxError: invalid syntax - >>> dict(**x for x in [{1:2}]) # doctest: +IGNORE_EXCPTION_DETAIL + >>> dict(**x for x in [{1:2}]) # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ... dict(**x for x in [{1:2}]) From pypy.commits at gmail.com Tue Apr 10 22:56:16 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 10 Apr 2018 19:56:16 -0700 (PDT) Subject: [pypy-commit] pypy default: fix for linux32 (how did it ever pass???) Message-ID: <5acd7950.c3b0df0a.c58f0.09c5@mx.google.com> Author: Matti Picus Branch: Changeset: r94303:db187a245708 Date: 2018-04-11 05:45 +0300 http://bitbucket.org/pypy/pypy/changeset/db187a245708/ Log: fix for linux32 (how did it ever pass???) diff --git a/pypy/module/_rawffi/alt/test/test_struct.py b/pypy/module/_rawffi/alt/test/test_struct.py --- a/pypy/module/_rawffi/alt/test/test_struct.py +++ b/pypy/module/_rawffi/alt/test/test_struct.py @@ -43,7 +43,11 @@ def setup_class(cls): BaseAppTestFFI.setup_class.im_func(cls) - @unwrap_spec(addr=int, typename='text', length=int) + from rpython.rlib import clibffi + from rpython.rlib.rarithmetic import r_uint + from rpython.rtyper.lltypesystem import lltype, rffi + + @unwrap_spec(addr=r_uint, typename='text', length=int) def read_raw_mem(space, addr, typename, length): import ctypes addr = ctypes.cast(addr, ctypes.c_void_p) @@ -58,9 +62,6 @@ else: cls.w_read_raw_mem = cls.space.wrap(interp2app(read_raw_mem)) # - from rpython.rlib import clibffi - from rpython.rlib.rarithmetic import r_uint - from rpython.rtyper.lltypesystem import lltype, rffi dummy_type = lltype.malloc(clibffi.FFI_TYPE_P.TO, flavor='raw') dummy_type.c_size = r_uint(123) dummy_type.c_alignment = rffi.cast(rffi.USHORT, 0) From pypy.commits at gmail.com Tue Apr 10 22:56:18 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 10 Apr 2018 19:56:18 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: fix for linux32 Message-ID: <5acd7952.14721c0a.b43fd.097b@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r94304:242e7ff776ce Date: 2018-04-11 05:52 +0300 http://bitbucket.org/pypy/pypy/changeset/242e7ff776ce/ Log: fix for linux32 diff --git a/pypy/module/_rawffi/alt/test/test_struct.py b/pypy/module/_rawffi/alt/test/test_struct.py --- a/pypy/module/_rawffi/alt/test/test_struct.py +++ b/pypy/module/_rawffi/alt/test/test_struct.py @@ -54,17 +54,18 @@ def setup_class(cls): BaseAppTestFFI.setup_class.im_func(cls) + from rpython.rlib import clibffi + from rpython.rlib.rarithmetic import r_uint + from rpython.rtyper.lltypesystem import lltype, rffi + if cls.runappdirect: cls.w_read_raw_mem = cls.read_raw_mem else: - @unwrap_spec(addr=int, typename='text', length=int) + @unwrap_spec(addr=r_uint, typename='text', length=int) def read_raw_mem_w(space, addr, typename, length): return space.wrap(cls.read_raw_mem(addr, typename, length)) cls.w_read_raw_mem = cls.space.wrap(interp2app(read_raw_mem_w)) # - from rpython.rlib import clibffi - from rpython.rlib.rarithmetic import r_uint - from rpython.rtyper.lltypesystem import lltype, rffi dummy_type = lltype.malloc(clibffi.FFI_TYPE_P.TO, flavor='raw') dummy_type.c_size = r_uint(123) dummy_type.c_alignment = rffi.cast(rffi.USHORT, 0) From pypy.commits at gmail.com Tue Apr 10 23:19:30 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 10 Apr 2018 20:19:30 -0700 (PDT) Subject: [pypy-commit] pypy default: update release notice, prioritize and update cpython differences Message-ID: <5acd7ec2.491a1c0a.a307d.0d6e@mx.google.com> Author: Matti Picus Branch: Changeset: r94306:60c5692d6d40 Date: 2018-04-11 06:17 +0300 http://bitbucket.org/pypy/pypy/changeset/60c5692d6d40/ Log: update release notice, prioritize and update cpython differences diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -10,89 +10,6 @@ PyPy. -.. _extension-modules: - -Extension modules ------------------ - -List of extension modules that we support: - -* Supported as built-in modules (in :source:`pypy/module/`): - - __builtin__ - :doc:`__pypy__ <__pypy__-module>` - _ast - _codecs - _collections - :doc:`_continuation ` - :doc:`_ffi ` - _hashlib - _io - _locale - _lsprof - _md5 - :doc:`_minimal_curses ` - _multiprocessing - _random - :doc:`_rawffi ` - _sha - _socket - _sre - _ssl - _warnings - _weakref - _winreg - array - binascii - bz2 - cStringIO - cmath - `cpyext`_ - crypt - errno - exceptions - fcntl - gc - imp - itertools - marshal - math - mmap - operator - parser - posix - pyexpat - select - signal - struct - symbol - sys - termios - thread - time - token - unicodedata - zipimport - zlib - - When translated on Windows, a few Unix-only modules are skipped, - and the following module is built instead: - - _winreg - -* Supported by being rewritten in pure Python (possibly using ``cffi``): - see the :source:`lib_pypy/` directory. Examples of modules that we - support this way: ``ctypes``, ``cPickle``, ``cmath``, ``dbm``, ``datetime``... - Note that some modules are both in there and in the list above; - by default, the built-in module is used (but can be disabled - at translation time). - -The extension modules (i.e. modules written in C, in the standard CPython) -that are neither mentioned above nor in :source:`lib_pypy/` are not available in PyPy. -(You may have a chance to use them anyway with `cpyext`_.) - -.. _cpyext: http://morepypy.blogspot.com/2010/04/using-cpython-extension-modules-with.html - Differences related to garbage collection strategies ---------------------------------------------------- @@ -559,7 +476,96 @@ environment variable. CPython searches for ``vcvarsall.bat`` somewhere **above** that value. +* SyntaxError_ s try harder to give details about the cause of the failure, so + the error messages are not the same as in CPython + + +.. _extension-modules: + +Extension modules +----------------- + +List of extension modules that we support: + +* Supported as built-in modules (in :source:`pypy/module/`): + + __builtin__ + :doc:`__pypy__ <__pypy__-module>` + _ast + _codecs + _collections + :doc:`_continuation ` + :doc:`_ffi ` + _hashlib + _io + _locale + _lsprof + _md5 + :doc:`_minimal_curses ` + _multiprocessing + _random + :doc:`_rawffi ` + _sha + _socket + _sre + _ssl + _warnings + _weakref + _winreg + array + binascii + bz2 + cStringIO + cmath + `cpyext`_ + crypt + errno + exceptions + fcntl + gc + imp + itertools + marshal + math + mmap + operator + parser + posix + pyexpat + select + signal + struct + symbol + sys + termios + thread + time + token + unicodedata + zipimport + zlib + + When translated on Windows, a few Unix-only modules are skipped, + and the following module is built instead: + + _winreg + +* Supported by being rewritten in pure Python (possibly using ``cffi``): + see the :source:`lib_pypy/` directory. Examples of modules that we + support this way: ``ctypes``, ``cPickle``, ``cmath``, ``dbm``, ``datetime``... + Note that some modules are both in there and in the list above; + by default, the built-in module is used (but can be disabled + at translation time). + +The extension modules (i.e. modules written in C, in the standard CPython) +that are neither mentioned above nor in :source:`lib_pypy/` are not available in PyPy. +(You may have a chance to use them anyway with `cpyext`_.) + +.. _cpyext: http://morepypy.blogspot.com/2010/04/using-cpython-extension-modules-with.html + + .. _`is ignored in PyPy`: http://bugs.python.org/issue14621 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/ .. _`issue #2653`: https://bitbucket.org/pypy/pypy/issues/2653/ +.. _SyntaxError: https://morepypy.blogspot.co.il/2018/04/improving-syntaxerror-in-pypy.html diff --git a/pypy/doc/release-v6.0.0.rst b/pypy/doc/release-v6.0.0.rst --- a/pypy/doc/release-v6.0.0.rst +++ b/pypy/doc/release-v6.0.0.rst @@ -14,10 +14,14 @@ affect the included python development header files, all c-extension modules must be recompiled for this version. +First-time python users are often stumped by silly typos and emissions when +getting started writing code. We have improved our parser to emit more friendly +`syntax errors`_, making PyPy not only faster but more friendly. + The Windows PyPy3.5 release is still considered beta-quality. There are open issues with unicode handling especially around system calls and c-extensions. -The Matplotlib TkAgg backend now works with PyPy, as do pygame and pygobject. +The Matplotlib TkAgg backend now works with PyPy, as do pygame and pygobject_. As always, this release is 100% compatible with the previous one and fixed several issues and bugs raised by the growing community of PyPy users. @@ -47,6 +51,8 @@ .. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly .. _`help`: project-ideas.html .. _`blog post`: https://morepypy.blogspot.it/2017/10/cape-of-good-hope-for-pypy-hello-from.html +.. _pygobject: https://lazka.github.io/posts/2018-04_pypy-pygobject/index.html +.. _`syntax errors`: https://morepypy.blogspot.com/2018/04/improving-syntaxerror-in-pypy.html What is PyPy? ============= From pypy.commits at gmail.com Tue Apr 10 23:19:32 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 10 Apr 2018 20:19:32 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: merge default into branch Message-ID: <5acd7ec4.cdaa1c0a.7aae4.2170@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r94307:e6d85e030f12 Date: 2018-04-11 06:18 +0300 http://bitbucket.org/pypy/pypy/changeset/e6d85e030f12/ Log: merge default into branch diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -10,89 +10,6 @@ PyPy. -.. _extension-modules: - -Extension modules ------------------ - -List of extension modules that we support: - -* Supported as built-in modules (in :source:`pypy/module/`): - - __builtin__ - :doc:`__pypy__ <__pypy__-module>` - _ast - _codecs - _collections - :doc:`_continuation ` - :doc:`_ffi ` - _hashlib - _io - _locale - _lsprof - _md5 - :doc:`_minimal_curses ` - _multiprocessing - _random - :doc:`_rawffi ` - _sha - _socket - _sre - _ssl - _warnings - _weakref - _winreg - array - binascii - bz2 - cStringIO - cmath - `cpyext`_ - crypt - errno - exceptions - fcntl - gc - imp - itertools - marshal - math - mmap - operator - parser - posix - pyexpat - select - signal - struct - symbol - sys - termios - thread - time - token - unicodedata - zipimport - zlib - - When translated on Windows, a few Unix-only modules are skipped, - and the following module is built instead: - - _winreg - -* Supported by being rewritten in pure Python (possibly using ``cffi``): - see the :source:`lib_pypy/` directory. Examples of modules that we - support this way: ``ctypes``, ``cPickle``, ``cmath``, ``dbm``, ``datetime``... - Note that some modules are both in there and in the list above; - by default, the built-in module is used (but can be disabled - at translation time). - -The extension modules (i.e. modules written in C, in the standard CPython) -that are neither mentioned above nor in :source:`lib_pypy/` are not available in PyPy. -(You may have a chance to use them anyway with `cpyext`_.) - -.. _cpyext: http://morepypy.blogspot.com/2010/04/using-cpython-extension-modules-with.html - Differences related to garbage collection strategies ---------------------------------------------------- @@ -572,7 +489,96 @@ environment variable. CPython searches for ``vcvarsall.bat`` somewhere **above** that value. +* SyntaxError_ s try harder to give details about the cause of the failure, so + the error messages are not the same as in CPython + + +.. _extension-modules: + +Extension modules +----------------- + +List of extension modules that we support: + +* Supported as built-in modules (in :source:`pypy/module/`): + + __builtin__ + :doc:`__pypy__ <__pypy__-module>` + _ast + _codecs + _collections + :doc:`_continuation ` + :doc:`_ffi ` + _hashlib + _io + _locale + _lsprof + _md5 + :doc:`_minimal_curses ` + _multiprocessing + _random + :doc:`_rawffi ` + _sha + _socket + _sre + _ssl + _warnings + _weakref + _winreg + array + binascii + bz2 + cStringIO + cmath + `cpyext`_ + crypt + errno + exceptions + fcntl + gc + imp + itertools + marshal + math + mmap + operator + parser + posix + pyexpat + select + signal + struct + symbol + sys + termios + thread + time + token + unicodedata + zipimport + zlib + + When translated on Windows, a few Unix-only modules are skipped, + and the following module is built instead: + + _winreg + +* Supported by being rewritten in pure Python (possibly using ``cffi``): + see the :source:`lib_pypy/` directory. Examples of modules that we + support this way: ``ctypes``, ``cPickle``, ``cmath``, ``dbm``, ``datetime``... + Note that some modules are both in there and in the list above; + by default, the built-in module is used (but can be disabled + at translation time). + +The extension modules (i.e. modules written in C, in the standard CPython) +that are neither mentioned above nor in :source:`lib_pypy/` are not available in PyPy. +(You may have a chance to use them anyway with `cpyext`_.) + +.. _cpyext: http://morepypy.blogspot.com/2010/04/using-cpython-extension-modules-with.html + + .. _`is ignored in PyPy`: http://bugs.python.org/issue14621 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/ .. _`issue #2653`: https://bitbucket.org/pypy/pypy/issues/2653/ +.. _SyntaxError: https://morepypy.blogspot.co.il/2018/04/improving-syntaxerror-in-pypy.html diff --git a/pypy/doc/release-v6.0.0.rst b/pypy/doc/release-v6.0.0.rst --- a/pypy/doc/release-v6.0.0.rst +++ b/pypy/doc/release-v6.0.0.rst @@ -14,10 +14,14 @@ affect the included python development header files, all c-extension modules must be recompiled for this version. +First-time python users are often stumped by silly typos and emissions when +getting started writing code. We have improved our parser to emit more friendly +`syntax errors`_, making PyPy not only faster but more friendly. + The Windows PyPy3.5 release is still considered beta-quality. There are open issues with unicode handling especially around system calls and c-extensions. -The Matplotlib TkAgg backend now works with PyPy, as do pygame and pygobject. +The Matplotlib TkAgg backend now works with PyPy, as do pygame and pygobject_. As always, this release is 100% compatible with the previous one and fixed several issues and bugs raised by the growing community of PyPy users. @@ -47,6 +51,8 @@ .. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly .. _`help`: project-ideas.html .. _`blog post`: https://morepypy.blogspot.it/2017/10/cape-of-good-hope-for-pypy-hello-from.html +.. _pygobject: https://lazka.github.io/posts/2018-04_pypy-pygobject/index.html +.. _`syntax errors`: https://morepypy.blogspot.com/2018/04/improving-syntaxerror-in-pypy.html What is PyPy? ============= From pypy.commits at gmail.com Thu Apr 12 03:34:29 2018 From: pypy.commits at gmail.com (arigo) Date: Thu, 12 Apr 2018 00:34:29 -0700 (PDT) Subject: [pypy-commit] pypy default: Issue #2797 Message-ID: <5acf0c05.8edbdf0a.a15dd.e37d@mx.google.com> Author: Armin Rigo Branch: Changeset: r94308:0dafeea23cbb Date: 2018-04-12 09:33 +0200 http://bitbucket.org/pypy/pypy/changeset/0dafeea23cbb/ Log: Issue #2797 Fix for PySet_Discard() diff --git a/pypy/module/cpyext/setobject.py b/pypy/module/cpyext/setobject.py --- a/pypy/module/cpyext/setobject.py +++ b/pypy/module/cpyext/setobject.py @@ -1,4 +1,4 @@ -from pypy.interpreter.error import oefmt +from pypy.interpreter.error import OperationError, oefmt from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.cpyext.api import (cpython_api, Py_ssize_t, CANNOT_FAIL, build_type_checkers) @@ -64,8 +64,13 @@ instance of set or its subtype.""" if not PySet_Check(space, w_s): PyErr_BadInternalCall(space) - space.call_method(space.w_set, 'discard', w_s, w_obj) - return 0 + try: + space.call_method(space.w_set, 'remove', w_s, w_obj) + except OperationError as e: + if e.match(space, space.w_KeyError): + return 0 + raise + return 1 @cpython_api([PyObject], PyObject) diff --git a/pypy/module/cpyext/test/test_setobject.py b/pypy/module/cpyext/test/test_setobject.py --- a/pypy/module/cpyext/test/test_setobject.py +++ b/pypy/module/cpyext/test/test_setobject.py @@ -28,7 +28,11 @@ assert api.PySet_Size(w_set) == 4 api.PySet_Add(w_set, space.wrap(6)) assert api.PySet_Size(w_set) == 5 - api.PySet_Discard(w_set, space.wrap(6)) + res = api.PySet_Discard(w_set, space.wrap(6)) + assert res == 1 + assert api.PySet_Size(w_set) == 4 + res = api.PySet_Discard(w_set, space.wrap(6)) + assert res == 0 assert api.PySet_Size(w_set) == 4 def test_set_contains(self, space, api): From pypy.commits at gmail.com Thu Apr 12 03:40:11 2018 From: pypy.commits at gmail.com (arigo) Date: Thu, 12 Apr 2018 00:40:11 -0700 (PDT) Subject: [pypy-commit] pypy default: Issue #2768 Message-ID: <5acf0d5b.69b9df0a.bb8f3.6ba5@mx.google.com> Author: Armin Rigo Branch: Changeset: r94309:0cb72dae7486 Date: 2018-04-12 09:39 +0200 http://bitbucket.org/pypy/pypy/changeset/0cb72dae7486/ Log: Issue #2768 Fall back to not using _py3k_acquire(), in case eventlet patches various classes diff --git a/lib-python/2.7/threading.py b/lib-python/2.7/threading.py --- a/lib-python/2.7/threading.py +++ b/lib-python/2.7/threading.py @@ -351,6 +351,21 @@ # forward-compatibility reasons we do the same. waiter.acquire() gotit = True + except AttributeError: + # someone patched the 'waiter' class, probably. + # Fall back to the standard CPython logic. + # See the CPython lib for the comments about it... + endtime = _time() + timeout + delay = 0.0005 # 500 us -> initial delay of 1 ms + while True: + gotit = waiter.acquire(0) + if gotit: + break + remaining = endtime - _time() + if remaining <= 0: + break + delay = min(delay * 2, remaining, .05) + _sleep(delay) else: gotit = waiter.acquire(False) if not gotit: From pypy.commits at gmail.com Thu Apr 12 04:31:16 2018 From: pypy.commits at gmail.com (arigo) Date: Thu, 12 Apr 2018 01:31:16 -0700 (PDT) Subject: [pypy-commit] pypy default: Fix Message-ID: <5acf1954.05f71c0a.2b1fa.8b03@mx.google.com> Author: Armin Rigo Branch: Changeset: r94310:ce764b945eeb Date: 2018-04-12 10:30 +0200 http://bitbucket.org/pypy/pypy/changeset/ce764b945eeb/ Log: Fix diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -644,7 +644,10 @@ def get_rpy_type_index(gcref): from rpython.rlib.rarithmetic import intmask Class = gcref._x.__class__ - return intmask(id(Class)) + i = intmask(id(Class)) + if i < 0: + i = ~i # always return a positive number, at least + return i def cast_gcref_to_int(gcref): # This is meant to be used on cast_instance_to_gcref results. From pypy.commits at gmail.com Thu Apr 12 05:26:17 2018 From: pypy.commits at gmail.com (arigo) Date: Thu, 12 Apr 2018 02:26:17 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Issue #2768: rare case involving __del__ methods Message-ID: <5acf2639.08bedf0a.259ab.8d7d@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r94311:4db4624409b4 Date: 2018-04-12 10:25 +0100 http://bitbucket.org/pypy/pypy/changeset/4db4624409b4/ Log: Issue #2768: rare case involving __del__ methods diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -1098,7 +1098,13 @@ # sub-iterable first before continuing on the next bytecode. from pypy.interpreter.generator import Coroutine in_generator = self.get_generator() - assert in_generator is not None + if in_generator is None: + # Issue #2768: rare case involving __del__ methods. + # XXX This is a workaround, not a proper solution. + raise oefmt(self.space.w_RuntimeError, + "PyPy limitation: cannot use 'yield from' or 'await' " + "in a generator/coroutine that has been partially " + "deallocated already, typically seen via __del__") w_inputvalue = self.popvalue() # that's always w_None, actually w_gen = self.popvalue() # From pypy.commits at gmail.com Thu Apr 12 12:04:04 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 12 Apr 2018 09:04:04 -0700 (PDT) Subject: [pypy-commit] pypy gc-hooks: hg merge default Message-ID: <5acf8374.929cdf0a.3d940.f45d@mx.google.com> Author: Antonio Cuni Branch: gc-hooks Changeset: r94312:f157561218b4 Date: 2018-04-12 18:03 +0200 http://bitbucket.org/pypy/pypy/changeset/f157561218b4/ Log: hg merge default diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -6,36 +6,36 @@ Except when otherwise stated (look for LICENSE files in directories or information at the beginning of each file) all software and documentation in the 'rpython', 'pypy', 'ctype_configure', 'dotviewer', 'demo', 'lib_pypy', -'py', and '_pytest' directories is licensed as follows: +'py', and '_pytest' directories is licensed as follows: The MIT License - Permission is hereby granted, free of charge, to any person - obtaining a copy of this software and associated documentation - files (the "Software"), to deal in the Software without - restriction, including without limitation the rights to use, - copy, modify, merge, publish, distribute, sublicense, and/or - sell copies of the Software, and to permit persons to whom the + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation + files (the "Software"), to deal in the Software without + restriction, including without limitation the rights to use, + copy, modify, merge, publish, distribute, sublicense, and/or + sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included + The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. PyPy Copyright holders 2003-2018 ------------------------------------ +-------------------------------- Except when otherwise stated (look for LICENSE files or information at the beginning of each file) the files in the 'pypy' directory are each -copyrighted by one or more of the following people and organizations: +copyrighted by one or more of the following people and organizations: Armin Rigo Maciej Fijalkowski @@ -89,13 +89,13 @@ Niko Matsakis Alexander Hesse Ludovic Aubry + stian Jacob Hallen Jason Creighton Mark Young Alex Martelli Spenser Bauman Michal Bendowski - stian Jan de Mooij Tyler Wade Vincent Legoll @@ -123,10 +123,10 @@ Wenzhu Man Konstantin Lopuhin John Witulski + Jeremy Thurgood Greg Price Ivan Sichmann Freitas Dario Bertini - Jeremy Thurgood Mark Pearse Simon Cross Tobias Pape @@ -145,18 +145,19 @@ Adrian Kuhn tav Georg Brandl + Joannah Nanjekye Bert Freudenberg Stian Andreassen Wanja Saatkamp Mike Blume - Joannah Nanjekye Gerald Klix Oscar Nierstrasz Rami Chowdhury Stefan H. Muller + Dodan Mihai Tim Felgentreff Eugene Oden - Dodan Mihai + Colin Valliant Jeff Terrace Henry Mason Vasily Kuznetsov @@ -225,12 +226,14 @@ Vaibhav Sood Reuben Cummings Attila Gobi + Floris Bruynooghe Christopher Pope Tristan Arthur Christian Tismer Dan Stromberg Carl Meyer Florin Papa + Arianna Avanzini Jens-Uwe Mager Valentina Mukhamedzhanova Stefano Parmesan @@ -250,9 +253,11 @@ Alejandro J. Cura Vladimir Kryachko Gabriel + Thomas Hisch Mark Williams Kunal Grover Nathan Taylor + Barry Hart Travis Francis Athougies Yasir Suhail Sergey Kishchenko @@ -260,6 +265,7 @@ Lutz Paelike Ian Foote Philipp Rustemeuer + Logan Chien Catalin Gabriel Manciu Jacob Oscarson Ryan Gonzalez @@ -295,7 +301,6 @@ Akira Li Gustavo Niemeyer Rafał Gałczyński - Logan Chien Lucas Stadler roberto at goyle Matt Bogosian @@ -308,6 +313,7 @@ Anna Katrina Dominguez Kim Jin Su Amber Brown + Miro Hrončok Anthony Sottile Nate Bragg Ben Darnell @@ -315,7 +321,6 @@ Godefroid Chappelle Julian Berman Michael Hudson-Doyle - Floris Bruynooghe Stephan Busemann Dan Colish timo @@ -357,6 +362,7 @@ Michael Chermside Anna Ravencroft remarkablerocket + Pauli Virtanen Petre Vijiac Berker Peksag Christian Muirhead @@ -381,6 +387,7 @@ Graham Markall Dan Loewenherz werat + Andrew Stepanov Niclas Olofsson Chris Pressey Tobias Diaz @@ -395,14 +402,14 @@ m at funkyhat.org Stefan Marr - Heinrich-Heine University, Germany + Heinrich-Heine University, Germany Open End AB (formerly AB Strakt), Sweden - merlinux GmbH, Germany - tismerysoft GmbH, Germany - Logilab Paris, France - DFKI GmbH, Germany + merlinux GmbH, Germany + tismerysoft GmbH, Germany + Logilab Paris, France + DFKI GmbH, Germany Impara, Germany - Change Maker, Sweden + Change Maker, Sweden University of California Berkeley, USA Google Inc. King's College London @@ -410,14 +417,14 @@ The PyPy Logo as used by http://speed.pypy.org and others was created by Samuel Reis and is distributed on terms of Creative Commons Share Alike License. - -License for 'lib-python/2.7' -============================ + +License for 'lib-python/2.7, lib-python/3' +========================================== Except when otherwise stated (look for LICENSE files or copyright/license -information at the beginning of each file) the files in the 'lib-python/2.7' +information at the beginning of each file) the files in the 'lib-python' directory are all copyrighted by the Python Software Foundation and licensed -under the terms that you can find here: https://docs.python.org/2/license.html +under the terms that you can find here: https://docs.python.org/3/license.html License for 'pypy/module/unicodedata/' ====================================== @@ -441,9 +448,9 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/lib-python/2.7/test/test_eof.py b/lib-python/2.7/test/test_eof.py --- a/lib-python/2.7/test/test_eof.py +++ b/lib-python/2.7/test/test_eof.py @@ -5,7 +5,7 @@ class EOFTestCase(unittest.TestCase): def test_EOFC(self): - expect = "EOL while scanning string literal (, line 1)" + expect = "end of line (EOL) while scanning string literal (, line 1)" try: eval("""'this is a test\ """) @@ -15,7 +15,7 @@ raise test_support.TestFailed def test_EOFS(self): - expect = ("EOF while scanning triple-quoted string literal " + expect = ("end of file (EOF) while scanning triple-quoted string literal " "(, line 1)") try: eval("""'''this is a test""") diff --git a/lib-python/2.7/test/test_generators.py b/lib-python/2.7/test/test_generators.py --- a/lib-python/2.7/test/test_generators.py +++ b/lib-python/2.7/test/test_generators.py @@ -398,7 +398,10 @@ 0 >>> type(i.gi_frame) ->>> i.gi_running = 42 + +PyPy prints "readonly attribute 'gi_running'" so ignore the exception detail + +>>> i.gi_running = 42 # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ... TypeError: readonly attribute diff --git a/lib-python/2.7/test/test_traceback.py b/lib-python/2.7/test/test_traceback.py --- a/lib-python/2.7/test/test_traceback.py +++ b/lib-python/2.7/test/test_traceback.py @@ -123,10 +123,7 @@ self.assertEqual(len(err), 4) self.assertEqual(err[1].strip(), "print(2)") self.assertIn("^", err[2]) - if check_impl_detail(): - self.assertEqual(err[1].find("p"), err[2].find("^")) - if check_impl_detail(pypy=True): - self.assertEqual(err[1].find("2)") + 1, err[2].find("^")) + self.assertEqual(err[1].find("p"), err[2].find("^")) def test_base_exception(self): # Test that exceptions derived from BaseException are formatted right diff --git a/lib-python/2.7/threading.py b/lib-python/2.7/threading.py --- a/lib-python/2.7/threading.py +++ b/lib-python/2.7/threading.py @@ -351,6 +351,21 @@ # forward-compatibility reasons we do the same. waiter.acquire() gotit = True + except AttributeError: + # someone patched the 'waiter' class, probably. + # Fall back to the standard CPython logic. + # See the CPython lib for the comments about it... + endtime = _time() + timeout + delay = 0.0005 # 500 us -> initial delay of 1 ms + while True: + gotit = waiter.acquire(0) + if gotit: + break + remaining = endtime - _time() + if remaining <= 0: + break + delay = min(delay * 2, remaining, .05) + _sleep(delay) else: gotit = waiter.acquire(False) if not gotit: diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -56,13 +56,13 @@ Niko Matsakis Alexander Hesse Ludovic Aubry + stian Jacob Hallen Jason Creighton Mark Young Alex Martelli Spenser Bauman Michal Bendowski - stian Jan de Mooij Tyler Wade Vincent Legoll @@ -90,10 +90,10 @@ Wenzhu Man Konstantin Lopuhin John Witulski + Jeremy Thurgood Greg Price Ivan Sichmann Freitas Dario Bertini - Jeremy Thurgood Mark Pearse Simon Cross Tobias Pape @@ -112,18 +112,19 @@ Adrian Kuhn tav Georg Brandl + Joannah Nanjekye Bert Freudenberg Stian Andreassen Wanja Saatkamp Mike Blume - Joannah Nanjekye Gerald Klix Oscar Nierstrasz Rami Chowdhury Stefan H. Muller + Dodan Mihai Tim Felgentreff Eugene Oden - Dodan Mihai + Colin Valliant Jeff Terrace Henry Mason Vasily Kuznetsov @@ -192,12 +193,14 @@ Vaibhav Sood Reuben Cummings Attila Gobi + Floris Bruynooghe Christopher Pope Tristan Arthur Christian Tismer Dan Stromberg Carl Meyer Florin Papa + Arianna Avanzini Jens-Uwe Mager Valentina Mukhamedzhanova Stefano Parmesan @@ -221,6 +224,7 @@ Mark Williams Kunal Grover Nathan Taylor + Barry Hart Travis Francis Athougies Yasir Suhail Sergey Kishchenko @@ -228,6 +232,7 @@ Lutz Paelike Ian Foote Philipp Rustemeuer + Logan Chien Catalin Gabriel Manciu Jacob Oscarson Ryan Gonzalez @@ -263,7 +268,6 @@ Akira Li Gustavo Niemeyer Rafał Gałczyński - Logan Chien Lucas Stadler roberto at goyle Matt Bogosian @@ -276,6 +280,7 @@ Anna Katrina Dominguez Kim Jin Su Amber Brown + Miro Hrončok Anthony Sottile Nate Bragg Ben Darnell @@ -283,7 +288,6 @@ Godefroid Chappelle Julian Berman Michael Hudson-Doyle - Floris Bruynooghe Stephan Busemann Dan Colish timo @@ -325,6 +329,7 @@ Michael Chermside Anna Ravencroft remarkablerocket + Pauli Virtanen Petre Vijiac Berker Peksag Christian Muirhead @@ -349,6 +354,7 @@ Graham Markall Dan Loewenherz werat + Andrew Stepanov Niclas Olofsson Chris Pressey Tobias Diaz diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -10,89 +10,6 @@ PyPy. -.. _extension-modules: - -Extension modules ------------------ - -List of extension modules that we support: - -* Supported as built-in modules (in :source:`pypy/module/`): - - __builtin__ - :doc:`__pypy__ <__pypy__-module>` - _ast - _codecs - _collections - :doc:`_continuation ` - :doc:`_ffi ` - _hashlib - _io - _locale - _lsprof - _md5 - :doc:`_minimal_curses ` - _multiprocessing - _random - :doc:`_rawffi ` - _sha - _socket - _sre - _ssl - _warnings - _weakref - _winreg - array - binascii - bz2 - cStringIO - cmath - `cpyext`_ - crypt - errno - exceptions - fcntl - gc - imp - itertools - marshal - math - mmap - operator - parser - posix - pyexpat - select - signal - struct - symbol - sys - termios - thread - time - token - unicodedata - zipimport - zlib - - When translated on Windows, a few Unix-only modules are skipped, - and the following module is built instead: - - _winreg - -* Supported by being rewritten in pure Python (possibly using ``cffi``): - see the :source:`lib_pypy/` directory. Examples of modules that we - support this way: ``ctypes``, ``cPickle``, ``cmath``, ``dbm``, ``datetime``... - Note that some modules are both in there and in the list above; - by default, the built-in module is used (but can be disabled - at translation time). - -The extension modules (i.e. modules written in C, in the standard CPython) -that are neither mentioned above nor in :source:`lib_pypy/` are not available in PyPy. -(You may have a chance to use them anyway with `cpyext`_.) - -.. _cpyext: http://morepypy.blogspot.com/2010/04/using-cpython-extension-modules-with.html - Differences related to garbage collection strategies ---------------------------------------------------- @@ -559,7 +476,96 @@ environment variable. CPython searches for ``vcvarsall.bat`` somewhere **above** that value. +* SyntaxError_ s try harder to give details about the cause of the failure, so + the error messages are not the same as in CPython + + +.. _extension-modules: + +Extension modules +----------------- + +List of extension modules that we support: + +* Supported as built-in modules (in :source:`pypy/module/`): + + __builtin__ + :doc:`__pypy__ <__pypy__-module>` + _ast + _codecs + _collections + :doc:`_continuation ` + :doc:`_ffi ` + _hashlib + _io + _locale + _lsprof + _md5 + :doc:`_minimal_curses ` + _multiprocessing + _random + :doc:`_rawffi ` + _sha + _socket + _sre + _ssl + _warnings + _weakref + _winreg + array + binascii + bz2 + cStringIO + cmath + `cpyext`_ + crypt + errno + exceptions + fcntl + gc + imp + itertools + marshal + math + mmap + operator + parser + posix + pyexpat + select + signal + struct + symbol + sys + termios + thread + time + token + unicodedata + zipimport + zlib + + When translated on Windows, a few Unix-only modules are skipped, + and the following module is built instead: + + _winreg + +* Supported by being rewritten in pure Python (possibly using ``cffi``): + see the :source:`lib_pypy/` directory. Examples of modules that we + support this way: ``ctypes``, ``cPickle``, ``cmath``, ``dbm``, ``datetime``... + Note that some modules are both in there and in the list above; + by default, the built-in module is used (but can be disabled + at translation time). + +The extension modules (i.e. modules written in C, in the standard CPython) +that are neither mentioned above nor in :source:`lib_pypy/` are not available in PyPy. +(You may have a chance to use them anyway with `cpyext`_.) + +.. _cpyext: http://morepypy.blogspot.com/2010/04/using-cpython-extension-modules-with.html + + .. _`is ignored in PyPy`: http://bugs.python.org/issue14621 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/ .. _`issue #2653`: https://bitbucket.org/pypy/pypy/issues/2653/ +.. _SyntaxError: https://morepypy.blogspot.co.il/2018/04/improving-syntaxerror-in-pypy.html diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-v6.0.0.rst release-v5.10.1.rst release-v5.10.0.rst release-v5.9.0.rst diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -7,6 +7,8 @@ .. toctree:: whatsnew-head.rst + whatsnew-pypy2-6.0.0.rst + whatsnew-pypy2-5.10.0.rst whatsnew-pypy2-5.10.0.rst whatsnew-pypy2-5.9.0.rst whatsnew-pypy2-5.8.0.rst diff --git a/pypy/doc/release-v6.0.0.rst b/pypy/doc/release-v6.0.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-v6.0.0.rst @@ -0,0 +1,109 @@ +====================================== +PyPy2.7 and PyPy3.5 v6.0 dual release +====================================== + +The PyPy team is proud to release both PyPy2.7 v6.0 (an interpreter supporting +Python 2.7 syntax), and a PyPy3.5 v6.0 (an interpreter supporting Python +3.5 syntax). The two releases are both based on much the same codebase, thus +the dual release. + +This release is a feature release following our previous 5.10 incremental +release in late December 2017. Our C-API compatability layer ``cpyext`` is +now much faster (see the `blog post`_) as well as more complete. We have made +many other improvements in speed and CPython compatibility. Since the changes +affect the included python development header files, all c-extension modules must +be recompiled for this version. + +First-time python users are often stumped by silly typos and emissions when +getting started writing code. We have improved our parser to emit more friendly +`syntax errors`_, making PyPy not only faster but more friendly. + +The Windows PyPy3.5 release is still considered beta-quality. There are open +issues with unicode handling especially around system calls and c-extensions. + +The Matplotlib TkAgg backend now works with PyPy, as do pygame and pygobject_. + +As always, this release is 100% compatible with the previous one and fixed +several issues and bugs raised by the growing community of PyPy users. +We strongly recommend updating. + +We updated the cffi module included in PyPy to version 1.11.5 + +The utf8 branch that changes internal representation of unicode to utf8 did not +make it into the release, so there is still more goodness coming. We also +began working on a Python3.6 implementation, help is welcome. + +You can download the v6.0 releases here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project. If PyPy is not quite good enough for your needs, we are available for +direct consulting work. + +We would also like to thank our contributors and encourage new people to join +the project. PyPy has many layers and we need help with all of them: `PyPy`_ +and `RPython`_ documentation improvements, tweaking popular `modules`_ to run +on pypy, or general `help`_ with making RPython's JIT even better. + +.. _`PyPy`: index.html +.. _`RPython`: https://rpython.readthedocs.org +.. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly +.. _`help`: project-ideas.html +.. _`blog post`: https://morepypy.blogspot.it/2017/10/cape-of-good-hope-for-pypy-hello-from.html +.. _pygobject: https://lazka.github.io/posts/2018-04_pypy-pygobject/index.html +.. _`syntax errors`: https://morepypy.blogspot.com/2018/04/improving-syntaxerror-in-pypy.html + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7 and CPython 3.5. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +The PyPy release supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html + +Changelog +========= + +* Speed up C-API method calls, and make most Py*_Check calls C macros +* Speed up C-API slot method calls +* Enable TkAgg backend support for matplotlib +* support ``hastzinfo`` and ``tzinfo`` in the C-API ``PyDateTime*`` structures +* datetime.h is now more similar to CPython +* We now support ``PyUnicode_AsUTF{16,32}String``, ``_PyLong_AsByteArray``, + ``_PyLong_AsByteArrayO``, +* PyPy3.5 on Windows is compiled with the Microsoft Visual Compiler v14, like + CPython +* Fix performance of attribute lookup when more than 80 attributes are used +* Improve performance on passing built-in types to C-API C code +* Improve the performance of datetime and timedelta by skipping the consistency + checks of the datetime values (they are correct by construction) +* Improve handling of ``bigint`` s, including fixing ``int_divmod`` +* Improve reporting of GC statistics +* Accept unicode filenames in ``dbm.open()`` +* Improve RPython support for half-floats +* Added missing attributes to C-API ``instancemethod`` on pypy3 +* Store error state in thread-local storage for C-API. +* Fix JIT bugs exposed in the sre module +* Improve speed of Python parser, improve ParseError messages slightly +* Handle JIT hooks more efficiently + +We also refactored many parts of the JIT bridge optimizations, as well as cpyext +internals, and together with new contributors fixed issues, added new +documentation, and cleaned up the codebase. diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -1,99 +1,18 @@ -=========================== -What's new in PyPy2.7 5.10+ -=========================== +========================== +What's new in PyPy2.7 6.0+ +========================== -.. this is a revision shortly after release-pypy2.7-v5.10.0 -.. startrev: 6b024edd9d12 +.. this is a revision shortly after release-pypy-6.0.0 +.. startrev: 2e04adf1b89f -.. branch: cpyext-avoid-roundtrip +.. branch: cpyext-subclass-setattr -Big refactoring of some cpyext code, which avoids a lot of nonsense when -calling C from Python and vice-versa: the result is a big speedup in -function/method calls, up to 6 times faster. +Fix for python-level classes that inherit from C-API types, previously the +`w_obj` was not necessarily preserved throughout the lifetime of the `pyobj` +which led to cases where instance attributes were lost. Fixes issue #2793 -.. branch: cpyext-datetime2 -Support ``tzinfo`` field on C-API datetime objects, fixes latest pandas HEAD +.. branch: pyparser-improvements-2 - -.. branch: mapdict-size-limit - -Fix a corner case of mapdict: When an instance is used like a dict (using -``setattr`` and ``getattr``, or ``.__dict__``) and a lot of attributes are -added, then the performance using mapdict is linear in the number of -attributes. This is now fixed (by switching to a regular dict after 80 -attributes). - - -.. branch: cpyext-faster-arg-passing - -When using cpyext, improve the speed of passing certain objects from PyPy to C -code, most notably None, True, False, types, all instances of C-defined types. -Before, a dict lookup was needed every time such an object crossed over, now it -is just a field read. - - -.. branch: 2634_datetime_timedelta_performance - -Improve datetime + timedelta performance. - -.. branch: memory-accounting - -Improve way to describe memory - -.. branch: msvc14 - -Allow compilaiton with Visual Studio 2017 compiler suite on windows - -.. branch: refactor-slots - -Refactor cpyext slots. - - -.. branch: call-loopinvariant-into-bridges - -Speed up branchy code that does a lot of function inlining by saving one call -to read the TLS in most bridges. - -.. branch: rpython-sprint - -Refactor in rpython signatures - -.. branch: cpyext-tls-operror2 - -Store error state thread-locally in executioncontext, fixes issue #2764 - -.. branch: cpyext-fast-typecheck - -Optimize `Py*_Check` for `Bool`, `Float`, `Set`. Also refactor and simplify -`W_PyCWrapperObject` which is used to call slots from the C-API, greatly -improving microbenchmarks in https://github.com/antocuni/cpyext-benchmarks - - -.. branch: fix-sre-problems - -Fix two (unrelated) JIT bugs manifesting in the re module: - -- green fields are broken and were thus disabled, plus their usage removed from - the _sre implementation - -- in rare "trace is too long" situations, the JIT could break behaviour - arbitrarily. - -.. branch: jit-hooks-can-be-disabled - -Be more efficient about JIT hooks. Make it possible for the frontend to declare -that jit hooks are currently not enabled at all. in that case, the list of ops -does not have to be created in the case of the on_abort hook (which is -expensive). - - -.. branch: pyparser-improvements - -Improve speed of Python parser, improve ParseError messages slightly. - -.. branch: ioctl-arg-size - -Work around possible bugs in upstream ioctl users, like CPython allocate at -least 1024 bytes for the arg in calls to ``ioctl(fd, request, arg)``. Fixes -issue #2776 +Improve line offsets that are reported by SyntaxError. Improve error messages +for a few situations, including mismatched parenthesis. diff --git a/pypy/doc/whatsnew-pypy2-6.0.0.rst b/pypy/doc/whatsnew-pypy2-6.0.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-pypy2-6.0.0.rst @@ -0,0 +1,99 @@ +=========================== +What's new in PyPy2.7 5.10+ +=========================== + +.. this is a revision shortly after release-pypy2.7-v5.10.0 +.. startrev: 6b024edd9d12 + +.. branch: cpyext-avoid-roundtrip + +Big refactoring of some cpyext code, which avoids a lot of nonsense when +calling C from Python and vice-versa: the result is a big speedup in +function/method calls, up to 6 times faster. + +.. branch: cpyext-datetime2 + +Support ``tzinfo`` field on C-API datetime objects, fixes latest pandas HEAD + + +.. branch: mapdict-size-limit + +Fix a corner case of mapdict: When an instance is used like a dict (using +``setattr`` and ``getattr``, or ``.__dict__``) and a lot of attributes are +added, then the performance using mapdict is linear in the number of +attributes. This is now fixed (by switching to a regular dict after 80 +attributes). + + +.. branch: cpyext-faster-arg-passing + +When using cpyext, improve the speed of passing certain objects from PyPy to C +code, most notably None, True, False, types, all instances of C-defined types. +Before, a dict lookup was needed every time such an object crossed over, now it +is just a field read. + + +.. branch: 2634_datetime_timedelta_performance + +Improve datetime + timedelta performance. + +.. branch: memory-accounting + +Improve way to describe memory + +.. branch: msvc14 + +Allow compilaiton with Visual Studio 2017 compiler suite on windows + +.. branch: refactor-slots + +Refactor cpyext slots. + + +.. branch: call-loopinvariant-into-bridges + +Speed up branchy code that does a lot of function inlining by saving one call +to read the TLS in most bridges. + +.. branch: rpython-sprint + +Refactor in rpython signatures + +.. branch: cpyext-tls-operror2 + +Store error state thread-locally in executioncontext, fixes issue #2764 + +.. branch: cpyext-fast-typecheck + +Optimize `Py*_Check` for `Bool`, `Float`, `Set`. Also refactor and simplify +`W_PyCWrapperObject` which is used to call slots from the C-API, greatly +improving microbenchmarks in https://github.com/antocuni/cpyext-benchmarks + + +.. branch: fix-sre-problems + +Fix two (unrelated) JIT bugs manifesting in the re module: + +- green fields are broken and were thus disabled, plus their usage removed from + the _sre implementation + +- in rare "trace is too long" situations, the JIT could break behaviour + arbitrarily. + +.. branch: jit-hooks-can-be-disabled + +Be more efficient about JIT hooks. Make it possible for the frontend to declare +that jit hooks are currently not enabled at all. in that case, the list of ops +does not have to be created in the case of the on_abort hook (which is +expensive). + + +.. branch: pyparser-improvements + +Improve speed of Python parser, improve ParseError messages slightly. + +.. branch: ioctl-arg-size + +Work around possible bugs in upstream ioctl users, like CPython allocate at +least 1024 bytes for the arg in calls to ``ioctl(fd, request, arg)``. Fixes +issue #2776 diff --git a/pypy/interpreter/pyparser/error.py b/pypy/interpreter/pyparser/error.py --- a/pypy/interpreter/pyparser/error.py +++ b/pypy/interpreter/pyparser/error.py @@ -6,6 +6,7 @@ lastlineno=0): self.msg = msg self.lineno = lineno + # NB: offset is a 1-based index! self.offset = offset self.text = text self.filename = filename diff --git a/pypy/interpreter/pyparser/parser.py b/pypy/interpreter/pyparser/parser.py --- a/pypy/interpreter/pyparser/parser.py +++ b/pypy/interpreter/pyparser/parser.py @@ -199,6 +199,7 @@ self.token_type = token_type self.value = value self.lineno = lineno + # this is a 0-based index self.column = column self.line = line self.expected = expected diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py --- a/pypy/interpreter/pyparser/pyparse.py +++ b/pypy/interpreter/pyparser/pyparse.py @@ -188,7 +188,9 @@ if e.expected_str is not None: msg += " (expected '%s')" % e.expected_str - raise new_err(msg, e.lineno, e.column, e.line, + # parser.ParseError(...).column is 0-based, but the offsets in the + # exceptions in the error module are 1-based, hence the '+ 1' + raise new_err(msg, e.lineno, e.column + 1, e.line, compile_info.filename) else: tree = self.root diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py --- a/pypy/interpreter/pyparser/pytokenizer.py +++ b/pypy/interpreter/pyparser/pytokenizer.py @@ -73,14 +73,14 @@ logical line; continuation lines are included. """ token_list = [] - lnum = parenlev = continued = 0 + lnum = continued = 0 namechars = NAMECHARS numchars = NUMCHARS contstr, needcont = '', 0 contline = None indents = [0] last_comment = '' - parenlevstart = (0, 0, "") + parenstack = [] # make the annotator happy endDFA = DUMMY_DFA @@ -97,7 +97,7 @@ if contstr: if not line: raise TokenError( - "EOF while scanning triple-quoted string literal", + "end of file (EOF) while scanning triple-quoted string literal", strstart[2], strstart[0], strstart[1]+1, token_list, lnum-1) endmatch = endDFA.recognize(line) @@ -123,7 +123,7 @@ contline = contline + line continue - elif parenlev == 0 and not continued: # new statement + elif not parenstack and not continued: # new statement if not line: break column = 0 while pos < max: # measure leading whitespace @@ -143,21 +143,21 @@ token_list.append((tokens.INDENT, line[:pos], lnum, 0, line)) last_comment = '' while column < indents[-1]: - indents = indents[:-1] + indents.pop() token_list.append((tokens.DEDENT, '', lnum, pos, line)) last_comment = '' if column != indents[-1]: err = "unindent does not match any outer indentation level" - raise TokenIndentationError(err, line, lnum, 0, token_list) + raise TokenIndentationError(err, line, lnum, column+1, token_list) else: # continued statement if not line: - if parenlev > 0: - lnum1, start1, line1 = parenlevstart + if parenstack: + _, lnum1, start1, line1 = parenstack[0] raise TokenError("parenthesis is never closed", line1, lnum1, start1 + 1, token_list, lnum) - raise TokenError("EOF in multi-line statement", line, - lnum, 0, token_list) + raise TokenError("end of file (EOF) in multi-line statement", line, + lnum, 0, token_list) # XXX why is the offset 0 here? continued = 0 while pos < max: @@ -180,7 +180,7 @@ token_list.append((tokens.NUMBER, token, lnum, start, line)) last_comment = '' elif initial in '\r\n': - if parenlev <= 0: + if not parenstack: tok = (tokens.NEWLINE, last_comment, lnum, start, line) token_list.append(tok) last_comment = '' @@ -222,14 +222,22 @@ continued = 1 else: if initial in '([{': - if parenlev == 0: - parenlevstart = (lnum, start, line) - parenlev = parenlev + 1 + parenstack.append((initial, lnum, start, line)) elif initial in ')]}': - parenlev = parenlev - 1 - if parenlev < 0: + if not parenstack: raise TokenError("unmatched '%s'" % initial, line, lnum, start + 1, token_list) + opening, lnum1, start1, line1 = parenstack.pop() + if not ((opening == "(" and initial == ")") or + (opening == "[" and initial == "]") or + (opening == "{" and initial == "}")): + msg = "closing parenthesis '%s' does not match opening parenthesis '%s'" % ( + initial, opening) + + if lnum1 != lnum: + msg += " on line " + str(lnum1) + raise TokenError( + msg, line, lnum, start + 1, token_list) if token in python_opmap: punct = python_opmap[token] else: @@ -241,7 +249,7 @@ if start < 0: start = pos if start", "exec") parser = pyparse.PythonParser(fakespace) tree = parser._parse(s, info) b = time.clock() - print title, (b-a) + print fn, (b-a) def entry_point(argv): - bench("foo") + if len(argv) == 2: + fn = argv[1] + else: + fn = "../../../../rpython/rlib/unicodedata/unicodedb_5_2_0.py" + fd = os.open(fn, os.O_RDONLY, 0777) + res = [] + while True: + s = os.read(fd, 4096) + if not s: + break + res.append(s) + os.close(fd) + s = "".join(res) + print len(s) + bench(fn, s) return 0 diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py --- a/pypy/interpreter/pyparser/test/test_pyparse.py +++ b/pypy/interpreter/pyparser/test/test_pyparse.py @@ -76,14 +76,14 @@ exc = py.test.raises(SyntaxError, parse, "name another for").value assert exc.msg == "invalid syntax" assert exc.lineno == 1 - assert exc.offset == 5 + assert exc.offset == 6 assert exc.text.startswith("name another for") exc = py.test.raises(SyntaxError, parse, "x = \"blah\n\n\n").value - assert exc.msg == "EOL while scanning string literal" + assert exc.msg == "end of line (EOL) while scanning string literal" assert exc.lineno == 1 assert exc.offset == 5 exc = py.test.raises(SyntaxError, parse, "x = '''\n\n\n").value - assert exc.msg == "EOF while scanning triple-quoted string literal" + assert exc.msg == "end of file (EOF) while scanning triple-quoted string literal" assert exc.lineno == 1 assert exc.offset == 5 assert exc.lastlineno == 3 @@ -112,7 +112,7 @@ assert exc.msg == "expected an indented block" assert exc.lineno == 3 assert exc.text.startswith("pass") - assert exc.offset == 0 + assert exc.offset == 1 input = "hi\n indented" exc = py.test.raises(IndentationError, parse, input).value assert exc.msg == "unexpected indent" @@ -120,6 +120,7 @@ exc = py.test.raises(IndentationError, parse, input).value assert exc.msg == "unindent does not match any outer indentation level" assert exc.lineno == 3 + assert exc.offset == 3 def test_mac_newline(self): self.parse("this_is\ra_mac\rfile") diff --git a/pypy/interpreter/pyparser/test/test_pytokenizer.py b/pypy/interpreter/pyparser/test/test_pytokenizer.py new file mode 100644 --- /dev/null +++ b/pypy/interpreter/pyparser/test/test_pytokenizer.py @@ -0,0 +1,66 @@ +import pytest +from pypy.interpreter.pyparser import pytokenizer +from pypy.interpreter.pyparser.pygram import tokens +from pypy.interpreter.pyparser.error import TokenError + +def tokenize(s): + return pytokenizer.generate_tokens(s.splitlines(True) + ["\n"], 0) + +def check_token_error(s, msg=None, pos=-1, line=-1): + error = pytest.raises(TokenError, tokenize, s) + if msg is not None: + assert error.value.msg == msg + if pos != -1: + assert error.value.offset == pos + if line != -1: + assert error.value.lineno == line + + +class TestTokenizer(object): + + def test_simple(self): + line = "a+1" + tks = tokenize(line) + assert tks == [ + (tokens.NAME, 'a', 1, 0, line), + (tokens.PLUS, '+', 1, 1, line), + (tokens.NUMBER, '1', 1, 2, line), + (tokens.NEWLINE, '', 2, 0, '\n'), + (tokens.NEWLINE, '', 2, 0, '\n'), + (tokens.ENDMARKER, '', 2, 0, ''), + ] + + def test_error_parenthesis(self): + for paren in "([{": + check_token_error(paren + "1 + 2", + "parenthesis is never closed", + 1) + + for paren in ")]}": + check_token_error("1 + 2" + paren, + "unmatched '%s'" % (paren, ), + 6) + + for i, opening in enumerate("([{"): + for j, closing in enumerate(")]}"): + if i == j: + continue + check_token_error(opening + "1\n" + closing, + "closing parenthesis '%s' does not match opening parenthesis '%s' on line 1" % (closing, opening), + pos=1, line=2) + check_token_error(opening + "1" + closing, + "closing parenthesis '%s' does not match opening parenthesis '%s'" % (closing, opening), + pos=3, line=1) + check_token_error(opening + closing, + "closing parenthesis '%s' does not match opening parenthesis '%s'" % (closing, opening), + pos=2, line=1) + + + def test_unknown_char(self): + check_token_error("?", "Unknown character", 1) + + def test_eol_string(self): + check_token_error("x = 'a", pos=5, line=1) + + def test_eof_triple_quoted(self): + check_token_error("'''", pos=1, line=1) diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py --- a/pypy/interpreter/test/test_compiler.py +++ b/pypy/interpreter/test/test_compiler.py @@ -77,7 +77,7 @@ """) assert self.space.unwrap(w_args) == ( 'unindent does not match any outer indentation level', - ('', 3, 0, ' y\n')) + ('', 3, 2, ' y\n')) def test_getcodeflags(self): code = self.compiler.compile('from __future__ import division\n', diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py --- a/pypy/interpreter/test/test_syntax.py +++ b/pypy/interpreter/test/test_syntax.py @@ -750,7 +750,7 @@ except SyntaxError as e: assert e.lineno == 4 assert e.text.endswith('a b c d e\n') - assert e.offset == e.text.index('b') + assert e.offset == e.text.index('b') + 1 # offset is 1-based else: raise Exception("no SyntaxError??") diff --git a/pypy/interpreter/test/test_typedef.py b/pypy/interpreter/test/test_typedef.py --- a/pypy/interpreter/test/test_typedef.py +++ b/pypy/interpreter/test/test_typedef.py @@ -423,3 +423,10 @@ def test_get_with_none_arg(self): raises(TypeError, type.__dict__['__mro__'].__get__, None) raises(TypeError, type.__dict__['__mro__'].__get__, None, None) + + def test_builtin_readonly_property(self): + import sys + x = lambda: 5 + e = raises(TypeError, 'x.func_globals = {}') + if '__pypy__' in sys.builtin_module_names: + assert str(e.value) == "readonly attribute 'func_globals'" diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py --- a/pypy/interpreter/typedef.py +++ b/pypy/interpreter/typedef.py @@ -309,12 +309,18 @@ self.reqcls, Arguments(space, [w_obj, space.newtext(self.name)])) + def readonly_attribute(self, space): # overwritten in cpyext + if self.name == '': + raise oefmt(space.w_TypeError, "readonly attribute") + else: + raise oefmt(space.w_TypeError, "readonly attribute '%s'", self.name) + def descr_property_set(self, space, w_obj, w_value): """property.__set__(obj, value) Change the value of the property of the given obj.""" fset = self.fset if fset is None: - raise oefmt(space.w_TypeError, "readonly attribute") + raise self.readonly_attribute(space) try: fset(self, space, w_obj, w_value) except DescrMismatch: diff --git a/pypy/module/_rawffi/alt/test/test_struct.py b/pypy/module/_rawffi/alt/test/test_struct.py --- a/pypy/module/_rawffi/alt/test/test_struct.py +++ b/pypy/module/_rawffi/alt/test/test_struct.py @@ -43,7 +43,11 @@ def setup_class(cls): BaseAppTestFFI.setup_class.im_func(cls) - @unwrap_spec(addr=int, typename='text', length=int) + from rpython.rlib import clibffi + from rpython.rlib.rarithmetic import r_uint + from rpython.rtyper.lltypesystem import lltype, rffi + + @unwrap_spec(addr=r_uint, typename='text', length=int) def read_raw_mem(space, addr, typename, length): import ctypes addr = ctypes.cast(addr, ctypes.c_void_p) @@ -58,9 +62,6 @@ else: cls.w_read_raw_mem = cls.space.wrap(interp2app(read_raw_mem)) # - from rpython.rlib import clibffi - from rpython.rlib.rarithmetic import r_uint - from rpython.rtyper.lltypesystem import lltype, rffi dummy_type = lltype.malloc(clibffi.FFI_TYPE_P.TO, flavor='raw') dummy_type.c_size = r_uint(123) dummy_type.c_alignment = rffi.cast(rffi.USHORT, 0) diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -640,7 +640,7 @@ 'Py_DivisionWarningFlag', 'Py_DontWriteBytecodeFlag', 'Py_NoUserSiteDirectory', '_Py_QnewFlag', 'Py_Py3kWarningFlag', 'Py_HashRandomizationFlag', '_Py_PackageContext', '_PyTraceMalloc_Track', '_PyTraceMalloc_Untrack', 'PyMem_Malloc', - 'Py_IncRef', 'Py_DecRef', 'PyObject_Free', 'PyObject_GC_Del', 'PyType_GenericAlloc', + 'PyObject_Free', 'PyObject_GC_Del', 'PyType_GenericAlloc', '_PyObject_New', '_PyObject_NewVar', '_PyObject_GC_New', '_PyObject_GC_NewVar', 'PyObject_Init', 'PyObject_InitVar', 'PyInt_FromLong', diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -29,8 +29,8 @@ #define PY_VERSION "2.7.13" /* PyPy version as a string */ -#define PYPY_VERSION "5.11.0-alpha0" -#define PYPY_VERSION_NUM 0x050B0000 +#define PYPY_VERSION "6.1.0-alpha0" +#define PYPY_VERSION_NUM 0x06010000 /* Defined to mean a PyPy where cpyext holds more regular references to PyObjects, e.g. staying alive as long as the internal PyPy object diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py --- a/pypy/module/cpyext/pyobject.py +++ b/pypy/module/cpyext/pyobject.py @@ -60,10 +60,10 @@ def _cpyext_attach_pyobj(self, space, py_obj): self._cpy_ref = py_obj - rawrefcount.create_link_pyobj(self, py_obj) + rawrefcount.create_link_pypy(self, py_obj) cls._cpyext_attach_pyobj = _cpyext_attach_pyobj -add_direct_pyobj_storage(W_BaseCPyObject) +add_direct_pyobj_storage(W_BaseCPyObject) add_direct_pyobj_storage(W_TypeObject) add_direct_pyobj_storage(W_NoneObject) add_direct_pyobj_storage(W_BoolObject) @@ -414,3 +414,14 @@ @cpython_api([rffi.VOIDP], lltype.Signed, error=CANNOT_FAIL) def _Py_HashPointer(space, ptr): return rffi.cast(lltype.Signed, ptr) + + at cpython_api([PyObject], lltype.Void) +def Py_IncRef(space, obj): + # used only ifdef PYPY_DEBUG_REFCOUNT + if obj: + incref(space, obj) + + at cpython_api([PyObject], lltype.Void) +def Py_DecRef(space, obj): + # used only ifdef PYPY_DEBUG_REFCOUNT + decref(space, obj) diff --git a/pypy/module/cpyext/setobject.py b/pypy/module/cpyext/setobject.py --- a/pypy/module/cpyext/setobject.py +++ b/pypy/module/cpyext/setobject.py @@ -1,4 +1,4 @@ -from pypy.interpreter.error import oefmt +from pypy.interpreter.error import OperationError, oefmt from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.cpyext.api import (cpython_api, Py_ssize_t, CANNOT_FAIL, build_type_checkers) @@ -64,8 +64,13 @@ instance of set or its subtype.""" if not PySet_Check(space, w_s): PyErr_BadInternalCall(space) - space.call_method(space.w_set, 'discard', w_s, w_obj) - return 0 + try: + space.call_method(space.w_set, 'remove', w_s, w_obj) + except OperationError as e: + if e.match(space, space.w_KeyError): + return 0 + raise + return 1 @cpython_api([PyObject], PyObject) diff --git a/pypy/module/cpyext/src/object.c b/pypy/module/cpyext/src/object.c --- a/pypy/module/cpyext/src/object.c +++ b/pypy/module/cpyext/src/object.c @@ -5,18 +5,6 @@ extern void _PyPy_Free(void *ptr); extern void *_PyPy_Malloc(Py_ssize_t size); -void -Py_IncRef(PyObject *o) -{ - Py_XINCREF(o); -} - -void -Py_DecRef(PyObject *o) -{ - Py_XDECREF(o); -} - /* * The actual value of this variable will be the address of * pyobject.w_marker_deallocating, and will be set by diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c --- a/pypy/module/cpyext/test/array.c +++ b/pypy/module/cpyext/test/array.c @@ -2492,6 +2492,87 @@ return PyLong_FromLong(obj1->ob_type->tp_dealloc == obj2->ob_type->tp_dealloc); } +static PyObject * +subclass_with_attribute(PyObject *self, PyObject* args) { + /* what happens when we use tp_alloc to create the subclass, then + * assign to the w_obj via python, then get the GC to collect? + * The w_obj should not be collected!! + */ + PyObject * obj, *sub, *attrib, *funcname, *attribname, *collect, *res, *tup; + PyTypeObject * subtype; + int i; + if (!PyArg_ParseTuple(args, "OOOO", &obj, &funcname, &attribname, &collect)) { + return NULL; + } + if (!PyType_Check(obj)) { + PyErr_SetString(PyExc_TypeError, + "expected type object"); + return NULL; + } + subtype = (PyTypeObject*)obj; + sub = subtype->tp_alloc(subtype, 0); + if (!sub) { + return NULL; + } + attrib = PyObject_GetAttr(sub, funcname); + if (!attrib || (attrib == Py_None) ) { + PyErr_SetString(PyExc_ValueError, + "could not find function to call"); + Py_XDECREF(attrib); + Py_DECREF(sub); + return NULL; + } + tup = PyTuple_New(0); + /* + #ifdef PYPY_VERSION + printf("calling addattrib pypylink %lu \n", sub->ob_pypy_link); + #endif + */ + res = PyObject_Call(attrib, tup, NULL); + /* + #ifdef PYPY_VERSION + printf("after addattrib pypylink %lu \n", sub->ob_pypy_link); + #endif + */ + Py_DECREF(attrib); + if (res == NULL) { + Py_DECREF(tup); + Py_DECREF(sub); + return NULL; + } + Py_DECREF(res); + for(i=0; i<10; i++) { + /* + #ifdef PYPY_VERSION + printf("starting loop iteration %d refcnt %lu pypylink %lu \n", i, + sub->ob_refcnt, sub->ob_pypy_link); + #else + printf("starting loop iteration %d refcnt %lu\n", i, sub->ob_refcnt); + #endif + */ + attrib = PyObject_GetAttr(sub, attribname); + if (!attrib || (attrib == Py_None)) { + PyErr_SetString(PyExc_ValueError, + "could not find attrib on object"); + Py_XDECREF(attrib); + Py_DECREF(tup); + Py_DECREF(sub); + return NULL; + } + Py_XDECREF(attrib); + res = PyObject_Call(collect, tup, NULL); + if (res == NULL) { + Py_DECREF(tup); + Py_DECREF(sub); + return NULL; + } + Py_DECREF(res); + } + Py_DECREF(tup); + Py_DECREF(sub); + Py_RETURN_NONE; +} + /*********************** Install Module **************************/ static PyMethodDef a_methods[] = { @@ -2502,6 +2583,7 @@ {"create_and_release_buffer", (PyCFunction)create_and_release_buffer, METH_O, NULL}, {"same_dealloc", (PyCFunction)same_dealloc, METH_VARARGS, NULL}, {"getitem", (PyCFunction)getitem, METH_VARARGS, NULL}, + {"subclass_with_attribute", (PyCFunction)subclass_with_attribute, METH_VARARGS, NULL}, {NULL, NULL, 0, NULL} /* Sentinel */ }; diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py --- a/pypy/module/cpyext/test/test_arraymodule.py +++ b/pypy/module/cpyext/test/test_arraymodule.py @@ -186,3 +186,15 @@ # array_subscr does) raises(IndexError, module.getitem, a, -5) + def test_subclass_with_attribute(self): + module = self.import_module(name='array') + class Sub(module.array): + def addattrib(self): + print('called addattrib') + self.attrib = True + import gc + module.subclass_with_attribute(Sub, "addattrib", "attrib", gc.collect) + if self.runappdirect: + assert Sub.__module__ == 'pypy.module.cpyext.test.test_arraymodule' + assert str(Sub) == "" + diff --git a/pypy/module/cpyext/test/test_setobject.py b/pypy/module/cpyext/test/test_setobject.py --- a/pypy/module/cpyext/test/test_setobject.py +++ b/pypy/module/cpyext/test/test_setobject.py @@ -28,7 +28,11 @@ assert api.PySet_Size(w_set) == 4 api.PySet_Add(w_set, space.wrap(6)) assert api.PySet_Size(w_set) == 5 - api.PySet_Discard(w_set, space.wrap(6)) + res = api.PySet_Discard(w_set, space.wrap(6)) + assert res == 1 + assert api.PySet_Size(w_set) == 4 + res = api.PySet_Discard(w_set, space.wrap(6)) + assert res == 0 assert api.PySet_Size(w_set) == 4 def test_set_contains(self, space, api): diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py --- a/pypy/module/cpyext/test/test_tupleobject.py +++ b/pypy/module/cpyext/test/test_tupleobject.py @@ -67,6 +67,18 @@ assert space.int_w(space.getitem(w_tuple, space.wrap(i))) == 42 + i decref(space, ar[0]) + py_tuple = state.ccall("PyTuple_New", 1) + ar[0] = py_tuple + api._PyTuple_Resize(ar, 1) + assert api.PyTuple_Size(ar[0]) == 1 + decref(space, ar[0]) + + py_tuple = state.ccall("PyTuple_New", 1) + ar[0] = py_tuple + api._PyTuple_Resize(ar, 5) + assert api.PyTuple_Size(ar[0]) == 5 + decref(space, ar[0]) + lltype.free(ar, flavor='raw') def test_setitem(self, space, api): diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -5,7 +5,7 @@ from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.api import generic_cpy_call from pypy.module.cpyext.pyobject import make_ref, from_ref, decref, as_pyobj -from pypy.module.cpyext.typeobject import PyTypeObjectPtr +from pypy.module.cpyext.typeobject import PyTypeObjectPtr, W_PyCTypeObject class AppTestTypeObject(AppTestCpythonExtensionBase): @@ -412,33 +412,42 @@ def test_type_dict(self): foo = self.import_module("foo") module = self.import_extension('test', [ - ("hack_tp_dict", "METH_O", + ("hack_tp_dict", "METH_VARARGS", ''' - PyTypeObject *type = args->ob_type; + PyTypeObject *type, *obj; PyObject *a1 = PyLong_FromLong(1); PyObject *a2 = PyLong_FromLong(2); PyObject *value; + PyObject * key; + if (!PyArg_ParseTuple(args, "OO", &obj, &key)) + return NULL; + type = obj->ob_type; - if (PyDict_SetItemString(type->tp_dict, "a", + if (PyDict_SetItem(type->tp_dict, key, a1) < 0) return NULL; Py_DECREF(a1); PyType_Modified(type); - value = PyObject_GetAttrString((PyObject *)type, "a"); + value = PyObject_GetAttr((PyObject *)type, key); Py_DECREF(value); - if (PyDict_SetItemString(type->tp_dict, "a", + if (PyDict_SetItem(type->tp_dict, key, a2) < 0) return NULL; Py_DECREF(a2); PyType_Modified(type); - value = PyObject_GetAttrString((PyObject *)type, "a"); + value = PyObject_GetAttr((PyObject *)type, key); return value; ''' ) ]) obj = foo.new() - assert module.hack_tp_dict(obj) == 2 + assert module.hack_tp_dict(obj, "a") == 2 + class Sub(foo.fooType): + pass + obj = Sub() + assert module.hack_tp_dict(obj, "b") == 2 + def test_tp_descr_get(self): module = self.import_extension('foo', [ @@ -560,6 +569,23 @@ assert w_obj is None assert api.PyErr_Occurred() is None + def test_subclass_not_PyCTypeObject(self, space, api): + pyobj = make_ref(space, api.PyLong_Type) + py_type = rffi.cast(PyTypeObjectPtr, pyobj) + w_pyclass = W_PyCTypeObject(space, py_type) + w_class = space.appexec([w_pyclass], """(base): + class Sub(base): + def addattrib(self, value): + self.attrib = value + return Sub + """) + assert w_pyclass in w_class.mro_w + assert isinstance(w_pyclass, W_PyCTypeObject) + assert not isinstance(w_class, W_PyCTypeObject) + assert w_pyclass.is_cpytype() + # XXX document the current status, not clear if this is desirable + assert w_class.is_cpytype() + class AppTestSlots(AppTestCpythonExtensionBase): def setup_class(cls): @@ -1600,6 +1626,65 @@ pass C(42) # assert is not aborting + def test_getset(self): + module = self.import_extension('foo', [ + ("get_instance", "METH_NOARGS", + ''' + return PyObject_New(PyObject, &Foo_Type); + ''' + ), ("get_number", "METH_NOARGS", + ''' + return PyInt_FromLong(my_global_number); + ''' + )], prologue=''' + #if PY_MAJOR_VERSION > 2 + #define PyInt_FromLong PyLong_FromLong + #define PyInt_AsLong PyLong_AsLong + #endif + static long my_global_number; + static PyTypeObject Foo_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "foo.foo", + }; + static PyObject *bar_get(PyObject *foo, void *closure) + { + return PyInt_FromLong(1000 + (long)closure); + } + static PyObject *baz_get(PyObject *foo, void *closure) + { + return PyInt_FromLong(2000 + (long)closure); + } + static int baz_set(PyObject *foo, PyObject *x, void *closure) + { + if (x != NULL) + my_global_number = 3000 + (long)closure + PyInt_AsLong(x); + else + my_global_number = 4000 + (long)closure; + return 0; + } + static PyGetSetDef foo_getset[] = { + { "bar", bar_get, NULL, "mybardoc", (void *)42 }, + { "baz", baz_get, baz_set, "mybazdoc", (void *)43 }, + { NULL } + }; + ''', more_init = ''' + Foo_Type.tp_getset = foo_getset; + Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT; + if (PyType_Ready(&Foo_Type) < 0) INITERROR; + ''') + foo = module.get_instance() + assert foo.bar == 1042 + assert foo.bar == 1042 + assert foo.baz == 2043 + foo.baz = 50000 + assert module.get_number() == 53043 + e = raises(AttributeError, "foo.bar = 0") + assert str(e.value).startswith("attribute 'bar' of '") + assert str(e.value).endswith("foo' objects is not writable") + del foo.baz + assert module.get_number() == 4043 + raises(AttributeError, "del foo.bar") + class AppTestHashable(AppTestCpythonExtensionBase): def test_unhashable(self): diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py --- a/pypy/module/cpyext/tupleobject.py +++ b/pypy/module/cpyext/tupleobject.py @@ -187,6 +187,8 @@ PyErr_BadInternalCall(space) oldref = rffi.cast(PyTupleObject, ref) oldsize = oldref.c_ob_size + if oldsize == newsize: + return 0 ptup = state.ccall("PyTuple_New", newsize) if not ptup: state.check_and_raise_exception(always=True) @@ -199,8 +201,9 @@ to_cp = newsize for i in range(to_cp): ob = oldref.c_ob_item[i] - incref(space, ob) - newref.c_ob_item[i] = ob + if ob: + incref(space, ob) + newref.c_ob_item[i] = ob except: decref(space, p_ref[0]) p_ref[0] = lltype.nullptr(PyObject.TO) diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -1,6 +1,6 @@ import os -from rpython.rlib import jit +from rpython.rlib import jit, rawrefcount from rpython.rlib.objectmodel import specialize, we_are_translated from rpython.rtyper.lltypesystem import rffi, lltype @@ -54,19 +54,26 @@ class W_GetSetPropertyEx(GetSetProperty): def __init__(self, getset, w_type): self.getset = getset - self.name = rffi.charp2str(getset.c_name) self.w_type = w_type - doc = set = get = None + doc = fset = fget = fdel = None if doc: # XXX dead code? doc = rffi.charp2str(getset.c_doc) if getset.c_get: - get = GettersAndSetters.getter.im_func + fget = GettersAndSetters.getter.im_func if getset.c_set: - set = GettersAndSetters.setter.im_func - GetSetProperty.__init__(self, get, set, None, doc, + fset = GettersAndSetters.setter.im_func + fdel = GettersAndSetters.deleter.im_func + GetSetProperty.__init__(self, fget, fset, fdel, doc, cls=None, use_closure=True, tag="cpyext_1") + self.name = rffi.charp2str(getset.c_name) + + def readonly_attribute(self, space): # overwritten + raise oefmt(space.w_AttributeError, + "attribute '%s' of '%N' objects is not writable", + self.name, self.w_type) + def PyDescr_NewGetSet(space, getset, w_type): return W_GetSetPropertyEx(getset, w_type) @@ -454,6 +461,16 @@ state = space.fromcache(State) state.check_and_raise_exception() + def deleter(self, space, w_self): + assert isinstance(self, W_GetSetPropertyEx) + check_descr(space, w_self, self.w_type) + res = generic_cpy_call( + space, self.getset.c_set, w_self, None, + self.getset.c_closure) + if rffi.cast(lltype.Signed, res) < 0: + state = space.fromcache(State) + state.check_and_raise_exception() + def member_getter(self, space, w_self): assert isinstance(self, W_MemberDescr) check_descr(space, w_self, self.w_type) @@ -517,6 +534,10 @@ self.w_doc = space.newtext( rffi.charp2str(cts.cast('char*', pto.c_tp_doc))) + def _cpyext_attach_pyobj(self, space, py_obj): + self._cpy_ref = py_obj + rawrefcount.create_link_pyobj(self, py_obj) + @bootstrap_function def init_typeobject(space): make_typedescr(space.w_type.layout.typedef, @@ -777,7 +798,6 @@ try: w_obj = _type_realize(space, py_obj) finally: - name = rffi.charp2str(cts.cast('char*', pto.c_tp_name)) pto.c_tp_flags &= ~Py_TPFLAGS_READYING pto.c_tp_flags |= Py_TPFLAGS_READY return w_obj @@ -884,7 +904,6 @@ base = pto.c_tp_base base_pyo = rffi.cast(PyObject, pto.c_tp_base) if base and not base.c_tp_flags & Py_TPFLAGS_READY: - name = rffi.charp2str(cts.cast('char*', base.c_tp_name)) type_realize(space, base_pyo) if base and not pto.c_ob_type: # will be filled later pto.c_ob_type = base.c_ob_type diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -10,7 +10,7 @@ #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py CPYTHON_API_VERSION = 1013 #XXX # sync with include/modsupport.h -PYPY_VERSION = (5, 11, 0, "alpha", 0) #XXX # sync patchlevel.h +PYPY_VERSION = (6, 1, 0, "alpha", 0) #XXX # sync patchlevel.h import pypy diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh --- a/pypy/tool/release/repackage.sh +++ b/pypy/tool/release/repackage.sh @@ -1,8 +1,8 @@ # Edit these appropriately before running this script pmaj=2 # python main version pmin=7 # python minor version -maj=5 -min=8 +maj=6 +min=0 rev=0 branchname=release-pypy$pmaj.$pmin-$maj.x # ==OR== release-$maj.x # ==OR== release-$maj.$min.x tagname=release-pypy$pmaj.$pmin-v$maj.$min.$rev # ==OR== release-$maj.$min diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -644,7 +644,10 @@ def get_rpy_type_index(gcref): from rpython.rlib.rarithmetic import intmask Class = gcref._x.__class__ - return intmask(id(Class)) + i = intmask(id(Class)) + if i < 0: + i = ~i # always return a positive number, at least + return i def cast_gcref_to_int(gcref): # This is meant to be used on cast_instance_to_gcref results. From pypy.commits at gmail.com Fri Apr 13 03:55:40 2018 From: pypy.commits at gmail.com (mattip) Date: Fri, 13 Apr 2018 00:55:40 -0700 (PDT) Subject: [pypy-commit] buildbot default: test, fix for displaying more than one builder, add link to all RPython builders Message-ID: <5ad0627c.83e11c0a.895d2.636a@mx.google.com> Author: Matti Picus Branch: Changeset: r1062:2416342c7b7b Date: 2018-04-13 10:55 +0300 http://bitbucket.org/pypy/buildbot/changeset/2416342c7b7b/ Log: test, fix for displaying more than one builder, add link to all RPython builders diff --git a/bot2/pypybuildbot/summary.py b/bot2/pypybuildbot/summary.py --- a/bot2/pypybuildbot/summary.py +++ b/bot2/pypybuildbot/summary.py @@ -324,7 +324,7 @@ def __init__(self, status): self.sections = [] self.cur_cat_branch=None - self.fixed_builder = False + self.n_builders = 0 self.status = status def make_longrepr_url_for(self, outcome_set, namekey): @@ -347,7 +347,7 @@ builder = cachekey[0] anchors.append(' ') timing = "" - if self.fixed_builder and info['elapsed'] is not None: + if self.n_builders == 1 and info['elapsed'] is not None: timing = " in %s" % show_elapsed(info['elapsed']) if info['times'][1] is not None: day = time.localtime(info['times'][1])[:3] @@ -396,7 +396,7 @@ self.sections.append(html.h2(cat_anchor," ",branch_anchor, " ", extra)) def _builder_anchor(self, builder): - if self.fixed_builder: + if self.n_builders > 0: url = self.status.getURLForThing(self.status.getBuilder(builder)) cls = "builder" else: @@ -409,7 +409,7 @@ return outcome_set.map.values()[0].key def _label(self, outcome_set): - if self.fixed_builder: + if self.n_builders > 0: # (rev, buildNumber) buildNumber = self._builder_num(outcome_set)[1] return (outcome_set.revision, buildNumber) @@ -419,7 +419,7 @@ def _label_for_sorting(self, outcome_set): encodedrev = encode_rev_for_ordering(outcome_set.revision) - if self.fixed_builder: + if self.n_builders > 0: # (rev, buildNumber) buildNumber = self._builder_num(outcome_set)[1] return (encodedrev, buildNumber) @@ -429,7 +429,7 @@ def _label_anchor(self, outcome_set, revsize): rev = outcome_set.revision - if self.fixed_builder: + if self.n_builders > 0: pick = "builder=%s&builds=%d" % self._builder_num(outcome_set) else: pick = "recentrev=%s" % rev @@ -738,7 +738,7 @@ test_rev = make_test(only_recentrevs) test_branch = make_test(only_branches) test_builder = make_test(only_builder) - fixed_builder = bool(only_builder) + n_builders = len(only_builder) if only_builder else 0 prune_old = not (only_builds or only_recentrevs or only_builder or only_branches) @@ -781,7 +781,7 @@ else: rev = got_rev buildNumber = build.getNumber() - if fixed_builder: + if n_builders > 0: builds = runs.setdefault((buildNumber, rev), {}) else: builds = runs.setdefault(rev, {}) @@ -858,8 +858,7 @@ only_builder = request.args.get('builder', None) only_builds = None if only_builder is not None: - only_builder = only_builder[-1:] # pick exactly one - page.fixed_builder = True + page.n_builders = len(only_builder) build_select = request.args.get('builds', None) if build_select is not None: only_builds = self._parse_builds(build_select) diff --git a/bot2/pypybuildbot/test/test_summary.py b/bot2/pypybuildbot/test/test_summary.py --- a/bot2/pypybuildbot/test/test_summary.py +++ b/bot2/pypybuildbot/test/test_summary.py @@ -687,6 +687,45 @@ assert 'TEST1' in out + def test_many_builders_query_builder(self): + builder0 = status_builder.BuilderStatus('builder0', None, self.master, '') + builder1 = status_builder.BuilderStatus('builder1', None, self.master, '') + add_builds(builder0, [('60000', "F TEST1\n. b"), + ('60000', ". a\n. b"), + ('60001', "F TEST1\n. b")]) + add_builds(builder1, [('60000', "F TEST1\n. b"), + ('60000', ". a\n. b"), + ('60001', "F TEST1\n. b")]) + + s = summary.Summary() + res = witness_cat_branch(s) + req = FakeRequest([builder0, builder1]) + req.args={'builder': ['builder0', 'builder1']} + out = s.body(req) + cat_branch = res() + + runs = cat_branch[(None, METABRANCH)][0] + assert sorted(runs.keys()) == [(0, '60000'), (1, '60000'), (2, '60001')] + outcome = runs[(0, '60000')]['builder0'] + assert outcome.revision == '60000' + assert outcome.key == ('builder0', 0) + outcome = runs[(1, '60000')]['builder0'] + assert outcome.revision == '60000' + assert outcome.key == ('builder0', 1) + outcome = runs[(2, '60001')]['builder0'] + assert outcome.revision == '60001' + assert outcome.key == ('builder0', 2) + + runs = [] + for m in re.finditer(r'builder=(\w+)&builds=(\d+)', out): + runs.append((m.group(1), int(m.group(2)))) + + assert runs == [('builder1', 0), + ('builder1', 1), + ('builder1', 2)] + + assert 'TEST1' in out + def test_many_builds_query_builder_builds(self): builder = status_builder.BuilderStatus('builder0', None, self.master, '') diff --git a/bot2/pypybuildbot/util.py b/bot2/pypybuildbot/util.py --- a/bot2/pypybuildbot/util.py +++ b/bot2/pypybuildbot/util.py @@ -1,6 +1,6 @@ import os import socket -from buildbot.buildslave.base import log +from twisted.python import log def we_are_debugging(): return socket.gethostname() != 'baroquesoftware' diff --git a/master/templates/layout.html b/master/templates/layout.html --- a/master/templates/layout.html +++ b/master/templates/layout.html @@ -30,6 +30,7 @@ - Summary (py3.5) - Summary (py3.6) - Summary + - RPython - Nightly builds From pypy.commits at gmail.com Fri Apr 13 05:43:55 2018 From: pypy.commits at gmail.com (mattip) Date: Fri, 13 Apr 2018 02:43:55 -0700 (PDT) Subject: [pypy-commit] pypy default: update rlib/rvmprof/shared from upstream, emphasis RPython builds for release Message-ID: <5ad07bdb.e489500a.375be.1957@mx.google.com> Author: Matti Picus Branch: Changeset: r94313:f22145c34985 Date: 2018-04-13 11:07 +0300 http://bitbucket.org/pypy/pypy/changeset/f22145c34985/ Log: update rlib/rvmprof/shared from upstream, emphasis RPython builds for release diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst --- a/pypy/doc/how-to-release.rst +++ b/pypy/doc/how-to-release.rst @@ -40,6 +40,8 @@ sure things are ported back to the trunk and to the branch as necessary. +* Make sure the RPython builds on the buildbot pass with no failures + * Maybe bump the SOABI number in module/imp/importing. This has many implications, so make sure the PyPy community agrees to the change. diff --git a/pypy/tool/release/force-builds.py b/pypy/tool/release/force-builds.py --- a/pypy/tool/release/force-builds.py +++ b/pypy/tool/release/force-builds.py @@ -31,6 +31,9 @@ 'pypy-c-jit-linux-s390x', 'build-pypy-c-jit-linux-armhf-raspbian', 'build-pypy-c-jit-linux-armel', + 'rpython-linux-x86-32', + 'rpython-linux-x86-64' + 'rpython-win-x86-32' ] def get_user(): diff --git a/rpython/rlib/rvmprof/src/shared/_vmprof.c b/rpython/rlib/rvmprof/src/shared/_vmprof.c --- a/rpython/rlib/rvmprof/src/shared/_vmprof.c +++ b/rpython/rlib/rvmprof/src/shared/_vmprof.c @@ -36,6 +36,8 @@ register PY_STACK_FRAME_T * callee_saved asm("rbx"); #elif defined(X86_32) register PY_STACK_FRAME_T * callee_saved asm("edi"); +#elif defined(__arm__) + register PY_STACK_FRAME_T * callee_saved asm("r4"); #else # error "platform not supported" #endif @@ -45,6 +47,8 @@ "movq %1, %0\t\n" #elif defined(X86_32) "mov %1, %0\t\n" +#elif defined(__arm__) + "mov %1, %0\t\n" #else # error "platform not supported" #endif diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.c b/rpython/rlib/rvmprof/src/shared/vmp_stack.c --- a/rpython/rlib/rvmprof/src/shared/vmp_stack.c +++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.c @@ -16,7 +16,7 @@ #ifdef VMP_SUPPORTS_NATIVE_PROFILING -#ifdef VMPROF_LINUX +#if defined(VMPROF_LINUX) || defined(VMPROF_BSD) #include "unwind/vmprof_unwind.h" typedef mcontext_t unw_context_t; @@ -510,13 +510,15 @@ static const char * vmprof_error = NULL; static void * libhandle = NULL; - #ifdef VMPROF_LINUX +#include #define LIBUNWIND "libunwind.so" #ifdef __i386__ #define PREFIX "x86" +#define LIBUNWIND_SUFFIX "" #elif __x86_64__ #define PREFIX "x86_64" +#define LIBUNWIND_SUFFIX "-x86_64" #endif #define U_PREFIX "_U" #define UL_PREFIX "_UL" @@ -524,10 +526,41 @@ int vmp_native_enable(void) { #ifdef VMPROF_LINUX + void * oldhandle = NULL; + struct link_map * map = NULL; if (libhandle == NULL) { + // on linux, the wheel includes the libunwind shared object. + libhandle = dlopen(NULL, RTLD_NOW); + if (libhandle != NULL) { + // load the link map, it will contain an entry to + // .libs_vmprof/libunwind-...so, this is the file that is + // distributed with the wheel. + if (dlinfo(libhandle, RTLD_DI_LINKMAP, &map) != 0) { + (void)dlclose(libhandle); + libhandle = NULL; + goto bail_out; + } + // grab the new handle + do { + if (strstr(map->l_name, ".libs_vmprof/libunwind" LIBUNWIND_SUFFIX) != NULL) { + oldhandle = libhandle; + libhandle = dlopen(map->l_name, RTLD_LAZY|RTLD_LOCAL); + (void)dlclose(oldhandle); + oldhandle = NULL; + goto loaded_libunwind; + } + map = map->l_next; + } while (map != NULL); + // did not find .libs_vmprof/libunwind... + (void)dlclose(libhandle); + libhandle = NULL; + } + + // fallback! try to load the system's libunwind.so if ((libhandle = dlopen(LIBUNWIND, RTLD_LAZY | RTLD_LOCAL)) == NULL) { goto bail_out; } +loaded_libunwind: if ((unw_get_reg = dlsym(libhandle, UL_PREFIX PREFIX "_get_reg")) == NULL) { goto bail_out; } diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_common.h b/rpython/rlib/rvmprof/src/shared/vmprof_common.h --- a/rpython/rlib/rvmprof/src/shared/vmprof_common.h +++ b/rpython/rlib/rvmprof/src/shared/vmprof_common.h @@ -23,6 +23,10 @@ #include #endif +#ifdef VMPROF_BSD +#include +#endif + #define MAX_FUNC_NAME 1024 #ifdef VMPROF_UNIX From pypy.commits at gmail.com Fri Apr 13 16:39:36 2018 From: pypy.commits at gmail.com (rlamy) Date: Fri, 13 Apr 2018 13:39:36 -0700 (PDT) Subject: [pypy-commit] pypy apptest-file: fixes Message-ID: <5ad11588.166a1c0a.806bd.f323@mx.google.com> Author: Ronan Lamy Branch: apptest-file Changeset: r94314:808b8574832a Date: 2018-04-13 21:37 +0100 http://bitbucket.org/pypy/pypy/changeset/808b8574832a/ Log: fixes diff --git a/pypy/tool/pytest/appsupport.py b/pypy/tool/pytest/appsupport.py --- a/pypy/tool/pytest/appsupport.py +++ b/pypy/tool/pytest/appsupport.py @@ -198,7 +198,7 @@ w_BuiltinAssertionError = space.getitem(space.builtin.w_dict, space.wrap('AssertionError')) w_metaclass = space.type(w_BuiltinAssertionError) - w_init = space.wrap(gateway.interp2app_temp(my_init)) + w_init = space.wrap(gateway.interp2app(my_init)) w_dict = space.newdict() space.setitem(w_dict, space.wrap('__init__'), w_init) return space.call_function(w_metaclass, @@ -265,14 +265,14 @@ raise raise oefmt(space.w_AssertionError, "DID NOT RAISE") -app_raises = gateway.interp2app_temp(pypyraises) +app_raises = gateway.interp2app(pypyraises) def pypyskip(space, w_message): """skip a test at app-level. """ msg = space.unwrap(w_message) py.test.skip(msg) -app_skip = gateway.interp2app_temp(pypyskip) +app_skip = gateway.interp2app(pypyskip) def raises_w(space, w_ExpectedException, *args, **kwds): try: diff --git a/pypy/tool/pytest/apptest2.py b/pypy/tool/pytest/apptest2.py --- a/pypy/tool/pytest/apptest2.py +++ b/pypy/tool/pytest/apptest2.py @@ -23,20 +23,21 @@ os.path.join(pypydir, 'tool', 'pytest', 'ast-rewriter')) w_source = space.newtext(source) fname = str(self.fspath) + w_name = space.newtext(str(self.fspath.purebasename)) w_fname = space.newtext(fname) if self.rewrite_asserts: - w_mod = space.appexec([w_rootdir, w_source, w_fname], - """(rootdir, source, fname): + w_mod = space.appexec([w_rootdir, w_source, w_fname, w_name], + """(rootdir, source, fname, name): import sys sys.path.insert(0, rootdir) from ast_rewrite import rewrite_asserts, create_module co = rewrite_asserts(source, fname) - mod = create_module(fname, co) + mod = create_module(name, co) return mod """) else: - w_mod = create_module(space, w_fname, fname, source) + w_mod = create_module(space, w_name, fname, source) mod_dict = w_mod.getdict(space).unwrap(space) items = [] for name, w_obj in mod_dict.items(): @@ -51,7 +52,7 @@ pass def create_module(space, w_name, filename, source): - w_mod = Module(space, w_name) + w_mod = Module(space, w_name, add_package=False) w_dict = w_mod.getdict(space) space.setitem(w_dict, space.newtext('__file__'), space.newtext(filename)) space.exec_(source, w_dict, w_dict, filename=filename) @@ -83,10 +84,9 @@ def execute_appex(self, space, w_func): space.getexecutioncontext().set_sys_exc_info(None) sig = w_func.code._signature - if sig.varargname or sig.kwargname or sig.kwonlyargnames: + if sig.varargname or sig.kwargname: raise ValueError( - 'Test functions may not use *args, **kwargs or ' - 'keyword-only args') + 'Test functions may not use *args or **kwargs') args_w = self.get_fixtures(space, sig.argnames) try: space.call_function(w_func, *args_w) diff --git a/pypy/tool/pytest/ast-rewriter/ast_rewrite.py b/pypy/tool/pytest/ast-rewriter/ast_rewrite.py --- a/pypy/tool/pytest/ast-rewriter/ast_rewrite.py +++ b/pypy/tool/pytest/ast-rewriter/ast_rewrite.py @@ -3,14 +3,21 @@ import ast import itertools import marshal -import struct import sys -from ast_util import assertrepr_compare, format_explanation as _format_explanation +from ast_util import callbinrepr, format_explanation as _format_explanation # pytest caches rewritten pycs in __pycache__. -PYTEST_TAG = sys.implementation.cache_tag + "-PYTEST" +if hasattr(sys, "pypy_version_info"): + impl = "pypy" +elif sys.platform == "java": + impl = "jython" +else: + impl = "cpython" +ver = sys.version_info +PYTEST_TAG = "%s-%s%s-PYTEST" % (impl, ver[0], ver[1]) +del ver, impl PYC_EXT = ".py" + (__debug__ and "c" or "o") PYC_TAIL = "." + PYTEST_TAG + PYC_EXT @@ -29,6 +36,7 @@ # import. However, there's little reason deviate, and I hope # sometime to be able to use imp.load_compiled to load them. (See # the comment in load_module above.) + import struct try: fp = open(pyc, "wb") except IOError: @@ -91,6 +99,7 @@ Return rewritten code if successful or None if not. """ + import struct try: fp = open(pyc, "rb") except IOError: @@ -161,7 +170,7 @@ done = True if done: break - custom = assertrepr_compare(ops[i], each_obj[i], each_obj[i + 1]) + custom = callbinrepr(ops[i], each_obj[i], each_obj[i + 1]) if custom is not None: return custom return expl diff --git a/pypy/tool/pytest/ast-rewriter/ast_util.py b/pypy/tool/pytest/ast-rewriter/ast_util.py --- a/pypy/tool/pytest/ast-rewriter/ast_util.py +++ b/pypy/tool/pytest/ast-rewriter/ast_util.py @@ -10,6 +10,15 @@ _reprcompare = None +# the re-encoding is needed for python2 repr +# with non-ascii characters (see issue 877 and 1379) +def ecu(s): + try: + return u(s, 'utf-8', 'replace') + except TypeError: + return s + + def format_explanation(explanation): """This formats an explanation @@ -20,6 +29,7 @@ for when one explanation needs to span multiple lines, e.g. when displaying diffs. """ + explanation = ecu(explanation) lines = _split_explanation(explanation) result = _format_lines(lines) return '\n'.join(result) @@ -90,6 +100,13 @@ s = s[:maxsize] return s +def callbinrepr(op, left, right): + new_expl = assertrepr_compare(op, left, right) + new_expl = [line.replace("\n", "\\n") for line in new_expl] + res = "\n~".join(new_expl) + res = res.replace("%", "%%") + return res + def assertrepr_compare(op, left, right, verbose=False): """Return specialised explanations for some operators/operands""" diff --git a/pypy/tool/pytest/objspace.py b/pypy/tool/pytest/objspace.py --- a/pypy/tool/pytest/objspace.py +++ b/pypy/tool/pytest/objspace.py @@ -30,6 +30,7 @@ config = make_config(option) if config.objspace.usemodules.thread: config.translation.thread = True + config.objspace.extmodules = 'pypy.tool.pytest.fake_pytest' space = make_objspace(config) space.startup() # Initialize all builtin modules space.setitem(space.builtin.w_dict, space.wrap('AssertionError'), From pypy.commits at gmail.com Fri Apr 13 16:39:39 2018 From: pypy.commits at gmail.com (rlamy) Date: Fri, 13 Apr 2018 13:39:39 -0700 (PDT) Subject: [pypy-commit] pypy apptest-file: Convert test_pyframe.py to new-style apptests Message-ID: <5ad1158b.89c4df0a.182ee.e73c@mx.google.com> Author: Ronan Lamy Branch: apptest-file Changeset: r94315:00ee97dd7467 Date: 2018-04-13 21:38 +0100 http://bitbucket.org/pypy/pypy/changeset/00ee97dd7467/ Log: Convert test_pyframe.py to new-style apptests diff --git a/pypy/interpreter/test/apptest_pyframe.py b/pypy/interpreter/test/apptest_pyframe.py new file mode 100644 --- /dev/null +++ b/pypy/interpreter/test/apptest_pyframe.py @@ -0,0 +1,575 @@ +import pytest + + at pytest.fixture +def tempfile(tmpdir): + return str(tmpdir / 'tempfile1') + +def test_f_locals(): + import sys + f = sys._getframe() + assert f.f_locals is locals() + +def test_f_globals(): + import sys + f = sys._getframe() + assert f.f_globals is globals() + pytest.raises(TypeError, "f.f_globals = globals()") + +def test_f_builtins(): + import sys, __builtin__ + f = sys._getframe() + assert f.f_builtins is __builtin__.__dict__ + +def test_f_code(): + def g(): + import sys + f = sys._getframe() + return f.f_code + assert g() is g.func_code + +def test_f_trace_del(): + import sys + f = sys._getframe() + del f.f_trace + assert f.f_trace is None + +def test_f_lineno(): + def g(): + import sys + f = sys._getframe() + x = f.f_lineno + y = f.f_lineno + z = f.f_lineno + return [x, y, z] + origin = g.func_code.co_firstlineno + assert g() == [origin+3, origin+4, origin+5] + +def test_f_lineno_set(tempfile): + def tracer(f, *args): + def y(f, *args): + return y + def x(f, *args): + f.f_lineno += 1 + return y # "return None" should have the same effect, but see + # test_local_trace_function_returning_None_ignored + return x + + open # force fetching of this name now + + def function(): + xyz + with open(tempfile, 'w') as f: + pass + return 3 + + import sys + sys.settrace(tracer) + function() + sys.settrace(None) + # assert did not crash + +def test_f_lineno_set_firstline(): + seen = [] + def tracer(f, event, *args): + seen.append((event, f.f_lineno)) + if len(seen) == 5: + f.f_lineno = 1 # bug shown only when setting lineno to 1 + return tracer + + def g(): + import sys + sys.settrace(tracer) + exec "x=1\ny=x+1\nz=y+1\nt=z+1\ns=t+1\n" in {} + sys.settrace(None) + + g() + assert seen == [('call', 1), + ('line', 1), + ('line', 2), + ('line', 3), + ('line', 4), + ('line', 2), + ('line', 3), + ('line', 4), + ('line', 5), + ('return', 5)] + +def test_f_back(): + import sys + def f(): + assert sys._getframe().f_code.co_name == g() + def g(): + return sys._getframe().f_back.f_code.co_name + f() + +def test_f_back_virtualref(): + import sys + def f(): + return g() + def g(): + return sys._getframe() + frame = f() + assert frame.f_back.f_code.co_name == 'f' + +def test_f_exc_xxx(): + import sys + + class OuterException(Exception): + pass + class InnerException(Exception): + pass + + def g(exc_info): + f = sys._getframe() + assert f.f_exc_type is None + assert f.f_exc_value is None + assert f.f_exc_traceback is None + try: + raise InnerException + except: + assert f.f_exc_type is exc_info[0] + assert f.f_exc_value is exc_info[1] + assert f.f_exc_traceback is exc_info[2] + try: + raise OuterException + except: + g(sys.exc_info()) + +def test_virtualref_through_traceback(): + import sys + def g(): + try: + raise ValueError + except: + _, _, tb = sys.exc_info() + return tb + def f(): + return g() + # + tb = f() + assert tb.tb_frame.f_code.co_name == 'g' + assert tb.tb_frame.f_back.f_code.co_name == 'f' + +def test_trace_basic(): + import sys + l = [] + class Tracer: + def __init__(self, i): + self.i = i + def trace(self, frame, event, arg): + l.append((self.i, frame.f_code.co_name, event, arg)) + if frame.f_code.co_name == 'g2': + return None # don't trace g2 + return Tracer(self.i+1).trace + def g3(n): + n -= 5 + return n + def g2(n): + n += g3(2) + n += g3(7) + return n + def g(n): + n += g2(3) + return n + def f(n): + n = g(n) + return n * 7 + sys.settrace(Tracer(0).trace) + x = f(4) + sys.settrace(None) + assert x == 42 + print l + assert l == [(0, 'f', 'call', None), + (1, 'f', 'line', None), + (0, 'g', 'call', None), + (1, 'g', 'line', None), + (0, 'g2', 'call', None), + (0, 'g3', 'call', None), + (1, 'g3', 'line', None), + (2, 'g3', 'line', None), + (3, 'g3', 'return', -3), + (0, 'g3', 'call', None), + (1, 'g3', 'line', None), + (2, 'g3', 'line', None), + (3, 'g3', 'return', 2), + (2, 'g', 'line', None), + (3, 'g', 'return', 6), + (2, 'f', 'line', None), + (3, 'f', 'return', 42)] + +def test_trace_exc(): + import sys + l = [] + def ltrace(a,b,c): + if b == 'exception': + l.append(c) + return ltrace + def trace(a,b,c): return ltrace + def f(): + try: + raise Exception + except: + pass + sys.settrace(trace) + f() + sys.settrace(None) + assert len(l) == 1 + assert isinstance(l[0][1], Exception) + +def test_trace_ignore_hidden(): + import sys + import _testing + + l = [] + def trace(a,b,c): + l.append((a,b,c)) + + def f(): + h = _testing.Hidden() + r = h.meth() + return r + + sys.settrace(trace) + res = f() + sys.settrace(None) + assert len(l) == 1 + assert l[0][1] == 'call' + assert res == 'hidden' # sanity + +def test_trace_hidden_prints(tempfile): + import sys + + l = [] + def trace(a,b,c): + l.append((a,b,c)) + return trace + + outputf = open(tempfile, 'w') + def f(): + print >> outputf, 1 + print >> outputf, 2 + print >> outputf, 3 + return "that's the return value" + + sys.settrace(trace) + f() + sys.settrace(None) + outputf.close() + # should get 1 "call", 3 "line" and 1 "return" events, and no call + # or return for the internal app-level implementation of 'print' + assert len(l) == 6 + assert [what for (frame, what, arg) in l] == [ + 'call', 'line', 'line', 'line', 'line', 'return'] + assert l[-1][2] == "that's the return value" + +def test_trace_return_exc(): + import sys + l = [] + def trace(a,b,c): + if b in ('exception', 'return'): + l.append((b, c)) + return trace + + def g(): + raise Exception + def f(): + try: + g() + except: + pass + sys.settrace(trace) + f() + sys.settrace(None) + assert len(l) == 4 + assert l[0][0] == 'exception' + assert isinstance(l[0][1][1], Exception) + assert l[1] == ('return', None) + assert l[2][0] == 'exception' + assert isinstance(l[2][1][1], Exception) + assert l[3] == ('return', None) + +def test_trace_raises_on_return(): + import sys + def trace(frame, event, arg): + if event == 'return': + raise ValueError + else: + return trace + + def f(): return 1 + + for i in xrange(sys.getrecursionlimit() + 1): + sys.settrace(trace) + try: + f() + except ValueError: + pass + +def test_trace_try_finally(): + import sys + l = [] + def trace(frame, event, arg): + if event == 'exception': + l.append(arg) + return trace + + def g(): + try: + raise Exception + finally: + pass + + def f(): + try: + g() + except: + pass + + sys.settrace(trace) + f() + sys.settrace(None) + assert len(l) == 2 + assert issubclass(l[0][0], Exception) + assert issubclass(l[1][0], Exception) + +def test_trace_raise_three_arg(): + import sys + l = [] + def trace(frame, event, arg): + if event == 'exception': + l.append(arg) + return trace + + def g(): + try: + raise Exception + except Exception as e: + import sys + raise Exception, e, sys.exc_info()[2] + + def f(): + try: + g() + except: + pass + + sys.settrace(trace) + f() + sys.settrace(None) + assert len(l) == 2 + assert issubclass(l[0][0], Exception) + assert issubclass(l[1][0], Exception) + +def test_trace_generator_finalisation(): + import sys + l = [] + got_exc = [] + def trace(frame, event, arg): + l.append((frame.f_lineno, event)) + if event == 'exception': + got_exc.append(arg) + return trace + + d = {} + exec """if 1: + def g(): + try: + yield True + finally: + pass + + def f(): + try: + gen = g() + gen.next() + gen.close() + except: + pass + """ in d + f = d['f'] + + sys.settrace(trace) + f() + sys.settrace(None) + assert len(got_exc) == 1 + assert issubclass(got_exc[0][0], GeneratorExit) + assert l == [(8, 'call'), + (9, 'line'), + (10, 'line'), + (11, 'line'), + (2, 'call'), + (3, 'line'), + (4, 'line'), + (4, 'return'), + (12, 'line'), + (4, 'call'), + (4, 'exception'), + (6, 'line'), + (6, 'return'), + (12, 'return')] + +def test_dont_trace_on_reraise(): + import sys + l = [] + def ltrace(a,b,c): + if b == 'exception': + l.append(c) + return ltrace + def trace(a,b,c): return ltrace + def f(): + try: + 1/0 + except: + try: + raise + except: + pass + sys.settrace(trace) + f() + sys.settrace(None) + assert len(l) == 1 + assert issubclass(l[0][0], Exception) + +def test_dont_trace_on_raise_with_tb(): + import sys + l = [] + def ltrace(a,b,c): + if b == 'exception': + l.append(c) + return ltrace + def trace(a,b,c): return ltrace + def f(): + try: + raise Exception + except: + return sys.exc_info() + def g(): + exc, val, tb = f() + try: + raise exc, val, tb + except: + pass + sys.settrace(trace) + g() + sys.settrace(None) + assert len(l) == 1 + assert isinstance(l[0][1], Exception) + +def test_trace_changes_locals(): + import sys + def trace(frame, what, arg): + frame.f_locals['x'] = 42 + return trace + def f(x): + return x + sys.settrace(trace) + res = f(1) + sys.settrace(None) + assert res == 42 + +def test_set_unset_f_trace(): + import sys + seen = [] + def trace1(frame, what, arg): + seen.append((1, frame, frame.f_lineno, what, arg)) + return trace1 + def trace2(frame, what, arg): + seen.append((2, frame, frame.f_lineno, what, arg)) + return trace2 + def set_the_trace(f): + f.f_trace = trace1 + sys.settrace(trace2) + len(seen) # take one line: should not be traced + f = sys._getframe() + set_the_trace(f) + len(seen) # take one line: should not be traced + len(seen) # take one line: should not be traced + sys.settrace(None) # and this line should be the last line traced + len(seen) # take one line + del f.f_trace + len(seen) # take one line + firstline = set_the_trace.func_code.co_firstlineno + assert seen == [(1, f, firstline + 6, 'line', None), + (1, f, firstline + 7, 'line', None), + (1, f, firstline + 8, 'line', None)] + +def test_locals2fast_freevar_bug(): + import sys + def f(n): + class A(object): + def g(self): + return n + n = 42 + return A() + res = f(10).g() + assert res == 10 + # + def trace(*args): + return trace + sys.settrace(trace) + res = f(10).g() + sys.settrace(None) + assert res == 10 + +def test_throw_trace_bug(): + import sys + def f(): + yield 5 + gen = f() + assert next(gen) == 5 + seen = [] + def trace_func(frame, event, *args): + seen.append(event) + return trace_func + sys.settrace(trace_func) + try: + gen.throw(ValueError) + except ValueError: + pass + sys.settrace(None) + assert seen == ['call', 'exception', 'return'] + +def test_generator_trace_stopiteration(): + import sys + def f(): + yield 5 + gen = f() + assert next(gen) == 5 + seen = [] + def trace_func(frame, event, *args): + print('TRACE:', frame, event, args) + seen.append(event) + return trace_func + def g(): + for x in gen: + never_entered + sys.settrace(trace_func) + g() + sys.settrace(None) + print 'seen:', seen + # on Python 3 we get an extra 'exception' when 'for' catches + # StopIteration + assert seen == ['call', 'line', 'call', 'return', 'return'] + +def test_local_trace_function_returning_None_ignored(): + # behave the same as CPython does, and in contradiction with + # the documentation. + def tracer(f, event, arg): + assert event == 'call' + return local_tracer + + seen = [] + def local_tracer(f, event, arg): + seen.append(event) + return None # but 'local_tracer' will be called again + + def function(): + a = 1 + a = 2 + a = 3 + + import sys + sys.settrace(tracer) + function() + sys.settrace(None) + assert seen == ["line", "line", "line", "return"] diff --git a/pypy/interpreter/test/fixtures.py b/pypy/interpreter/test/fixtures.py new file mode 100644 --- /dev/null +++ b/pypy/interpreter/test/fixtures.py @@ -0,0 +1,5 @@ +from _pytest.tmpdir import TempdirFactory + +def tempfile(space, config): + tmpdir = TempdirFactory(config).getbasetemp() + return space.newtext(str(tmpdir / 'tempfile1')) diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py --- a/pypy/interpreter/test/test_pyframe.py +++ b/pypy/interpreter/test/test_pyframe.py @@ -1,4 +1,3 @@ -from rpython.tool import udir from pypy.conftest import option from pypy.interpreter.gateway import interp2app @@ -9,8 +8,6 @@ def setup_class(cls): space = cls.space - cls.w_udir = cls.space.wrap(str(udir.udir)) - cls.w_tempfile1 = cls.space.wrap(str(udir.udir.join('tempfile1'))) if not option.runappdirect: w_call_further = cls.space.appexec([], """(): def call_further(f): @@ -25,113 +22,6 @@ # test for the presence of the attributes, not functionality - def test_f_locals(self): - import sys - f = sys._getframe() - assert f.f_locals is locals() - - def test_f_globals(self): - import sys - f = sys._getframe() - assert f.f_globals is globals() - raises(TypeError, "f.f_globals = globals()") - - def test_f_builtins(self): - import sys, __builtin__ - f = sys._getframe() - assert f.f_builtins is __builtin__.__dict__ - - def test_f_code(self): - def g(): - import sys - f = sys._getframe() - return f.f_code - assert g() is g.func_code - - def test_f_trace_del(self): - import sys - f = sys._getframe() - del f.f_trace - assert f.f_trace is None - - def test_f_lineno(self): - def g(): - import sys - f = sys._getframe() - x = f.f_lineno - y = f.f_lineno - z = f.f_lineno - return [x, y, z] - origin = g.func_code.co_firstlineno - assert g() == [origin+3, origin+4, origin+5] - - def test_f_lineno_set(self): - def tracer(f, *args): - def y(f, *args): - return y - def x(f, *args): - f.f_lineno += 1 - return y # "return None" should have the same effect, but see - # test_local_trace_function_returning_None_ignored - return x - - open # force fetching of this name now - - def function(): - xyz - with open(self.tempfile1, 'w') as f: - pass - return 3 - - import sys - sys.settrace(tracer) - function() - sys.settrace(None) - # assert did not crash - - def test_f_lineno_set_firstline(self): - seen = [] - def tracer(f, event, *args): - seen.append((event, f.f_lineno)) - if len(seen) == 5: - f.f_lineno = 1 # bug shown only when setting lineno to 1 - return tracer - - def g(): - import sys - sys.settrace(tracer) - exec "x=1\ny=x+1\nz=y+1\nt=z+1\ns=t+1\n" in {} - sys.settrace(None) - - g() - assert seen == [('call', 1), - ('line', 1), - ('line', 2), - ('line', 3), - ('line', 4), - ('line', 2), - ('line', 3), - ('line', 4), - ('line', 5), - ('return', 5)] - - def test_f_back(self): - import sys - def f(): - assert sys._getframe().f_code.co_name == g() - def g(): - return sys._getframe().f_back.f_code.co_name - f() - - def test_f_back_virtualref(self): - import sys - def f(): - return g() - def g(): - return sys._getframe() - frame = f() - assert frame.f_back.f_code.co_name == 'f' - def test_f_back_hidden(self): if not hasattr(self, 'call_further'): skip("not for runappdirect testing") @@ -148,362 +38,6 @@ assert f1bis is f1 assert f0.f_back is f1 - def test_f_exc_xxx(self): - import sys - - class OuterException(Exception): - pass - class InnerException(Exception): - pass - - def g(exc_info): - f = sys._getframe() - assert f.f_exc_type is None - assert f.f_exc_value is None - assert f.f_exc_traceback is None - try: - raise InnerException - except: - assert f.f_exc_type is exc_info[0] - assert f.f_exc_value is exc_info[1] - assert f.f_exc_traceback is exc_info[2] - try: - raise OuterException - except: - g(sys.exc_info()) - - def test_virtualref_through_traceback(self): - import sys - def g(): - try: - raise ValueError - except: - _, _, tb = sys.exc_info() - return tb - def f(): - return g() - # - tb = f() - assert tb.tb_frame.f_code.co_name == 'g' - assert tb.tb_frame.f_back.f_code.co_name == 'f' - - def test_trace_basic(self): - import sys - l = [] - class Tracer: - def __init__(self, i): - self.i = i - def trace(self, frame, event, arg): - l.append((self.i, frame.f_code.co_name, event, arg)) - if frame.f_code.co_name == 'g2': - return None # don't trace g2 - return Tracer(self.i+1).trace - def g3(n): - n -= 5 - return n - def g2(n): - n += g3(2) - n += g3(7) - return n - def g(n): - n += g2(3) - return n - def f(n): - n = g(n) - return n * 7 - sys.settrace(Tracer(0).trace) - x = f(4) - sys.settrace(None) - assert x == 42 - print l - assert l == [(0, 'f', 'call', None), - (1, 'f', 'line', None), - (0, 'g', 'call', None), - (1, 'g', 'line', None), - (0, 'g2', 'call', None), - (0, 'g3', 'call', None), - (1, 'g3', 'line', None), - (2, 'g3', 'line', None), - (3, 'g3', 'return', -3), - (0, 'g3', 'call', None), - (1, 'g3', 'line', None), - (2, 'g3', 'line', None), - (3, 'g3', 'return', 2), - (2, 'g', 'line', None), - (3, 'g', 'return', 6), - (2, 'f', 'line', None), - (3, 'f', 'return', 42)] - - def test_trace_exc(self): - import sys - l = [] - def ltrace(a,b,c): - if b == 'exception': - l.append(c) - return ltrace - def trace(a,b,c): return ltrace - def f(): - try: - raise Exception - except: - pass - sys.settrace(trace) - f() - sys.settrace(None) - assert len(l) == 1 - assert isinstance(l[0][1], Exception) - - def test_trace_ignore_hidden(self): - import sys - import _testing - - l = [] - def trace(a,b,c): - l.append((a,b,c)) - - def f(): - h = _testing.Hidden() - r = h.meth() - return r - - sys.settrace(trace) - res = f() - sys.settrace(None) - assert len(l) == 1 - assert l[0][1] == 'call' - assert res == 'hidden' # sanity - - def test_trace_hidden_prints(self): - import sys - - l = [] - def trace(a,b,c): - l.append((a,b,c)) - return trace - - outputf = open(self.tempfile1, 'w') - def f(): - print >> outputf, 1 - print >> outputf, 2 - print >> outputf, 3 - return "that's the return value" - - sys.settrace(trace) - f() - sys.settrace(None) - outputf.close() - # should get 1 "call", 3 "line" and 1 "return" events, and no call - # or return for the internal app-level implementation of 'print' - assert len(l) == 6 - assert [what for (frame, what, arg) in l] == [ - 'call', 'line', 'line', 'line', 'line', 'return'] - assert l[-1][2] == "that's the return value" - - def test_trace_return_exc(self): - import sys - l = [] - def trace(a,b,c): - if b in ('exception', 'return'): - l.append((b, c)) - return trace - - def g(): - raise Exception - def f(): - try: - g() - except: - pass - sys.settrace(trace) - f() - sys.settrace(None) - assert len(l) == 4 - assert l[0][0] == 'exception' - assert isinstance(l[0][1][1], Exception) - assert l[1] == ('return', None) - assert l[2][0] == 'exception' - assert isinstance(l[2][1][1], Exception) - assert l[3] == ('return', None) - - def test_trace_raises_on_return(self): - import sys - def trace(frame, event, arg): - if event == 'return': - raise ValueError - else: - return trace - - def f(): return 1 - - for i in xrange(sys.getrecursionlimit() + 1): - sys.settrace(trace) - try: - f() - except ValueError: - pass - - def test_trace_try_finally(self): - import sys - l = [] - def trace(frame, event, arg): - if event == 'exception': - l.append(arg) - return trace - - def g(): - try: - raise Exception - finally: - pass - - def f(): - try: - g() - except: - pass - - sys.settrace(trace) - f() - sys.settrace(None) - assert len(l) == 2 - assert issubclass(l[0][0], Exception) - assert issubclass(l[1][0], Exception) - - def test_trace_raise_three_arg(self): - import sys - l = [] - def trace(frame, event, arg): - if event == 'exception': - l.append(arg) - return trace - - def g(): - try: - raise Exception - except Exception as e: - import sys - raise Exception, e, sys.exc_info()[2] - - def f(): - try: - g() - except: - pass - - sys.settrace(trace) - f() - sys.settrace(None) - assert len(l) == 2 - assert issubclass(l[0][0], Exception) - assert issubclass(l[1][0], Exception) - - def test_trace_generator_finalisation(self): - import sys - l = [] - got_exc = [] - def trace(frame, event, arg): - l.append((frame.f_lineno, event)) - if event == 'exception': - got_exc.append(arg) - return trace - - d = {} - exec """if 1: - def g(): - try: - yield True - finally: - pass - - def f(): - try: - gen = g() - gen.next() - gen.close() - except: - pass - """ in d - f = d['f'] - - sys.settrace(trace) - f() - sys.settrace(None) - assert len(got_exc) == 1 - assert issubclass(got_exc[0][0], GeneratorExit) - assert l == [(8, 'call'), - (9, 'line'), - (10, 'line'), - (11, 'line'), - (2, 'call'), - (3, 'line'), - (4, 'line'), - (4, 'return'), - (12, 'line'), - (4, 'call'), - (4, 'exception'), - (6, 'line'), - (6, 'return'), - (12, 'return')] - - def test_dont_trace_on_reraise(self): - import sys - l = [] - def ltrace(a,b,c): - if b == 'exception': - l.append(c) - return ltrace - def trace(a,b,c): return ltrace - def f(): - try: - 1/0 - except: - try: - raise - except: - pass - sys.settrace(trace) - f() - sys.settrace(None) - assert len(l) == 1 - assert issubclass(l[0][0], Exception) - - def test_dont_trace_on_raise_with_tb(self): - import sys - l = [] - def ltrace(a,b,c): - if b == 'exception': - l.append(c) - return ltrace - def trace(a,b,c): return ltrace - def f(): - try: - raise Exception - except: - return sys.exc_info() - def g(): - exc, val, tb = f() - try: - raise exc, val, tb - except: - pass - sys.settrace(trace) - g() - sys.settrace(None) - assert len(l) == 1 - assert isinstance(l[0][1], Exception) - - def test_trace_changes_locals(self): - import sys - def trace(frame, what, arg): - frame.f_locals['x'] = 42 - return trace - def f(x): - return x - sys.settrace(trace) - res = f(1) - sys.settrace(None) - assert res == 42 - def test_fast2locals_called_lazily(self): import sys class FrameHolder: @@ -522,110 +56,3 @@ assert res == 2 if hasattr(self, "check_no_w_locals"): # not appdirect assert self.check_no_w_locals(fh.frame) - - def test_set_unset_f_trace(self): - import sys - seen = [] - def trace1(frame, what, arg): - seen.append((1, frame, frame.f_lineno, what, arg)) - return trace1 - def trace2(frame, what, arg): - seen.append((2, frame, frame.f_lineno, what, arg)) - return trace2 - def set_the_trace(f): - f.f_trace = trace1 - sys.settrace(trace2) - len(seen) # take one line: should not be traced - f = sys._getframe() - set_the_trace(f) - len(seen) # take one line: should not be traced - len(seen) # take one line: should not be traced - sys.settrace(None) # and this line should be the last line traced - len(seen) # take one line - del f.f_trace - len(seen) # take one line - firstline = set_the_trace.func_code.co_firstlineno - assert seen == [(1, f, firstline + 6, 'line', None), - (1, f, firstline + 7, 'line', None), - (1, f, firstline + 8, 'line', None)] - - def test_locals2fast_freevar_bug(self): - import sys - def f(n): - class A(object): - def g(self): - return n - n = 42 - return A() - res = f(10).g() - assert res == 10 - # - def trace(*args): - return trace - sys.settrace(trace) - res = f(10).g() - sys.settrace(None) - assert res == 10 - - def test_throw_trace_bug(self): - import sys - def f(): - yield 5 - gen = f() - assert next(gen) == 5 - seen = [] - def trace_func(frame, event, *args): - seen.append(event) - return trace_func - sys.settrace(trace_func) - try: - gen.throw(ValueError) - except ValueError: - pass - sys.settrace(None) - assert seen == ['call', 'exception', 'return'] - - def test_generator_trace_stopiteration(self): - import sys - def f(): - yield 5 - gen = f() - assert next(gen) == 5 - seen = [] - def trace_func(frame, event, *args): - print('TRACE:', frame, event, args) - seen.append(event) - return trace_func - def g(): - for x in gen: - never_entered - sys.settrace(trace_func) - g() - sys.settrace(None) - print 'seen:', seen - # on Python 3 we get an extra 'exception' when 'for' catches - # StopIteration - assert seen == ['call', 'line', 'call', 'return', 'return'] - - def test_local_trace_function_returning_None_ignored(self): - # behave the same as CPython does, and in contradiction with - # the documentation. - def tracer(f, event, arg): - assert event == 'call' - return local_tracer - - seen = [] - def local_tracer(f, event, arg): - seen.append(event) - return None # but 'local_tracer' will be called again - - def function(): - a = 1 - a = 2 - a = 3 - - import sys - sys.settrace(tracer) - function() - sys.settrace(None) - assert seen == ["line", "line", "line", "return"] From pypy.commits at gmail.com Sat Apr 14 05:21:20 2018 From: pypy.commits at gmail.com (cfbolz) Date: Sat, 14 Apr 2018 02:21:20 -0700 (PDT) Subject: [pypy-commit] pypy pyparser-improvements-3: introduce a Token class instead of passing 5 values around all the time Message-ID: <5ad1c810.90d1df0a.6d2cb.255d@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: pyparser-improvements-3 Changeset: r94316:1065b72e0409 Date: 2018-04-14 10:56 +0200 http://bitbucket.org/pypy/pypy/changeset/1065b72e0409/ Log: introduce a Token class instead of passing 5 values around all the time diff --git a/pypy/interpreter/pyparser/parser.py b/pypy/interpreter/pyparser/parser.py --- a/pypy/interpreter/pyparser/parser.py +++ b/pypy/interpreter/pyparser/parser.py @@ -65,6 +65,20 @@ b[pos] |= bit return str(b) + +class Token(object): + def __init__(self, token_type, value, lineno, column, line): + self.token_type = token_type + self.value = value + self.lineno = lineno + # 0-based offset + self.column = column + self.line = line + + def __repr__(self): + return "Token(%s, %s)" % (self.token_type, self.value) + + class Node(object): __slots__ = ("type", ) @@ -99,11 +113,11 @@ class Terminal(Node): __slots__ = ("value", "lineno", "column") - def __init__(self, type, value, lineno, column): - Node.__init__(self, type) - self.value = value - self.lineno = lineno - self.column = column + def __init__(self, token): + Node.__init__(self, token.token_type) + self.value = token.value + self.lineno = token.lineno + self.column = token.column def __repr__(self): return "Terminal(type=%s, value=%r)" % (self.type, self.value) @@ -193,20 +207,14 @@ class ParseError(Exception): - def __init__(self, msg, token_type, value, lineno, column, line, - expected=-1, expected_str=None): + def __init__(self, msg, token, expected=-1, expected_str=None): self.msg = msg - self.token_type = token_type - self.value = value - self.lineno = lineno - # this is a 0-based index - self.column = column - self.line = line + self.token = token self.expected = expected self.expected_str = expected_str def __str__(self): - return "ParserError(%s, %r)" % (self.token_type, self.value) + return "ParserError(%s)" % (self.token, ) class StackEntry(object): @@ -249,8 +257,8 @@ self.root = None self.stack = StackEntry(None, self.grammar.dfas[start - 256], 0) - def add_token(self, token_type, value, lineno, column, line): - label_index = self.classify(token_type, value, lineno, column, line) + def add_token(self, token): + label_index = self.classify(token) sym_id = 0 # for the annotator while True: dfa = self.stack.dfa @@ -261,7 +269,7 @@ sym_id = self.grammar.labels[i] if label_index == i: # We matched a non-terminal. - self.shift(next_state, token_type, value, lineno, column) + self.shift(next_state, token) state = states[next_state] # While the only possible action is to accept, pop nodes off # the stack. @@ -278,8 +286,7 @@ sub_node_dfa = self.grammar.dfas[sym_id - 256] # Check if this token can start a child node. if sub_node_dfa.could_match_token(label_index): - self.push(sub_node_dfa, next_state, sym_id, lineno, - column) + self.push(sub_node_dfa, next_state, sym_id) break else: # We failed to find any arcs to another state, so unless this @@ -287,8 +294,7 @@ if is_accepting: self.pop() if self.stack is None: - raise ParseError("too much input", token_type, value, - lineno, column, line) + raise ParseError("too much input", token) else: # If only one possible input would satisfy, attach it to the # error. @@ -299,28 +305,26 @@ else: expected = -1 expected_str = None - raise ParseError("bad input", token_type, value, lineno, - column, line, expected, expected_str) + raise ParseError("bad input", token, expected, expected_str) - def classify(self, token_type, value, lineno, column, line): + def classify(self, token): """Find the label for a token.""" - if token_type == self.grammar.KEYWORD_TOKEN: - label_index = self.grammar.keyword_ids.get(value, -1) + if token.token_type == self.grammar.KEYWORD_TOKEN: + label_index = self.grammar.keyword_ids.get(token.value, -1) if label_index != -1: return label_index - label_index = self.grammar.token_ids.get(token_type, -1) + label_index = self.grammar.token_ids.get(token.token_type, -1) if label_index == -1: - raise ParseError("invalid token", token_type, value, lineno, column, - line) + raise ParseError("invalid token", token) return label_index - def shift(self, next_state, token_type, value, lineno, column): + def shift(self, next_state, token): """Shift a non-terminal and prepare for the next state.""" - new_node = Terminal(token_type, value, lineno, column) + new_node = Terminal(token) self.stack.node_append_child(new_node) self.stack.state = next_state - def push(self, next_dfa, next_state, node_type, lineno, column): + def push(self, next_dfa, next_state, node_type): """Push a terminal and adjust the current state.""" self.stack.state = next_state self.stack = self.stack.push(next_dfa, 0) diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py --- a/pypy/interpreter/pyparser/pyparse.py +++ b/pypy/interpreter/pyparser/pyparse.py @@ -166,7 +166,7 @@ self.grammar = pygram.python_grammar for tp, value, lineno, column, line in tokens: - if self.add_token(tp, value, lineno, column, line): + if self.add_token(parser.Token(tp, value, lineno, column, line)): break except error.TokenError as e: e.filename = compile_info.filename @@ -190,7 +190,7 @@ # parser.ParseError(...).column is 0-based, but the offsets in the # exceptions in the error module are 1-based, hence the '+ 1' - raise new_err(msg, e.lineno, e.column + 1, e.line, + raise new_err(msg, e.token.lineno, e.token.column + 1, e.token.line, compile_info.filename) else: tree = self.root diff --git a/pypy/interpreter/pyparser/test/test_parser.py b/pypy/interpreter/pyparser/test/test_parser.py --- a/pypy/interpreter/pyparser/test/test_parser.py +++ b/pypy/interpreter/pyparser/test/test_parser.py @@ -20,7 +20,7 @@ rl = StringIO.StringIO(input + "\n").readline gen = tokenize.generate_tokens(rl) for tp, value, begin, end, line in gen: - if self.add_token(tp, value, begin[0], begin[1], line): + if self.add_token(parser.Token(tp, value, begin[0], begin[1], line)): py.test.raises(StopIteration, gen.next) return self.root @@ -58,7 +58,7 @@ value = "\n" else: value = "" - n = parser.Terminal(tp, value, 0, 0) + n = parser.Terminal(parser.Token(tp, value, 0, 0, '')) else: tp = gram.symbol_ids[data[0]] n = parser.Nonterminal(tp) From pypy.commits at gmail.com Sat Apr 14 05:21:22 2018 From: pypy.commits at gmail.com (cfbolz) Date: Sat, 14 Apr 2018 02:21:22 -0700 (PDT) Subject: [pypy-commit] pypy pyparser-improvements-3: use Token class in pytokenizer too Message-ID: <5ad1c812.c1241c0a.5ed2f.ba24@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: pyparser-improvements-3 Changeset: r94317:d965ef8c054e Date: 2018-04-14 11:06 +0200 http://bitbucket.org/pypy/pypy/changeset/d965ef8c054e/ Log: use Token class in pytokenizer too diff --git a/pypy/interpreter/pyparser/future.py b/pypy/interpreter/pyparser/future.py --- a/pypy/interpreter/pyparser/future.py +++ b/pypy/interpreter/pyparser/future.py @@ -43,7 +43,7 @@ self.tok = self.tokens[index] def skip(self, n): - if self.tok[0] == n: + if self.tok.token_type == n: self.next() return True else: @@ -51,7 +51,7 @@ def skip_name(self, name): from pypy.interpreter.pyparser import pygram - if self.tok[0] == pygram.tokens.NAME and self.tok[1] == name: + if self.tok.token_type == pygram.tokens.NAME and self.tok.value == name: self.next() return True else: @@ -59,8 +59,8 @@ def next_feature_name(self): from pypy.interpreter.pyparser import pygram - if self.tok[0] == pygram.tokens.NAME: - name = self.tok[1] + if self.tok.token_type == pygram.tokens.NAME: + name = self.tok.value self.next() if self.skip_name("as"): self.skip(pygram.tokens.NAME) @@ -101,7 +101,7 @@ # somewhere inside the last __future__ import statement # (at the start would be fine too, but it's easier to grab a # random position inside) - last_position = (it.tok[2], it.tok[3]) + last_position = (it.tok.lineno, it.tok.column) result |= future_flags.get_compiler_feature(it.next_feature_name()) while it.skip(pygram.tokens.COMMA): result |= future_flags.get_compiler_feature(it.next_feature_name()) diff --git a/pypy/interpreter/pyparser/parser.py b/pypy/interpreter/pyparser/parser.py --- a/pypy/interpreter/pyparser/parser.py +++ b/pypy/interpreter/pyparser/parser.py @@ -78,6 +78,19 @@ def __repr__(self): return "Token(%s, %s)" % (self.token_type, self.value) + def __eq__(self, other): + # for tests + return ( + self.token_type == other.token_type and + self.value == other.value and + self.lineno == other.lineno and + self.column == other.column and + self.line == other.line + ) + + def __ne__(self, other): + return not self == other + class Node(object): diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py --- a/pypy/interpreter/pyparser/pyparse.py +++ b/pypy/interpreter/pyparser/pyparse.py @@ -147,7 +147,6 @@ flags &= ~consts.PyCF_DONT_IMPLY_DEDENT self.prepare(_targets[compile_info.mode]) - tp = 0 try: try: # Note: we no longer pass the CO_FUTURE_* to the tokenizer, @@ -165,8 +164,8 @@ else: self.grammar = pygram.python_grammar - for tp, value, lineno, column, line in tokens: - if self.add_token(parser.Token(tp, value, lineno, column, line)): + for token in tokens: + if self.add_token(token): break except error.TokenError as e: e.filename = compile_info.filename @@ -178,7 +177,7 @@ # Catch parse errors, pretty them up and reraise them as a # SyntaxError. new_err = error.IndentationError - if tp == pygram.tokens.INDENT: + if token.token_type == pygram.tokens.INDENT: msg = "unexpected indent" elif e.expected == pygram.tokens.INDENT: msg = "expected an indented block" diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py --- a/pypy/interpreter/pyparser/pytokenizer.py +++ b/pypy/interpreter/pyparser/pytokenizer.py @@ -1,4 +1,5 @@ from pypy.interpreter.pyparser import automata +from pypy.interpreter.pyparser.parser import Token from pypy.interpreter.pyparser.pygram import tokens from pypy.interpreter.pyparser.pytoken import python_opmap from pypy.interpreter.pyparser.error import TokenError, TokenIndentationError @@ -103,7 +104,7 @@ endmatch = endDFA.recognize(line) if endmatch >= 0: pos = end = endmatch - tok = (tokens.STRING, contstr + line[:end], strstart[0], + tok = Token(tokens.STRING, contstr + line[:end], strstart[0], strstart[1], line) token_list.append(tok) last_comment = '' @@ -111,7 +112,7 @@ contline = None elif (needcont and not line.endswith('\\\n') and not line.endswith('\\\r\n')): - tok = (tokens.ERRORTOKEN, contstr + line, strstart[0], + tok = Token(tokens.ERRORTOKEN, contstr + line, strstart[0], strstart[1], line) token_list.append(tok) last_comment = '' @@ -140,11 +141,11 @@ if column > indents[-1]: # count indents or dedents indents.append(column) - token_list.append((tokens.INDENT, line[:pos], lnum, 0, line)) + token_list.append(Token(tokens.INDENT, line[:pos], lnum, 0, line)) last_comment = '' while column < indents[-1]: indents.pop() - token_list.append((tokens.DEDENT, '', lnum, pos, line)) + token_list.append(Token(tokens.DEDENT, '', lnum, pos, line)) last_comment = '' if column != indents[-1]: err = "unindent does not match any outer indentation level" @@ -177,11 +178,11 @@ token, initial = line[start:end], line[start] if initial in numchars or \ (initial == '.' and token != '.'): # ordinary number - token_list.append((tokens.NUMBER, token, lnum, start, line)) + token_list.append(Token(tokens.NUMBER, token, lnum, start, line)) last_comment = '' elif initial in '\r\n': if not parenstack: - tok = (tokens.NEWLINE, last_comment, lnum, start, line) + tok = Token(tokens.NEWLINE, last_comment, lnum, start, line) token_list.append(tok) last_comment = '' elif initial == '#': @@ -193,7 +194,7 @@ if endmatch >= 0: # all on one line pos = endmatch token = line[start:pos] - tok = (tokens.STRING, token, lnum, start, line) + tok = Token(tokens.STRING, token, lnum, start, line) token_list.append(tok) last_comment = '' else: @@ -212,11 +213,11 @@ contline = line break else: # ordinary string - tok = (tokens.STRING, token, lnum, start, line) + tok = Token(tokens.STRING, token, lnum, start, line) token_list.append(tok) last_comment = '' elif initial in namechars: # ordinary name - token_list.append((tokens.NAME, token, lnum, start, line)) + token_list.append(Token(tokens.NAME, token, lnum, start, line)) last_comment = '' elif initial == '\\': # continued stmt continued = 1 @@ -242,7 +243,7 @@ punct = python_opmap[token] else: punct = tokens.OP - token_list.append((punct, token, lnum, start, line)) + token_list.append(Token(punct, token, lnum, start, line)) last_comment = '' else: start = whiteSpaceDFA.recognize(line, pos) @@ -251,22 +252,22 @@ if start Author: Carl Friedrich Bolz-Tereick Branch: pyparser-improvements-3 Changeset: r94318:d14ff1f951e6 Date: 2018-04-14 11:19 +0200 http://bitbucket.org/pypy/pypy/changeset/d14ff1f951e6/ Log: move classify method to Grammar where it makes more sense diff --git a/pypy/interpreter/pyparser/parser.py b/pypy/interpreter/pyparser/parser.py --- a/pypy/interpreter/pyparser/parser.py +++ b/pypy/interpreter/pyparser/parser.py @@ -33,6 +33,18 @@ new.token_ids = self.token_ids return new + + def classify(self, token): + """Find the label for a token.""" + if token.token_type == self.KEYWORD_TOKEN: + label_index = self.keyword_ids.get(token.value, -1) + if label_index != -1: + return label_index + label_index = self.token_ids.get(token.token_type, -1) + if label_index == -1: + raise ParseError("invalid token", token) + return label_index + def _freeze_(self): # Remove some attributes not used in parsing. try: @@ -271,7 +283,7 @@ self.stack = StackEntry(None, self.grammar.dfas[start - 256], 0) def add_token(self, token): - label_index = self.classify(token) + label_index = self.grammar.classify(token) sym_id = 0 # for the annotator while True: dfa = self.stack.dfa @@ -320,16 +332,6 @@ expected_str = None raise ParseError("bad input", token, expected, expected_str) - def classify(self, token): - """Find the label for a token.""" - if token.token_type == self.grammar.KEYWORD_TOKEN: - label_index = self.grammar.keyword_ids.get(token.value, -1) - if label_index != -1: - return label_index - label_index = self.grammar.token_ids.get(token.token_type, -1) - if label_index == -1: - raise ParseError("invalid token", token) - return label_index def shift(self, next_state, token): """Shift a non-terminal and prepare for the next state.""" From pypy.commits at gmail.com Sat Apr 14 05:21:26 2018 From: pypy.commits at gmail.com (cfbolz) Date: Sat, 14 Apr 2018 02:21:26 -0700 (PDT) Subject: [pypy-commit] pypy pyparser-improvements-3: less code in except block Message-ID: <5ad1c816.db85df0a.18b6.e2b1@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: pyparser-improvements-3 Changeset: r94319:2b67feca9fbd Date: 2018-04-14 11:20 +0200 http://bitbucket.org/pypy/pypy/changeset/2b67feca9fbd/ Log: less code in except block diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py --- a/pypy/interpreter/pyparser/pyparse.py +++ b/pypy/interpreter/pyparser/pyparse.py @@ -153,26 +153,27 @@ # which is expected to work independently of them. It's # certainly the case for all futures in Python <= 2.7. tokens = pytokenizer.generate_tokens(source_lines, flags) - - newflags, last_future_import = ( - future.add_future_flags(self.future_flags, tokens)) - compile_info.last_future_import = last_future_import - compile_info.flags |= newflags - - if compile_info.flags & consts.CO_FUTURE_PRINT_FUNCTION: - self.grammar = pygram.python_grammar_no_print - else: - self.grammar = pygram.python_grammar - - for token in tokens: - if self.add_token(token): - break except error.TokenError as e: e.filename = compile_info.filename raise except error.TokenIndentationError as e: e.filename = compile_info.filename raise + + newflags, last_future_import = ( + future.add_future_flags(self.future_flags, tokens)) + compile_info.last_future_import = last_future_import + compile_info.flags |= newflags + + if compile_info.flags & consts.CO_FUTURE_PRINT_FUNCTION: + self.grammar = pygram.python_grammar_no_print + else: + self.grammar = pygram.python_grammar + + try: + for token in tokens: + if self.add_token(token): + break except parser.ParseError as e: # Catch parse errors, pretty them up and reraise them as a # SyntaxError. From pypy.commits at gmail.com Sat Apr 14 05:57:15 2018 From: pypy.commits at gmail.com (cfbolz) Date: Sat, 14 Apr 2018 02:57:15 -0700 (PDT) Subject: [pypy-commit] pypy pyparser-improvements-3: add at least a very minimal recognize test Message-ID: <5ad1d07b.95b5df0a.b4e85.a0c7@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: pyparser-improvements-3 Changeset: r94320:2b4152c6c930 Date: 2018-04-14 11:56 +0200 http://bitbucket.org/pypy/pypy/changeset/2b4152c6c930/ Log: add at least a very minimal recognize test diff --git a/pypy/interpreter/pyparser/test/test_automata.py b/pypy/interpreter/pyparser/test/test_automata.py --- a/pypy/interpreter/pyparser/test/test_automata.py +++ b/pypy/interpreter/pyparser/test/test_automata.py @@ -1,4 +1,4 @@ -from pypy.interpreter.pyparser.automata import DFA, DEFAULT +from pypy.interpreter.pyparser.automata import DFA, NonGreedyDFA, DEFAULT def test_states(): d = DFA([{"\x00": 1}, {"\x01": 0}], [False, True]) @@ -10,3 +10,20 @@ assert d.states == "\x01\x00" assert d.defaults == "\xff\x00" assert d.max_char == 1 + +def test_recognize(): + d = DFA([{"a": 1}, {"b": 0}], [False, True]) + assert d.recognize("ababab") == 5 + assert d.recognize("c") == -1 + + d = DFA([{"a": 1}, {DEFAULT: 0}], [False, True]) + assert d.recognize("a,a?ab") == 5 + assert d.recognize("c") == -1 + + d = NonGreedyDFA([{"a": 1}, {"b": 0}], [False, True]) + assert d.recognize("ababab") == 1 + assert d.recognize("c") == -1 + + d = NonGreedyDFA([{"a": 1}, {DEFAULT: 0}], [False, True]) + assert d.recognize("a,a?ab") == 1 + assert d.recognize("c") == -1 From pypy.commits at gmail.com Sat Apr 14 06:54:34 2018 From: pypy.commits at gmail.com (cfbolz) Date: Sat, 14 Apr 2018 03:54:34 -0700 (PDT) Subject: [pypy-commit] pypy pyparser-improvements-3: manually transplant 1e634f696054 to python2 pyparser: Message-ID: <5ad1ddea.9885df0a.d8cf6.a564@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: pyparser-improvements-3 Changeset: r94321:5c8a5d8bb882 Date: 2018-04-14 12:53 +0200 http://bitbucket.org/pypy/pypy/changeset/5c8a5d8bb882/ Log: manually transplant 1e634f696054 to python2 pyparser: (antocuni, romain): use a better way to regenerate the DFA machinery: earlier you had to manually copy&paste lines into pytokenize.py, now you can just redirect the output to dfa_generated.py diff --git a/pypy/interpreter/pyparser/dfa_generated.py b/pypy/interpreter/pyparser/dfa_generated.py new file mode 100644 --- /dev/null +++ b/pypy/interpreter/pyparser/dfa_generated.py @@ -0,0 +1,290 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY gendfa.py +# DO NOT EDIT +# TO REGENERATE THE FILE, RUN: +# python gendfa.py > dfa_generated.py + +from pypy.interpreter.pyparser import automata +accepts = [True, True, True, True, True, True, True, True, + True, True, False, True, True, True, True, False, + False, False, True, False, False, True, False, + False, True, False, True, False, True, False, + False, True, False, False, True, True, True, + False, False, True, False, False, False, True] +states = [ + # 0 + {'\t': 0, '\n': 13, '\x0c': 0, + '\r': 14, ' ': 0, '!': 10, '"': 16, + '#': 18, '%': 12, '&': 12, "'": 15, + '(': 13, ')': 13, '*': 7, '+': 12, + ',': 13, '-': 12, '.': 6, '/': 11, + '0': 4, '1': 5, '2': 5, '3': 5, + '4': 5, '5': 5, '6': 5, '7': 5, + '8': 5, '9': 5, ':': 13, ';': 13, + '<': 9, '=': 12, '>': 8, '@': 13, + 'A': 1, 'B': 2, 'C': 1, 'D': 1, + 'E': 1, 'F': 1, 'G': 1, 'H': 1, + 'I': 1, 'J': 1, 'K': 1, 'L': 1, + 'M': 1, 'N': 1, 'O': 1, 'P': 1, + 'Q': 1, 'R': 3, 'S': 1, 'T': 1, + 'U': 2, 'V': 1, 'W': 1, 'X': 1, + 'Y': 1, 'Z': 1, '[': 13, '\\': 17, + ']': 13, '^': 12, '_': 1, '`': 13, + 'a': 1, 'b': 2, 'c': 1, 'd': 1, + 'e': 1, 'f': 1, 'g': 1, 'h': 1, + 'i': 1, 'j': 1, 'k': 1, 'l': 1, + 'm': 1, 'n': 1, 'o': 1, 'p': 1, + 'q': 1, 'r': 3, 's': 1, 't': 1, + 'u': 2, 'v': 1, 'w': 1, 'x': 1, + 'y': 1, 'z': 1, '{': 13, '|': 12, + '}': 13, '~': 13}, + # 1 + {'0': 1, '1': 1, '2': 1, '3': 1, + '4': 1, '5': 1, '6': 1, '7': 1, + '8': 1, '9': 1, 'A': 1, 'B': 1, + 'C': 1, 'D': 1, 'E': 1, 'F': 1, + 'G': 1, 'H': 1, 'I': 1, 'J': 1, + 'K': 1, 'L': 1, 'M': 1, 'N': 1, + 'O': 1, 'P': 1, 'Q': 1, 'R': 1, + 'S': 1, 'T': 1, 'U': 1, 'V': 1, + 'W': 1, 'X': 1, 'Y': 1, 'Z': 1, + '_': 1, 'a': 1, 'b': 1, 'c': 1, + 'd': 1, 'e': 1, 'f': 1, 'g': 1, + 'h': 1, 'i': 1, 'j': 1, 'k': 1, + 'l': 1, 'm': 1, 'n': 1, 'o': 1, + 'p': 1, 'q': 1, 'r': 1, 's': 1, + 't': 1, 'u': 1, 'v': 1, 'w': 1, + 'x': 1, 'y': 1, 'z': 1}, + # 2 + {'"': 16, "'": 15, '0': 1, '1': 1, + '2': 1, '3': 1, '4': 1, '5': 1, + '6': 1, '7': 1, '8': 1, '9': 1, + 'A': 1, 'B': 1, 'C': 1, 'D': 1, + 'E': 1, 'F': 1, 'G': 1, 'H': 1, + 'I': 1, 'J': 1, 'K': 1, 'L': 1, + 'M': 1, 'N': 1, 'O': 1, 'P': 1, + 'Q': 1, 'R': 3, 'S': 1, 'T': 1, + 'U': 1, 'V': 1, 'W': 1, 'X': 1, + 'Y': 1, 'Z': 1, '_': 1, 'a': 1, + 'b': 1, 'c': 1, 'd': 1, 'e': 1, + 'f': 1, 'g': 1, 'h': 1, 'i': 1, + 'j': 1, 'k': 1, 'l': 1, 'm': 1, + 'n': 1, 'o': 1, 'p': 1, 'q': 1, + 'r': 3, 's': 1, 't': 1, 'u': 1, + 'v': 1, 'w': 1, 'x': 1, 'y': 1, + 'z': 1}, + # 3 + {'"': 16, "'": 15, '0': 1, '1': 1, + '2': 1, '3': 1, '4': 1, '5': 1, + '6': 1, '7': 1, '8': 1, '9': 1, + 'A': 1, 'B': 1, 'C': 1, 'D': 1, + 'E': 1, 'F': 1, 'G': 1, 'H': 1, + 'I': 1, 'J': 1, 'K': 1, 'L': 1, + 'M': 1, 'N': 1, 'O': 1, 'P': 1, + 'Q': 1, 'R': 1, 'S': 1, 'T': 1, + 'U': 1, 'V': 1, 'W': 1, 'X': 1, + 'Y': 1, 'Z': 1, '_': 1, 'a': 1, + 'b': 1, 'c': 1, 'd': 1, 'e': 1, + 'f': 1, 'g': 1, 'h': 1, 'i': 1, + 'j': 1, 'k': 1, 'l': 1, 'm': 1, + 'n': 1, 'o': 1, 'p': 1, 'q': 1, + 'r': 1, 's': 1, 't': 1, 'u': 1, + 'v': 1, 'w': 1, 'x': 1, 'y': 1, + 'z': 1}, + # 4 + {'.': 24, '0': 21, '1': 21, '2': 21, + '3': 21, '4': 21, '5': 21, '6': 21, + '7': 21, '8': 23, '9': 23, 'B': 22, + 'E': 25, 'J': 13, 'L': 13, 'O': 20, + 'X': 19, 'b': 22, 'e': 25, 'j': 13, + 'l': 13, 'o': 20, 'x': 19}, + # 5 + {'.': 24, '0': 5, '1': 5, '2': 5, + '3': 5, '4': 5, '5': 5, '6': 5, + '7': 5, '8': 5, '9': 5, 'E': 25, + 'J': 13, 'L': 13, 'e': 25, 'j': 13, + 'l': 13}, + # 6 + {'0': 26, '1': 26, '2': 26, '3': 26, + '4': 26, '5': 26, '6': 26, '7': 26, + '8': 26, '9': 26}, + # 7 + {'*': 12, '=': 13}, + # 8 + {'=': 13, '>': 12}, + # 9 + {'<': 12, '=': 13, '>': 13}, + # 10 + {'=': 13}, + # 11 + {'/': 12, '=': 13}, + # 12 + {'=': 13}, + # 13 + {}, + # 14 + {'\n': 13}, + # 15 + {automata.DEFAULT: 30, '\n': 27, + '\r': 27, "'": 28, '\\': 29}, + # 16 + {automata.DEFAULT: 33, '\n': 27, + '\r': 27, '"': 31, '\\': 32}, + # 17 + {'\n': 13, '\r': 14}, + # 18 + {automata.DEFAULT: 18, '\n': 27, '\r': 27}, + # 19 + {'0': 34, '1': 34, '2': 34, '3': 34, + '4': 34, '5': 34, '6': 34, '7': 34, + '8': 34, '9': 34, 'A': 34, 'B': 34, + 'C': 34, 'D': 34, 'E': 34, 'F': 34, + 'a': 34, 'b': 34, 'c': 34, 'd': 34, + 'e': 34, 'f': 34}, + # 20 + {'0': 35, '1': 35, '2': 35, '3': 35, + '4': 35, '5': 35, '6': 35, '7': 35}, + # 21 + {'.': 24, '0': 21, '1': 21, '2': 21, + '3': 21, '4': 21, '5': 21, '6': 21, + '7': 21, '8': 23, '9': 23, 'E': 25, + 'J': 13, 'L': 13, 'e': 25, 'j': 13, + 'l': 13}, + # 22 + {'0': 36, '1': 36}, + # 23 + {'.': 24, '0': 23, '1': 23, '2': 23, + '3': 23, '4': 23, '5': 23, '6': 23, + '7': 23, '8': 23, '9': 23, 'E': 25, + 'J': 13, 'e': 25, 'j': 13}, + # 24 + {'0': 24, '1': 24, '2': 24, '3': 24, + '4': 24, '5': 24, '6': 24, '7': 24, + '8': 24, '9': 24, 'E': 37, 'J': 13, + 'e': 37, 'j': 13}, + # 25 + {'+': 38, '-': 38, '0': 39, '1': 39, + '2': 39, '3': 39, '4': 39, '5': 39, + '6': 39, '7': 39, '8': 39, '9': 39}, + # 26 + {'0': 26, '1': 26, '2': 26, '3': 26, + '4': 26, '5': 26, '6': 26, '7': 26, + '8': 26, '9': 26, 'E': 37, 'J': 13, + 'e': 37, 'j': 13}, + # 27 + {}, + # 28 + {"'": 13}, + # 29 + {automata.DEFAULT: 40, '\n': 13, '\r': 14}, + # 30 + {automata.DEFAULT: 30, '\n': 27, + '\r': 27, "'": 13, '\\': 29}, + # 31 + {'"': 13}, + # 32 + {automata.DEFAULT: 41, '\n': 13, '\r': 14}, + # 33 + {automata.DEFAULT: 33, '\n': 27, + '\r': 27, '"': 13, '\\': 32}, + # 34 + {'0': 34, '1': 34, '2': 34, '3': 34, + '4': 34, '5': 34, '6': 34, '7': 34, + '8': 34, '9': 34, 'A': 34, 'B': 34, + 'C': 34, 'D': 34, 'E': 34, 'F': 34, + 'L': 13, 'a': 34, 'b': 34, 'c': 34, + 'd': 34, 'e': 34, 'f': 34, 'l': 13}, + # 35 + {'0': 35, '1': 35, '2': 35, '3': 35, + '4': 35, '5': 35, '6': 35, '7': 35, + 'L': 13, 'l': 13}, + # 36 + {'0': 36, '1': 36, 'L': 13, 'l': 13}, + # 37 + {'+': 42, '-': 42, '0': 43, '1': 43, + '2': 43, '3': 43, '4': 43, '5': 43, + '6': 43, '7': 43, '8': 43, '9': 43}, + # 38 + {'0': 39, '1': 39, '2': 39, '3': 39, + '4': 39, '5': 39, '6': 39, '7': 39, + '8': 39, '9': 39}, + # 39 + {'0': 39, '1': 39, '2': 39, '3': 39, + '4': 39, '5': 39, '6': 39, '7': 39, + '8': 39, '9': 39, 'J': 13, 'j': 13}, + # 40 + {automata.DEFAULT: 40, '\n': 27, + '\r': 27, "'": 13, '\\': 29}, + # 41 + {automata.DEFAULT: 41, '\n': 27, + '\r': 27, '"': 13, '\\': 32}, + # 42 + {'0': 43, '1': 43, '2': 43, '3': 43, + '4': 43, '5': 43, '6': 43, '7': 43, + '8': 43, '9': 43}, + # 43 + {'0': 43, '1': 43, '2': 43, '3': 43, + '4': 43, '5': 43, '6': 43, '7': 43, + '8': 43, '9': 43, 'J': 13, 'j': 13}, + ] +pseudoDFA = automata.DFA(states, accepts) + +accepts = [False, False, False, False, False, True] +states = [ + # 0 + {automata.DEFAULT: 0, '"': 1, '\\': 2}, + # 1 + {automata.DEFAULT: 4, '"': 3, '\\': 2}, + # 2 + {automata.DEFAULT: 4}, + # 3 + {automata.DEFAULT: 4, '"': 5, '\\': 2}, + # 4 + {automata.DEFAULT: 4, '"': 1, '\\': 2}, + # 5 + {automata.DEFAULT: 4, '"': 5, '\\': 2}, + ] +double3DFA = automata.NonGreedyDFA(states, accepts) + +accepts = [False, False, False, False, False, True] +states = [ + # 0 + {automata.DEFAULT: 0, "'": 1, '\\': 2}, + # 1 + {automata.DEFAULT: 4, "'": 3, '\\': 2}, + # 2 + {automata.DEFAULT: 4}, + # 3 + {automata.DEFAULT: 4, "'": 5, '\\': 2}, + # 4 + {automata.DEFAULT: 4, "'": 1, '\\': 2}, + # 5 + {automata.DEFAULT: 4, "'": 5, '\\': 2}, + ] +single3DFA = automata.NonGreedyDFA(states, accepts) + +accepts = [False, True, False, False] +states = [ + # 0 + {automata.DEFAULT: 0, "'": 1, '\\': 2}, + # 1 + {}, + # 2 + {automata.DEFAULT: 3}, + # 3 + {automata.DEFAULT: 3, "'": 1, '\\': 2}, + ] +singleDFA = automata.DFA(states, accepts) + +accepts = [False, True, False, False] +states = [ + # 0 + {automata.DEFAULT: 0, '"': 1, '\\': 2}, + # 1 + {}, + # 2 + {automata.DEFAULT: 3}, + # 3 + {automata.DEFAULT: 3, '"': 1, '\\': 2}, + ] +doubleDFA = automata.DFA(states, accepts) + diff --git a/pypy/interpreter/pyparser/genpytokenize.py b/pypy/interpreter/pyparser/gendfa.py rename from pypy/interpreter/pyparser/genpytokenize.py rename to pypy/interpreter/pyparser/gendfa.py --- a/pypy/interpreter/pyparser/genpytokenize.py +++ b/pypy/interpreter/pyparser/gendfa.py @@ -322,6 +322,12 @@ return ''.join(lines) def main (): + print "# THIS FILE IS AUTOMATICALLY GENERATED BY gendfa.py" + print "# DO NOT EDIT" + print "# TO REGENERATE THE FILE, RUN:" + print "# python gendfa.py > dfa_generated.py" + print + print "from pypy.interpreter.pyparser import automata" pseudoDFA, states_pseudoDFA = makePyPseudoDFA() print output("pseudoDFA", "DFA", pseudoDFA, states_pseudoDFA) endDFAMap = makePyEndDFAMap() diff --git a/pypy/interpreter/pyparser/pytokenize.py b/pypy/interpreter/pyparser/pytokenize.py --- a/pypy/interpreter/pyparser/pytokenize.py +++ b/pypy/interpreter/pyparser/pytokenize.py @@ -1,9 +1,6 @@ # ______________________________________________________________________ """Module pytokenize -THIS FILE WAS COPIED FROM pypy/module/parser/pytokenize.py AND ADAPTED -TO BE ANNOTABLE (Mainly made lists homogeneous) - This is a modified version of Ka-Ping Yee's tokenize module found in the Python standard library. @@ -12,303 +9,14 @@ expressions have been replaced with hand built DFA's using the basil.util.automata module. -$Id: pytokenize.py,v 1.3 2003/10/03 16:31:53 jriehl Exp $ """ # ______________________________________________________________________ from pypy.interpreter.pyparser import automata +from pypy.interpreter.pyparser.dfa_generated import * __all__ = [ "tokenize" ] -# ______________________________________________________________________ -# Automatically generated DFA's - -accepts = [True, True, True, True, True, True, True, True, - True, True, False, True, True, True, True, False, - False, False, True, False, False, True, False, - False, True, False, True, False, True, False, - False, True, False, False, True, True, True, - False, False, True, False, False, False, True] -states = [ - # 0 - {'\t': 0, '\n': 13, '\x0c': 0, - '\r': 14, ' ': 0, '!': 10, '"': 16, - '#': 18, '%': 12, '&': 12, "'": 15, - '(': 13, ')': 13, '*': 7, '+': 12, - ',': 13, '-': 12, '.': 6, '/': 11, - '0': 4, '1': 5, '2': 5, '3': 5, - '4': 5, '5': 5, '6': 5, '7': 5, - '8': 5, '9': 5, ':': 13, ';': 13, - '<': 9, '=': 12, '>': 8, '@': 13, - 'A': 1, 'B': 2, 'C': 1, 'D': 1, - 'E': 1, 'F': 1, 'G': 1, 'H': 1, - 'I': 1, 'J': 1, 'K': 1, 'L': 1, - 'M': 1, 'N': 1, 'O': 1, 'P': 1, - 'Q': 1, 'R': 3, 'S': 1, 'T': 1, - 'U': 2, 'V': 1, 'W': 1, 'X': 1, - 'Y': 1, 'Z': 1, '[': 13, '\\': 17, - ']': 13, '^': 12, '_': 1, '`': 13, - 'a': 1, 'b': 2, 'c': 1, 'd': 1, - 'e': 1, 'f': 1, 'g': 1, 'h': 1, - 'i': 1, 'j': 1, 'k': 1, 'l': 1, - 'm': 1, 'n': 1, 'o': 1, 'p': 1, - 'q': 1, 'r': 3, 's': 1, 't': 1, - 'u': 2, 'v': 1, 'w': 1, 'x': 1, - 'y': 1, 'z': 1, '{': 13, '|': 12, - '}': 13, '~': 13}, - # 1 - {'0': 1, '1': 1, '2': 1, '3': 1, - '4': 1, '5': 1, '6': 1, '7': 1, - '8': 1, '9': 1, 'A': 1, 'B': 1, - 'C': 1, 'D': 1, 'E': 1, 'F': 1, - 'G': 1, 'H': 1, 'I': 1, 'J': 1, - 'K': 1, 'L': 1, 'M': 1, 'N': 1, - 'O': 1, 'P': 1, 'Q': 1, 'R': 1, - 'S': 1, 'T': 1, 'U': 1, 'V': 1, - 'W': 1, 'X': 1, 'Y': 1, 'Z': 1, - '_': 1, 'a': 1, 'b': 1, 'c': 1, - 'd': 1, 'e': 1, 'f': 1, 'g': 1, - 'h': 1, 'i': 1, 'j': 1, 'k': 1, - 'l': 1, 'm': 1, 'n': 1, 'o': 1, - 'p': 1, 'q': 1, 'r': 1, 's': 1, - 't': 1, 'u': 1, 'v': 1, 'w': 1, - 'x': 1, 'y': 1, 'z': 1}, - # 2 - {'"': 16, "'": 15, '0': 1, '1': 1, - '2': 1, '3': 1, '4': 1, '5': 1, - '6': 1, '7': 1, '8': 1, '9': 1, - 'A': 1, 'B': 1, 'C': 1, 'D': 1, - 'E': 1, 'F': 1, 'G': 1, 'H': 1, - 'I': 1, 'J': 1, 'K': 1, 'L': 1, - 'M': 1, 'N': 1, 'O': 1, 'P': 1, - 'Q': 1, 'R': 3, 'S': 1, 'T': 1, - 'U': 1, 'V': 1, 'W': 1, 'X': 1, - 'Y': 1, 'Z': 1, '_': 1, 'a': 1, - 'b': 1, 'c': 1, 'd': 1, 'e': 1, - 'f': 1, 'g': 1, 'h': 1, 'i': 1, - 'j': 1, 'k': 1, 'l': 1, 'm': 1, - 'n': 1, 'o': 1, 'p': 1, 'q': 1, - 'r': 3, 's': 1, 't': 1, 'u': 1, - 'v': 1, 'w': 1, 'x': 1, 'y': 1, - 'z': 1}, - # 3 - {'"': 16, "'": 15, '0': 1, '1': 1, - '2': 1, '3': 1, '4': 1, '5': 1, - '6': 1, '7': 1, '8': 1, '9': 1, - 'A': 1, 'B': 1, 'C': 1, 'D': 1, - 'E': 1, 'F': 1, 'G': 1, 'H': 1, - 'I': 1, 'J': 1, 'K': 1, 'L': 1, - 'M': 1, 'N': 1, 'O': 1, 'P': 1, - 'Q': 1, 'R': 1, 'S': 1, 'T': 1, - 'U': 1, 'V': 1, 'W': 1, 'X': 1, - 'Y': 1, 'Z': 1, '_': 1, 'a': 1, - 'b': 1, 'c': 1, 'd': 1, 'e': 1, - 'f': 1, 'g': 1, 'h': 1, 'i': 1, - 'j': 1, 'k': 1, 'l': 1, 'm': 1, - 'n': 1, 'o': 1, 'p': 1, 'q': 1, - 'r': 1, 's': 1, 't': 1, 'u': 1, - 'v': 1, 'w': 1, 'x': 1, 'y': 1, - 'z': 1}, - # 4 - {'.': 24, '0': 21, '1': 21, '2': 21, - '3': 21, '4': 21, '5': 21, '6': 21, - '7': 21, '8': 23, '9': 23, 'B': 22, - 'E': 25, 'J': 13, 'L': 13, 'O': 20, - 'X': 19, 'b': 22, 'e': 25, 'j': 13, - 'l': 13, 'o': 20, 'x': 19}, - # 5 - {'.': 24, '0': 5, '1': 5, '2': 5, - '3': 5, '4': 5, '5': 5, '6': 5, - '7': 5, '8': 5, '9': 5, 'E': 25, - 'J': 13, 'L': 13, 'e': 25, 'j': 13, - 'l': 13}, - # 6 - {'0': 26, '1': 26, '2': 26, '3': 26, - '4': 26, '5': 26, '6': 26, '7': 26, - '8': 26, '9': 26}, - # 7 - {'*': 12, '=': 13}, - # 8 - {'=': 13, '>': 12}, - # 9 - {'<': 12, '=': 13, '>': 13}, - # 10 - {'=': 13}, - # 11 - {'/': 12, '=': 13}, - # 12 - {'=': 13}, - # 13 - {}, - # 14 - {'\n': 13}, - # 15 - {automata.DEFAULT: 30, '\n': 27, - '\r': 27, "'": 28, '\\': 29}, - # 16 - {automata.DEFAULT: 33, '\n': 27, - '\r': 27, '"': 31, '\\': 32}, - # 17 - {'\n': 13, '\r': 14}, - # 18 - {automata.DEFAULT: 18, '\n': 27, '\r': 27}, - # 19 - {'0': 34, '1': 34, '2': 34, '3': 34, - '4': 34, '5': 34, '6': 34, '7': 34, - '8': 34, '9': 34, 'A': 34, 'B': 34, - 'C': 34, 'D': 34, 'E': 34, 'F': 34, - 'a': 34, 'b': 34, 'c': 34, 'd': 34, - 'e': 34, 'f': 34}, - # 20 - {'0': 35, '1': 35, '2': 35, '3': 35, - '4': 35, '5': 35, '6': 35, '7': 35}, - # 21 - {'.': 24, '0': 21, '1': 21, '2': 21, - '3': 21, '4': 21, '5': 21, '6': 21, - '7': 21, '8': 23, '9': 23, 'E': 25, - 'J': 13, 'L': 13, 'e': 25, 'j': 13, - 'l': 13}, - # 22 - {'0': 36, '1': 36}, - # 23 - {'.': 24, '0': 23, '1': 23, '2': 23, - '3': 23, '4': 23, '5': 23, '6': 23, - '7': 23, '8': 23, '9': 23, 'E': 25, - 'J': 13, 'e': 25, 'j': 13}, - # 24 - {'0': 24, '1': 24, '2': 24, '3': 24, - '4': 24, '5': 24, '6': 24, '7': 24, - '8': 24, '9': 24, 'E': 37, 'J': 13, - 'e': 37, 'j': 13}, - # 25 - {'+': 38, '-': 38, '0': 39, '1': 39, - '2': 39, '3': 39, '4': 39, '5': 39, - '6': 39, '7': 39, '8': 39, '9': 39}, - # 26 - {'0': 26, '1': 26, '2': 26, '3': 26, - '4': 26, '5': 26, '6': 26, '7': 26, - '8': 26, '9': 26, 'E': 37, 'J': 13, - 'e': 37, 'j': 13}, - # 27 - {}, - # 28 - {"'": 13}, - # 29 - {automata.DEFAULT: 40, '\n': 13, '\r': 14}, - # 30 - {automata.DEFAULT: 30, '\n': 27, - '\r': 27, "'": 13, '\\': 29}, - # 31 - {'"': 13}, - # 32 - {automata.DEFAULT: 41, '\n': 13, '\r': 14}, - # 33 - {automata.DEFAULT: 33, '\n': 27, - '\r': 27, '"': 13, '\\': 32}, - # 34 - {'0': 34, '1': 34, '2': 34, '3': 34, - '4': 34, '5': 34, '6': 34, '7': 34, - '8': 34, '9': 34, 'A': 34, 'B': 34, - 'C': 34, 'D': 34, 'E': 34, 'F': 34, - 'L': 13, 'a': 34, 'b': 34, 'c': 34, - 'd': 34, 'e': 34, 'f': 34, 'l': 13}, - # 35 - {'0': 35, '1': 35, '2': 35, '3': 35, - '4': 35, '5': 35, '6': 35, '7': 35, - 'L': 13, 'l': 13}, - # 36 - {'0': 36, '1': 36, 'L': 13, 'l': 13}, - # 37 - {'+': 42, '-': 42, '0': 43, '1': 43, - '2': 43, '3': 43, '4': 43, '5': 43, - '6': 43, '7': 43, '8': 43, '9': 43}, - # 38 - {'0': 39, '1': 39, '2': 39, '3': 39, - '4': 39, '5': 39, '6': 39, '7': 39, - '8': 39, '9': 39}, - # 39 - {'0': 39, '1': 39, '2': 39, '3': 39, - '4': 39, '5': 39, '6': 39, '7': 39, - '8': 39, '9': 39, 'J': 13, 'j': 13}, - # 40 - {automata.DEFAULT: 40, '\n': 27, - '\r': 27, "'": 13, '\\': 29}, - # 41 - {automata.DEFAULT: 41, '\n': 27, - '\r': 27, '"': 13, '\\': 32}, - # 42 - {'0': 43, '1': 43, '2': 43, '3': 43, - '4': 43, '5': 43, '6': 43, '7': 43, - '8': 43, '9': 43}, - # 43 - {'0': 43, '1': 43, '2': 43, '3': 43, - '4': 43, '5': 43, '6': 43, '7': 43, - '8': 43, '9': 43, 'J': 13, 'j': 13}, - ] -pseudoDFA = automata.DFA(states, accepts) - -accepts = [False, False, False, False, False, True] -states = [ - # 0 - {automata.DEFAULT: 0, '"': 1, '\\': 2}, - # 1 - {automata.DEFAULT: 4, '"': 3, '\\': 2}, - # 2 - {automata.DEFAULT: 4}, - # 3 - {automata.DEFAULT: 4, '"': 5, '\\': 2}, - # 4 - {automata.DEFAULT: 4, '"': 1, '\\': 2}, - # 5 - {automata.DEFAULT: 4, '"': 5, '\\': 2}, - ] -double3DFA = automata.NonGreedyDFA(states, accepts) - -accepts = [False, False, False, False, False, True] -states = [ - # 0 - {automata.DEFAULT: 0, "'": 1, '\\': 2}, - # 1 - {automata.DEFAULT: 4, "'": 3, '\\': 2}, - # 2 - {automata.DEFAULT: 4}, - # 3 - {automata.DEFAULT: 4, "'": 5, '\\': 2}, - # 4 - {automata.DEFAULT: 4, "'": 1, '\\': 2}, - # 5 - {automata.DEFAULT: 4, "'": 5, '\\': 2}, - ] -single3DFA = automata.NonGreedyDFA(states, accepts) - -accepts = [False, True, False, False] -states = [ - # 0 - {automata.DEFAULT: 0, "'": 1, '\\': 2}, - # 1 - {}, - # 2 - {automata.DEFAULT: 3}, - # 3 - {automata.DEFAULT: 3, "'": 1, '\\': 2}, - ] -singleDFA = automata.DFA(states, accepts) - -accepts = [False, True, False, False] -states = [ - # 0 - {automata.DEFAULT: 0, '"': 1, '\\': 2}, - # 1 - {}, - # 2 - {automata.DEFAULT: 3}, - # 3 - {automata.DEFAULT: 3, '"': 1, '\\': 2}, - ] -doubleDFA = automata.DFA(states, accepts) - -#_______________________________________________________________________ -# End of automatically generated DFA's endDFAs = {"'" : singleDFA, '"' : doubleDFA, @@ -354,22 +62,3 @@ single_quoted[t] = t tabsize = 8 - -# PYPY MODIFICATION: removed TokenError class as it's not needed here - -# PYPY MODIFICATION: removed StopTokenizing class as it's not needed here - -# PYPY MODIFICATION: removed printtoken() as it's not needed here - -# PYPY MODIFICATION: removed tokenize() as it's not needed here - -# PYPY MODIFICATION: removed tokenize_loop() as it's not needed here - -# PYPY MODIFICATION: removed generate_tokens() as it was copied / modified -# in pythonlexer.py - -# PYPY MODIFICATION: removed main() as it's not needed here - -# ______________________________________________________________________ -# End of pytokenize.py - diff --git a/pypy/interpreter/pyparser/test/test_gendfa.py b/pypy/interpreter/pyparser/test/test_gendfa.py --- a/pypy/interpreter/pyparser/test/test_gendfa.py +++ b/pypy/interpreter/pyparser/test/test_gendfa.py @@ -1,5 +1,5 @@ from pypy.interpreter.pyparser.automata import DFA, DEFAULT -from pypy.interpreter.pyparser.genpytokenize import output +from pypy.interpreter.pyparser.gendfa import output def test_states(): states = [{"\x00": 1}, {"\x01": 0}] From pypy.commits at gmail.com Sat Apr 14 13:49:45 2018 From: pypy.commits at gmail.com (mattip) Date: Sat, 14 Apr 2018 10:49:45 -0700 (PDT) Subject: [pypy-commit] pypy default: reset whatsnew-head for release Message-ID: <5ad23f39.8fc9df0a.45012.f657@mx.google.com> Author: Matti Picus Branch: Changeset: r94322:1bdd44279495 Date: 2018-04-14 20:44 +0300 http://bitbucket.org/pypy/pypy/changeset/1bdd44279495/ Log: reset whatsnew-head for release diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -3,16 +3,6 @@ ========================== .. this is a revision shortly after release-pypy-6.0.0 -.. startrev: 2e04adf1b89f +.. startrev: f22145c34985 -.. branch: cpyext-subclass-setattr -Fix for python-level classes that inherit from C-API types, previously the -`w_obj` was not necessarily preserved throughout the lifetime of the `pyobj` -which led to cases where instance attributes were lost. Fixes issue #2793 - - -.. branch: pyparser-improvements-2 - -Improve line offsets that are reported by SyntaxError. Improve error messages -for a few situations, including mismatched parenthesis. diff --git a/pypy/doc/whatsnew-pypy2-6.0.0.rst b/pypy/doc/whatsnew-pypy2-6.0.0.rst --- a/pypy/doc/whatsnew-pypy2-6.0.0.rst +++ b/pypy/doc/whatsnew-pypy2-6.0.0.rst @@ -97,3 +97,15 @@ Work around possible bugs in upstream ioctl users, like CPython allocate at least 1024 bytes for the arg in calls to ``ioctl(fd, request, arg)``. Fixes issue #2776 + +.. branch: cpyext-subclass-setattr + +Fix for python-level classes that inherit from C-API types, previously the +`w_obj` was not necessarily preserved throughout the lifetime of the `pyobj` +which led to cases where instance attributes were lost. Fixes issue #2793 + + +.. branch: pyparser-improvements-2 + +Improve line offsets that are reported by SyntaxError. Improve error messages +for a few situations, including mismatched parenthesis. From pypy.commits at gmail.com Sat Apr 14 13:49:51 2018 From: pypy.commits at gmail.com (mattip) Date: Sat, 14 Apr 2018 10:49:51 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: merge default into py3.5 Message-ID: <5ad23f3f.44aadf0a.1e7b0.6148@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r94324:1be242f4be56 Date: 2018-04-14 20:47 +0300 http://bitbucket.org/pypy/pypy/changeset/1be242f4be56/ Log: merge default into py3.5 diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst --- a/pypy/doc/how-to-release.rst +++ b/pypy/doc/how-to-release.rst @@ -40,6 +40,8 @@ sure things are ported back to the trunk and to the branch as necessary. +* Make sure the RPython builds on the buildbot pass with no failures + * Maybe bump the SOABI number in module/imp/importing. This has many implications, so make sure the PyPy community agrees to the change. diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -3,16 +3,6 @@ ========================== .. this is a revision shortly after release-pypy-6.0.0 -.. startrev: 2e04adf1b89f +.. startrev: f22145c34985 -.. branch: cpyext-subclass-setattr -Fix for python-level classes that inherit from C-API types, previously the -`w_obj` was not necessarily preserved throughout the lifetime of the `pyobj` -which led to cases where instance attributes were lost. Fixes issue #2793 - - -.. branch: pyparser-improvements-2 - -Improve line offsets that are reported by SyntaxError. Improve error messages -for a few situations, including mismatched parenthesis. diff --git a/pypy/doc/whatsnew-pypy2-6.0.0.rst b/pypy/doc/whatsnew-pypy2-6.0.0.rst --- a/pypy/doc/whatsnew-pypy2-6.0.0.rst +++ b/pypy/doc/whatsnew-pypy2-6.0.0.rst @@ -101,3 +101,15 @@ Work around possible bugs in upstream ioctl users, like CPython allocate at least 1024 bytes for the arg in calls to ``ioctl(fd, request, arg)``. Fixes issue #2776 + +.. branch: cpyext-subclass-setattr + +Fix for python-level classes that inherit from C-API types, previously the +`w_obj` was not necessarily preserved throughout the lifetime of the `pyobj` +which led to cases where instance attributes were lost. Fixes issue #2793 + + +.. branch: pyparser-improvements-2 + +Improve line offsets that are reported by SyntaxError. Improve error messages +for a few situations, including mismatched parenthesis. diff --git a/pypy/module/cpyext/setobject.py b/pypy/module/cpyext/setobject.py --- a/pypy/module/cpyext/setobject.py +++ b/pypy/module/cpyext/setobject.py @@ -1,4 +1,4 @@ -from pypy.interpreter.error import oefmt +from pypy.interpreter.error import OperationError, oefmt from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.cpyext.api import (cpython_api, Py_ssize_t, CANNOT_FAIL, build_type_checkers) @@ -64,8 +64,13 @@ instance of set or its subtype.""" if not PySet_Check(space, w_s): PyErr_BadInternalCall(space) - space.call_method(space.w_set, 'discard', w_s, w_obj) - return 0 + try: + space.call_method(space.w_set, 'remove', w_s, w_obj) + except OperationError as e: + if e.match(space, space.w_KeyError): + return 0 + raise + return 1 @cpython_api([PyObject], PyObject) diff --git a/pypy/module/cpyext/test/test_setobject.py b/pypy/module/cpyext/test/test_setobject.py --- a/pypy/module/cpyext/test/test_setobject.py +++ b/pypy/module/cpyext/test/test_setobject.py @@ -28,7 +28,11 @@ assert api.PySet_Size(w_set) == 4 api.PySet_Add(w_set, space.wrap(6)) assert api.PySet_Size(w_set) == 5 - api.PySet_Discard(w_set, space.wrap(6)) + res = api.PySet_Discard(w_set, space.wrap(6)) + assert res == 1 + assert api.PySet_Size(w_set) == 4 + res = api.PySet_Discard(w_set, space.wrap(6)) + assert res == 0 assert api.PySet_Size(w_set) == 4 def test_set_contains(self, space, api): diff --git a/pypy/tool/release/force-builds.py b/pypy/tool/release/force-builds.py --- a/pypy/tool/release/force-builds.py +++ b/pypy/tool/release/force-builds.py @@ -31,6 +31,9 @@ 'pypy-c-jit-linux-s390x', 'build-pypy-c-jit-linux-armhf-raspbian', 'build-pypy-c-jit-linux-armel', + 'rpython-linux-x86-32', + 'rpython-linux-x86-64' + 'rpython-win-x86-32' ] def get_user(): diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -644,7 +644,10 @@ def get_rpy_type_index(gcref): from rpython.rlib.rarithmetic import intmask Class = gcref._x.__class__ - return intmask(id(Class)) + i = intmask(id(Class)) + if i < 0: + i = ~i # always return a positive number, at least + return i def cast_gcref_to_int(gcref): # This is meant to be used on cast_instance_to_gcref results. diff --git a/rpython/rlib/rvmprof/src/shared/_vmprof.c b/rpython/rlib/rvmprof/src/shared/_vmprof.c --- a/rpython/rlib/rvmprof/src/shared/_vmprof.c +++ b/rpython/rlib/rvmprof/src/shared/_vmprof.c @@ -36,6 +36,8 @@ register PY_STACK_FRAME_T * callee_saved asm("rbx"); #elif defined(X86_32) register PY_STACK_FRAME_T * callee_saved asm("edi"); +#elif defined(__arm__) + register PY_STACK_FRAME_T * callee_saved asm("r4"); #else # error "platform not supported" #endif @@ -45,6 +47,8 @@ "movq %1, %0\t\n" #elif defined(X86_32) "mov %1, %0\t\n" +#elif defined(__arm__) + "mov %1, %0\t\n" #else # error "platform not supported" #endif diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.c b/rpython/rlib/rvmprof/src/shared/vmp_stack.c --- a/rpython/rlib/rvmprof/src/shared/vmp_stack.c +++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.c @@ -16,7 +16,7 @@ #ifdef VMP_SUPPORTS_NATIVE_PROFILING -#ifdef VMPROF_LINUX +#if defined(VMPROF_LINUX) || defined(VMPROF_BSD) #include "unwind/vmprof_unwind.h" typedef mcontext_t unw_context_t; @@ -510,13 +510,15 @@ static const char * vmprof_error = NULL; static void * libhandle = NULL; - #ifdef VMPROF_LINUX +#include #define LIBUNWIND "libunwind.so" #ifdef __i386__ #define PREFIX "x86" +#define LIBUNWIND_SUFFIX "" #elif __x86_64__ #define PREFIX "x86_64" +#define LIBUNWIND_SUFFIX "-x86_64" #endif #define U_PREFIX "_U" #define UL_PREFIX "_UL" @@ -524,10 +526,41 @@ int vmp_native_enable(void) { #ifdef VMPROF_LINUX + void * oldhandle = NULL; + struct link_map * map = NULL; if (libhandle == NULL) { + // on linux, the wheel includes the libunwind shared object. + libhandle = dlopen(NULL, RTLD_NOW); + if (libhandle != NULL) { + // load the link map, it will contain an entry to + // .libs_vmprof/libunwind-...so, this is the file that is + // distributed with the wheel. + if (dlinfo(libhandle, RTLD_DI_LINKMAP, &map) != 0) { + (void)dlclose(libhandle); + libhandle = NULL; + goto bail_out; + } + // grab the new handle + do { + if (strstr(map->l_name, ".libs_vmprof/libunwind" LIBUNWIND_SUFFIX) != NULL) { + oldhandle = libhandle; + libhandle = dlopen(map->l_name, RTLD_LAZY|RTLD_LOCAL); + (void)dlclose(oldhandle); + oldhandle = NULL; + goto loaded_libunwind; + } + map = map->l_next; + } while (map != NULL); + // did not find .libs_vmprof/libunwind... + (void)dlclose(libhandle); + libhandle = NULL; + } + + // fallback! try to load the system's libunwind.so if ((libhandle = dlopen(LIBUNWIND, RTLD_LAZY | RTLD_LOCAL)) == NULL) { goto bail_out; } +loaded_libunwind: if ((unw_get_reg = dlsym(libhandle, UL_PREFIX PREFIX "_get_reg")) == NULL) { goto bail_out; } diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_common.h b/rpython/rlib/rvmprof/src/shared/vmprof_common.h --- a/rpython/rlib/rvmprof/src/shared/vmprof_common.h +++ b/rpython/rlib/rvmprof/src/shared/vmprof_common.h @@ -23,6 +23,10 @@ #include #endif +#ifdef VMPROF_BSD +#include +#endif + #define MAX_FUNC_NAME 1024 #ifdef VMPROF_UNIX From pypy.commits at gmail.com Sat Apr 14 13:49:48 2018 From: pypy.commits at gmail.com (mattip) Date: Sat, 14 Apr 2018 10:49:48 -0700 (PDT) Subject: [pypy-commit] pypy release-pypy2.7-6.x: merge default into release Message-ID: <5ad23f3c.d24a1c0a.902c0.45dd@mx.google.com> Author: Matti Picus Branch: release-pypy2.7-6.x Changeset: r94323:46a3d2b1b0ee Date: 2018-04-14 20:45 +0300 http://bitbucket.org/pypy/pypy/changeset/46a3d2b1b0ee/ Log: merge default into release diff --git a/lib-python/2.7/test/test_eof.py b/lib-python/2.7/test/test_eof.py --- a/lib-python/2.7/test/test_eof.py +++ b/lib-python/2.7/test/test_eof.py @@ -5,7 +5,7 @@ class EOFTestCase(unittest.TestCase): def test_EOFC(self): - expect = "EOL while scanning string literal (, line 1)" + expect = "end of line (EOL) while scanning string literal (, line 1)" try: eval("""'this is a test\ """) @@ -15,7 +15,7 @@ raise test_support.TestFailed def test_EOFS(self): - expect = ("EOF while scanning triple-quoted string literal " + expect = ("end of file (EOF) while scanning triple-quoted string literal " "(, line 1)") try: eval("""'''this is a test""") diff --git a/lib-python/2.7/test/test_generators.py b/lib-python/2.7/test/test_generators.py --- a/lib-python/2.7/test/test_generators.py +++ b/lib-python/2.7/test/test_generators.py @@ -398,7 +398,10 @@ 0 >>> type(i.gi_frame) ->>> i.gi_running = 42 + +PyPy prints "readonly attribute 'gi_running'" so ignore the exception detail + +>>> i.gi_running = 42 # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ... TypeError: readonly attribute diff --git a/lib-python/2.7/test/test_traceback.py b/lib-python/2.7/test/test_traceback.py --- a/lib-python/2.7/test/test_traceback.py +++ b/lib-python/2.7/test/test_traceback.py @@ -123,10 +123,7 @@ self.assertEqual(len(err), 4) self.assertEqual(err[1].strip(), "print(2)") self.assertIn("^", err[2]) - if check_impl_detail(): - self.assertEqual(err[1].find("p"), err[2].find("^")) - if check_impl_detail(pypy=True): - self.assertEqual(err[1].find("2)") + 1, err[2].find("^")) + self.assertEqual(err[1].find("p"), err[2].find("^")) def test_base_exception(self): # Test that exceptions derived from BaseException are formatted right diff --git a/lib-python/2.7/threading.py b/lib-python/2.7/threading.py --- a/lib-python/2.7/threading.py +++ b/lib-python/2.7/threading.py @@ -351,6 +351,21 @@ # forward-compatibility reasons we do the same. waiter.acquire() gotit = True + except AttributeError: + # someone patched the 'waiter' class, probably. + # Fall back to the standard CPython logic. + # See the CPython lib for the comments about it... + endtime = _time() + timeout + delay = 0.0005 # 500 us -> initial delay of 1 ms + while True: + gotit = waiter.acquire(0) + if gotit: + break + remaining = endtime - _time() + if remaining <= 0: + break + delay = min(delay * 2, remaining, .05) + _sleep(delay) else: gotit = waiter.acquire(False) if not gotit: diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -10,89 +10,6 @@ PyPy. -.. _extension-modules: - -Extension modules ------------------ - -List of extension modules that we support: - -* Supported as built-in modules (in :source:`pypy/module/`): - - __builtin__ - :doc:`__pypy__ <__pypy__-module>` - _ast - _codecs - _collections - :doc:`_continuation ` - :doc:`_ffi ` - _hashlib - _io - _locale - _lsprof - _md5 - :doc:`_minimal_curses ` - _multiprocessing - _random - :doc:`_rawffi ` - _sha - _socket - _sre - _ssl - _warnings - _weakref - _winreg - array - binascii - bz2 - cStringIO - cmath - `cpyext`_ - crypt - errno - exceptions - fcntl - gc - imp - itertools - marshal - math - mmap - operator - parser - posix - pyexpat - select - signal - struct - symbol - sys - termios - thread - time - token - unicodedata - zipimport - zlib - - When translated on Windows, a few Unix-only modules are skipped, - and the following module is built instead: - - _winreg - -* Supported by being rewritten in pure Python (possibly using ``cffi``): - see the :source:`lib_pypy/` directory. Examples of modules that we - support this way: ``ctypes``, ``cPickle``, ``cmath``, ``dbm``, ``datetime``... - Note that some modules are both in there and in the list above; - by default, the built-in module is used (but can be disabled - at translation time). - -The extension modules (i.e. modules written in C, in the standard CPython) -that are neither mentioned above nor in :source:`lib_pypy/` are not available in PyPy. -(You may have a chance to use them anyway with `cpyext`_.) - -.. _cpyext: http://morepypy.blogspot.com/2010/04/using-cpython-extension-modules-with.html - Differences related to garbage collection strategies ---------------------------------------------------- @@ -559,7 +476,96 @@ environment variable. CPython searches for ``vcvarsall.bat`` somewhere **above** that value. +* SyntaxError_ s try harder to give details about the cause of the failure, so + the error messages are not the same as in CPython + + +.. _extension-modules: + +Extension modules +----------------- + +List of extension modules that we support: + +* Supported as built-in modules (in :source:`pypy/module/`): + + __builtin__ + :doc:`__pypy__ <__pypy__-module>` + _ast + _codecs + _collections + :doc:`_continuation ` + :doc:`_ffi ` + _hashlib + _io + _locale + _lsprof + _md5 + :doc:`_minimal_curses ` + _multiprocessing + _random + :doc:`_rawffi ` + _sha + _socket + _sre + _ssl + _warnings + _weakref + _winreg + array + binascii + bz2 + cStringIO + cmath + `cpyext`_ + crypt + errno + exceptions + fcntl + gc + imp + itertools + marshal + math + mmap + operator + parser + posix + pyexpat + select + signal + struct + symbol + sys + termios + thread + time + token + unicodedata + zipimport + zlib + + When translated on Windows, a few Unix-only modules are skipped, + and the following module is built instead: + + _winreg + +* Supported by being rewritten in pure Python (possibly using ``cffi``): + see the :source:`lib_pypy/` directory. Examples of modules that we + support this way: ``ctypes``, ``cPickle``, ``cmath``, ``dbm``, ``datetime``... + Note that some modules are both in there and in the list above; + by default, the built-in module is used (but can be disabled + at translation time). + +The extension modules (i.e. modules written in C, in the standard CPython) +that are neither mentioned above nor in :source:`lib_pypy/` are not available in PyPy. +(You may have a chance to use them anyway with `cpyext`_.) + +.. _cpyext: http://morepypy.blogspot.com/2010/04/using-cpython-extension-modules-with.html + + .. _`is ignored in PyPy`: http://bugs.python.org/issue14621 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/ .. _`issue #2653`: https://bitbucket.org/pypy/pypy/issues/2653/ +.. _SyntaxError: https://morepypy.blogspot.co.il/2018/04/improving-syntaxerror-in-pypy.html diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst --- a/pypy/doc/how-to-release.rst +++ b/pypy/doc/how-to-release.rst @@ -40,6 +40,8 @@ sure things are ported back to the trunk and to the branch as necessary. +* Make sure the RPython builds on the buildbot pass with no failures + * Maybe bump the SOABI number in module/imp/importing. This has many implications, so make sure the PyPy community agrees to the change. diff --git a/pypy/doc/release-v6.0.0.rst b/pypy/doc/release-v6.0.0.rst --- a/pypy/doc/release-v6.0.0.rst +++ b/pypy/doc/release-v6.0.0.rst @@ -8,26 +8,30 @@ the dual release. This release is a feature release following our previous 5.10 incremental -release in late December 2017, with many improvements in the C-API -compatability layer and other improvements in speed and CPython compatibility. -Since the changes affect the included python development header files, all -c-extension modules must be recompiled for this version. +release in late December 2017. Our C-API compatability layer ``cpyext`` is +now much faster (see the `blog post`_) as well as more complete. We have made +many other improvements in speed and CPython compatibility. Since the changes +affect the included python development header files, all c-extension modules must +be recompiled for this version. -The Windows PyPy3.5 release is still considered beta-quality. There are issues -with unicode handling especially around system calls and c-extensions. +First-time python users are often stumped by silly typos and emissions when +getting started writing code. We have improved our parser to emit more friendly +`syntax errors`_, making PyPy not only faster but more friendly. -The Matplotlib TkAgg backend now works with PyPy. PyGame and pygtk also now can -work with PyPy. +The Windows PyPy3.5 release is still considered beta-quality. There are open +issues with unicode handling especially around system calls and c-extensions. + +The Matplotlib TkAgg backend now works with PyPy, as do pygame and pygobject_. As always, this release is 100% compatible with the previous one and fixed several issues and bugs raised by the growing community of PyPy users. We strongly recommend updating. +We updated the cffi module included in PyPy to version 1.11.5 + The utf8 branch that changes internal representation of unicode to utf8 did not -make it into the release. We also began working on a Python3.6 implementation, -help is welcome. - -We updated the cffi module included in PyPy to version 1.11.5 +make it into the release, so there is still more goodness coming. We also +began working on a Python3.6 implementation, help is welcome. You can download the v6.0 releases here: @@ -46,6 +50,9 @@ .. _`RPython`: https://rpython.readthedocs.org .. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly .. _`help`: project-ideas.html +.. _`blog post`: https://morepypy.blogspot.it/2017/10/cape-of-good-hope-for-pypy-hello-from.html +.. _pygobject: https://lazka.github.io/posts/2018-04_pypy-pygobject/index.html +.. _`syntax errors`: https://morepypy.blogspot.com/2018/04/improving-syntaxerror-in-pypy.html What is PyPy? ============= diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -3,6 +3,6 @@ ========================== .. this is a revision shortly after release-pypy-6.0.0 -.. startrev: 2e04adf1b89f +.. startrev: f22145c34985 diff --git a/pypy/doc/whatsnew-pypy2-6.0.0.rst b/pypy/doc/whatsnew-pypy2-6.0.0.rst --- a/pypy/doc/whatsnew-pypy2-6.0.0.rst +++ b/pypy/doc/whatsnew-pypy2-6.0.0.rst @@ -97,3 +97,15 @@ Work around possible bugs in upstream ioctl users, like CPython allocate at least 1024 bytes for the arg in calls to ``ioctl(fd, request, arg)``. Fixes issue #2776 + +.. branch: cpyext-subclass-setattr + +Fix for python-level classes that inherit from C-API types, previously the +`w_obj` was not necessarily preserved throughout the lifetime of the `pyobj` +which led to cases where instance attributes were lost. Fixes issue #2793 + + +.. branch: pyparser-improvements-2 + +Improve line offsets that are reported by SyntaxError. Improve error messages +for a few situations, including mismatched parenthesis. diff --git a/pypy/interpreter/pyparser/error.py b/pypy/interpreter/pyparser/error.py --- a/pypy/interpreter/pyparser/error.py +++ b/pypy/interpreter/pyparser/error.py @@ -6,6 +6,7 @@ lastlineno=0): self.msg = msg self.lineno = lineno + # NB: offset is a 1-based index! self.offset = offset self.text = text self.filename = filename diff --git a/pypy/interpreter/pyparser/parser.py b/pypy/interpreter/pyparser/parser.py --- a/pypy/interpreter/pyparser/parser.py +++ b/pypy/interpreter/pyparser/parser.py @@ -199,6 +199,7 @@ self.token_type = token_type self.value = value self.lineno = lineno + # this is a 0-based index self.column = column self.line = line self.expected = expected diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py --- a/pypy/interpreter/pyparser/pyparse.py +++ b/pypy/interpreter/pyparser/pyparse.py @@ -188,7 +188,9 @@ if e.expected_str is not None: msg += " (expected '%s')" % e.expected_str - raise new_err(msg, e.lineno, e.column, e.line, + # parser.ParseError(...).column is 0-based, but the offsets in the + # exceptions in the error module are 1-based, hence the '+ 1' + raise new_err(msg, e.lineno, e.column + 1, e.line, compile_info.filename) else: tree = self.root diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py --- a/pypy/interpreter/pyparser/pytokenizer.py +++ b/pypy/interpreter/pyparser/pytokenizer.py @@ -73,14 +73,14 @@ logical line; continuation lines are included. """ token_list = [] - lnum = parenlev = continued = 0 + lnum = continued = 0 namechars = NAMECHARS numchars = NUMCHARS contstr, needcont = '', 0 contline = None indents = [0] last_comment = '' - parenlevstart = (0, 0, "") + parenstack = [] # make the annotator happy endDFA = DUMMY_DFA @@ -97,7 +97,7 @@ if contstr: if not line: raise TokenError( - "EOF while scanning triple-quoted string literal", + "end of file (EOF) while scanning triple-quoted string literal", strstart[2], strstart[0], strstart[1]+1, token_list, lnum-1) endmatch = endDFA.recognize(line) @@ -123,7 +123,7 @@ contline = contline + line continue - elif parenlev == 0 and not continued: # new statement + elif not parenstack and not continued: # new statement if not line: break column = 0 while pos < max: # measure leading whitespace @@ -143,21 +143,21 @@ token_list.append((tokens.INDENT, line[:pos], lnum, 0, line)) last_comment = '' while column < indents[-1]: - indents = indents[:-1] + indents.pop() token_list.append((tokens.DEDENT, '', lnum, pos, line)) last_comment = '' if column != indents[-1]: err = "unindent does not match any outer indentation level" - raise TokenIndentationError(err, line, lnum, 0, token_list) + raise TokenIndentationError(err, line, lnum, column+1, token_list) else: # continued statement if not line: - if parenlev > 0: - lnum1, start1, line1 = parenlevstart + if parenstack: + _, lnum1, start1, line1 = parenstack[0] raise TokenError("parenthesis is never closed", line1, lnum1, start1 + 1, token_list, lnum) - raise TokenError("EOF in multi-line statement", line, - lnum, 0, token_list) + raise TokenError("end of file (EOF) in multi-line statement", line, + lnum, 0, token_list) # XXX why is the offset 0 here? continued = 0 while pos < max: @@ -180,7 +180,7 @@ token_list.append((tokens.NUMBER, token, lnum, start, line)) last_comment = '' elif initial in '\r\n': - if parenlev <= 0: + if not parenstack: tok = (tokens.NEWLINE, last_comment, lnum, start, line) token_list.append(tok) last_comment = '' @@ -222,14 +222,22 @@ continued = 1 else: if initial in '([{': - if parenlev == 0: - parenlevstart = (lnum, start, line) - parenlev = parenlev + 1 + parenstack.append((initial, lnum, start, line)) elif initial in ')]}': - parenlev = parenlev - 1 - if parenlev < 0: + if not parenstack: raise TokenError("unmatched '%s'" % initial, line, lnum, start + 1, token_list) + opening, lnum1, start1, line1 = parenstack.pop() + if not ((opening == "(" and initial == ")") or + (opening == "[" and initial == "]") or + (opening == "{" and initial == "}")): + msg = "closing parenthesis '%s' does not match opening parenthesis '%s'" % ( + initial, opening) + + if lnum1 != lnum: + msg += " on line " + str(lnum1) + raise TokenError( + msg, line, lnum, start + 1, token_list) if token in python_opmap: punct = python_opmap[token] else: @@ -241,7 +249,7 @@ if start < 0: start = pos if start", "exec") parser = pyparse.PythonParser(fakespace) tree = parser._parse(s, info) b = time.clock() - print title, (b-a) + print fn, (b-a) def entry_point(argv): - bench("foo") + if len(argv) == 2: + fn = argv[1] + else: + fn = "../../../../rpython/rlib/unicodedata/unicodedb_5_2_0.py" + fd = os.open(fn, os.O_RDONLY, 0777) + res = [] + while True: + s = os.read(fd, 4096) + if not s: + break + res.append(s) + os.close(fd) + s = "".join(res) + print len(s) + bench(fn, s) return 0 diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py --- a/pypy/interpreter/pyparser/test/test_pyparse.py +++ b/pypy/interpreter/pyparser/test/test_pyparse.py @@ -76,14 +76,14 @@ exc = py.test.raises(SyntaxError, parse, "name another for").value assert exc.msg == "invalid syntax" assert exc.lineno == 1 - assert exc.offset == 5 + assert exc.offset == 6 assert exc.text.startswith("name another for") exc = py.test.raises(SyntaxError, parse, "x = \"blah\n\n\n").value - assert exc.msg == "EOL while scanning string literal" + assert exc.msg == "end of line (EOL) while scanning string literal" assert exc.lineno == 1 assert exc.offset == 5 exc = py.test.raises(SyntaxError, parse, "x = '''\n\n\n").value - assert exc.msg == "EOF while scanning triple-quoted string literal" + assert exc.msg == "end of file (EOF) while scanning triple-quoted string literal" assert exc.lineno == 1 assert exc.offset == 5 assert exc.lastlineno == 3 @@ -112,7 +112,7 @@ assert exc.msg == "expected an indented block" assert exc.lineno == 3 assert exc.text.startswith("pass") - assert exc.offset == 0 + assert exc.offset == 1 input = "hi\n indented" exc = py.test.raises(IndentationError, parse, input).value assert exc.msg == "unexpected indent" @@ -120,6 +120,7 @@ exc = py.test.raises(IndentationError, parse, input).value assert exc.msg == "unindent does not match any outer indentation level" assert exc.lineno == 3 + assert exc.offset == 3 def test_mac_newline(self): self.parse("this_is\ra_mac\rfile") diff --git a/pypy/interpreter/pyparser/test/test_pytokenizer.py b/pypy/interpreter/pyparser/test/test_pytokenizer.py new file mode 100644 --- /dev/null +++ b/pypy/interpreter/pyparser/test/test_pytokenizer.py @@ -0,0 +1,66 @@ +import pytest +from pypy.interpreter.pyparser import pytokenizer +from pypy.interpreter.pyparser.pygram import tokens +from pypy.interpreter.pyparser.error import TokenError + +def tokenize(s): + return pytokenizer.generate_tokens(s.splitlines(True) + ["\n"], 0) + +def check_token_error(s, msg=None, pos=-1, line=-1): + error = pytest.raises(TokenError, tokenize, s) + if msg is not None: + assert error.value.msg == msg + if pos != -1: + assert error.value.offset == pos + if line != -1: + assert error.value.lineno == line + + +class TestTokenizer(object): + + def test_simple(self): + line = "a+1" + tks = tokenize(line) + assert tks == [ + (tokens.NAME, 'a', 1, 0, line), + (tokens.PLUS, '+', 1, 1, line), + (tokens.NUMBER, '1', 1, 2, line), + (tokens.NEWLINE, '', 2, 0, '\n'), + (tokens.NEWLINE, '', 2, 0, '\n'), + (tokens.ENDMARKER, '', 2, 0, ''), + ] + + def test_error_parenthesis(self): + for paren in "([{": + check_token_error(paren + "1 + 2", + "parenthesis is never closed", + 1) + + for paren in ")]}": + check_token_error("1 + 2" + paren, + "unmatched '%s'" % (paren, ), + 6) + + for i, opening in enumerate("([{"): + for j, closing in enumerate(")]}"): + if i == j: + continue + check_token_error(opening + "1\n" + closing, + "closing parenthesis '%s' does not match opening parenthesis '%s' on line 1" % (closing, opening), + pos=1, line=2) + check_token_error(opening + "1" + closing, + "closing parenthesis '%s' does not match opening parenthesis '%s'" % (closing, opening), + pos=3, line=1) + check_token_error(opening + closing, + "closing parenthesis '%s' does not match opening parenthesis '%s'" % (closing, opening), + pos=2, line=1) + + + def test_unknown_char(self): + check_token_error("?", "Unknown character", 1) + + def test_eol_string(self): + check_token_error("x = 'a", pos=5, line=1) + + def test_eof_triple_quoted(self): + check_token_error("'''", pos=1, line=1) diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py --- a/pypy/interpreter/test/test_compiler.py +++ b/pypy/interpreter/test/test_compiler.py @@ -77,7 +77,7 @@ """) assert self.space.unwrap(w_args) == ( 'unindent does not match any outer indentation level', - ('', 3, 0, ' y\n')) + ('', 3, 2, ' y\n')) def test_getcodeflags(self): code = self.compiler.compile('from __future__ import division\n', diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py --- a/pypy/interpreter/test/test_syntax.py +++ b/pypy/interpreter/test/test_syntax.py @@ -750,7 +750,7 @@ except SyntaxError as e: assert e.lineno == 4 assert e.text.endswith('a b c d e\n') - assert e.offset == e.text.index('b') + assert e.offset == e.text.index('b') + 1 # offset is 1-based else: raise Exception("no SyntaxError??") diff --git a/pypy/interpreter/test/test_typedef.py b/pypy/interpreter/test/test_typedef.py --- a/pypy/interpreter/test/test_typedef.py +++ b/pypy/interpreter/test/test_typedef.py @@ -423,3 +423,10 @@ def test_get_with_none_arg(self): raises(TypeError, type.__dict__['__mro__'].__get__, None) raises(TypeError, type.__dict__['__mro__'].__get__, None, None) + + def test_builtin_readonly_property(self): + import sys + x = lambda: 5 + e = raises(TypeError, 'x.func_globals = {}') + if '__pypy__' in sys.builtin_module_names: + assert str(e.value) == "readonly attribute 'func_globals'" diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py --- a/pypy/interpreter/typedef.py +++ b/pypy/interpreter/typedef.py @@ -309,12 +309,18 @@ self.reqcls, Arguments(space, [w_obj, space.newtext(self.name)])) + def readonly_attribute(self, space): # overwritten in cpyext + if self.name == '': + raise oefmt(space.w_TypeError, "readonly attribute") + else: + raise oefmt(space.w_TypeError, "readonly attribute '%s'", self.name) + def descr_property_set(self, space, w_obj, w_value): """property.__set__(obj, value) Change the value of the property of the given obj.""" fset = self.fset if fset is None: - raise oefmt(space.w_TypeError, "readonly attribute") + raise self.readonly_attribute(space) try: fset(self, space, w_obj, w_value) except DescrMismatch: diff --git a/pypy/module/_rawffi/alt/test/test_struct.py b/pypy/module/_rawffi/alt/test/test_struct.py --- a/pypy/module/_rawffi/alt/test/test_struct.py +++ b/pypy/module/_rawffi/alt/test/test_struct.py @@ -43,7 +43,11 @@ def setup_class(cls): BaseAppTestFFI.setup_class.im_func(cls) - @unwrap_spec(addr=int, typename='text', length=int) + from rpython.rlib import clibffi + from rpython.rlib.rarithmetic import r_uint + from rpython.rtyper.lltypesystem import lltype, rffi + + @unwrap_spec(addr=r_uint, typename='text', length=int) def read_raw_mem(space, addr, typename, length): import ctypes addr = ctypes.cast(addr, ctypes.c_void_p) @@ -58,9 +62,6 @@ else: cls.w_read_raw_mem = cls.space.wrap(interp2app(read_raw_mem)) # - from rpython.rlib import clibffi - from rpython.rlib.rarithmetic import r_uint - from rpython.rtyper.lltypesystem import lltype, rffi dummy_type = lltype.malloc(clibffi.FFI_TYPE_P.TO, flavor='raw') dummy_type.c_size = r_uint(123) dummy_type.c_alignment = rffi.cast(rffi.USHORT, 0) diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -640,7 +640,7 @@ 'Py_DivisionWarningFlag', 'Py_DontWriteBytecodeFlag', 'Py_NoUserSiteDirectory', '_Py_QnewFlag', 'Py_Py3kWarningFlag', 'Py_HashRandomizationFlag', '_Py_PackageContext', '_PyTraceMalloc_Track', '_PyTraceMalloc_Untrack', 'PyMem_Malloc', - 'Py_IncRef', 'Py_DecRef', 'PyObject_Free', 'PyObject_GC_Del', 'PyType_GenericAlloc', + 'PyObject_Free', 'PyObject_GC_Del', 'PyType_GenericAlloc', '_PyObject_New', '_PyObject_NewVar', '_PyObject_GC_New', '_PyObject_GC_NewVar', 'PyObject_Init', 'PyObject_InitVar', 'PyInt_FromLong', diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py --- a/pypy/module/cpyext/pyobject.py +++ b/pypy/module/cpyext/pyobject.py @@ -60,10 +60,10 @@ def _cpyext_attach_pyobj(self, space, py_obj): self._cpy_ref = py_obj - rawrefcount.create_link_pyobj(self, py_obj) + rawrefcount.create_link_pypy(self, py_obj) cls._cpyext_attach_pyobj = _cpyext_attach_pyobj -add_direct_pyobj_storage(W_BaseCPyObject) +add_direct_pyobj_storage(W_BaseCPyObject) add_direct_pyobj_storage(W_TypeObject) add_direct_pyobj_storage(W_NoneObject) add_direct_pyobj_storage(W_BoolObject) @@ -414,3 +414,14 @@ @cpython_api([rffi.VOIDP], lltype.Signed, error=CANNOT_FAIL) def _Py_HashPointer(space, ptr): return rffi.cast(lltype.Signed, ptr) + + at cpython_api([PyObject], lltype.Void) +def Py_IncRef(space, obj): + # used only ifdef PYPY_DEBUG_REFCOUNT + if obj: + incref(space, obj) + + at cpython_api([PyObject], lltype.Void) +def Py_DecRef(space, obj): + # used only ifdef PYPY_DEBUG_REFCOUNT + decref(space, obj) diff --git a/pypy/module/cpyext/setobject.py b/pypy/module/cpyext/setobject.py --- a/pypy/module/cpyext/setobject.py +++ b/pypy/module/cpyext/setobject.py @@ -1,4 +1,4 @@ -from pypy.interpreter.error import oefmt +from pypy.interpreter.error import OperationError, oefmt from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.cpyext.api import (cpython_api, Py_ssize_t, CANNOT_FAIL, build_type_checkers) @@ -64,8 +64,13 @@ instance of set or its subtype.""" if not PySet_Check(space, w_s): PyErr_BadInternalCall(space) - space.call_method(space.w_set, 'discard', w_s, w_obj) - return 0 + try: + space.call_method(space.w_set, 'remove', w_s, w_obj) + except OperationError as e: + if e.match(space, space.w_KeyError): + return 0 + raise + return 1 @cpython_api([PyObject], PyObject) diff --git a/pypy/module/cpyext/src/object.c b/pypy/module/cpyext/src/object.c --- a/pypy/module/cpyext/src/object.c +++ b/pypy/module/cpyext/src/object.c @@ -5,18 +5,6 @@ extern void _PyPy_Free(void *ptr); extern void *_PyPy_Malloc(Py_ssize_t size); -void -Py_IncRef(PyObject *o) -{ - Py_XINCREF(o); -} - -void -Py_DecRef(PyObject *o) -{ - Py_XDECREF(o); -} - /* * The actual value of this variable will be the address of * pyobject.w_marker_deallocating, and will be set by diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c --- a/pypy/module/cpyext/test/array.c +++ b/pypy/module/cpyext/test/array.c @@ -2492,6 +2492,87 @@ return PyLong_FromLong(obj1->ob_type->tp_dealloc == obj2->ob_type->tp_dealloc); } +static PyObject * +subclass_with_attribute(PyObject *self, PyObject* args) { + /* what happens when we use tp_alloc to create the subclass, then + * assign to the w_obj via python, then get the GC to collect? + * The w_obj should not be collected!! + */ + PyObject * obj, *sub, *attrib, *funcname, *attribname, *collect, *res, *tup; + PyTypeObject * subtype; + int i; + if (!PyArg_ParseTuple(args, "OOOO", &obj, &funcname, &attribname, &collect)) { + return NULL; + } + if (!PyType_Check(obj)) { + PyErr_SetString(PyExc_TypeError, + "expected type object"); + return NULL; + } + subtype = (PyTypeObject*)obj; + sub = subtype->tp_alloc(subtype, 0); + if (!sub) { + return NULL; + } + attrib = PyObject_GetAttr(sub, funcname); + if (!attrib || (attrib == Py_None) ) { + PyErr_SetString(PyExc_ValueError, + "could not find function to call"); + Py_XDECREF(attrib); + Py_DECREF(sub); + return NULL; + } + tup = PyTuple_New(0); + /* + #ifdef PYPY_VERSION + printf("calling addattrib pypylink %lu \n", sub->ob_pypy_link); + #endif + */ + res = PyObject_Call(attrib, tup, NULL); + /* + #ifdef PYPY_VERSION + printf("after addattrib pypylink %lu \n", sub->ob_pypy_link); + #endif + */ + Py_DECREF(attrib); + if (res == NULL) { + Py_DECREF(tup); + Py_DECREF(sub); + return NULL; + } + Py_DECREF(res); + for(i=0; i<10; i++) { + /* + #ifdef PYPY_VERSION + printf("starting loop iteration %d refcnt %lu pypylink %lu \n", i, + sub->ob_refcnt, sub->ob_pypy_link); + #else + printf("starting loop iteration %d refcnt %lu\n", i, sub->ob_refcnt); + #endif + */ + attrib = PyObject_GetAttr(sub, attribname); + if (!attrib || (attrib == Py_None)) { + PyErr_SetString(PyExc_ValueError, + "could not find attrib on object"); + Py_XDECREF(attrib); + Py_DECREF(tup); + Py_DECREF(sub); + return NULL; + } + Py_XDECREF(attrib); + res = PyObject_Call(collect, tup, NULL); + if (res == NULL) { + Py_DECREF(tup); + Py_DECREF(sub); + return NULL; + } + Py_DECREF(res); + } + Py_DECREF(tup); + Py_DECREF(sub); + Py_RETURN_NONE; +} + /*********************** Install Module **************************/ static PyMethodDef a_methods[] = { @@ -2502,6 +2583,7 @@ {"create_and_release_buffer", (PyCFunction)create_and_release_buffer, METH_O, NULL}, {"same_dealloc", (PyCFunction)same_dealloc, METH_VARARGS, NULL}, {"getitem", (PyCFunction)getitem, METH_VARARGS, NULL}, + {"subclass_with_attribute", (PyCFunction)subclass_with_attribute, METH_VARARGS, NULL}, {NULL, NULL, 0, NULL} /* Sentinel */ }; diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py --- a/pypy/module/cpyext/test/test_arraymodule.py +++ b/pypy/module/cpyext/test/test_arraymodule.py @@ -186,3 +186,15 @@ # array_subscr does) raises(IndexError, module.getitem, a, -5) + def test_subclass_with_attribute(self): + module = self.import_module(name='array') + class Sub(module.array): + def addattrib(self): + print('called addattrib') + self.attrib = True + import gc + module.subclass_with_attribute(Sub, "addattrib", "attrib", gc.collect) + if self.runappdirect: + assert Sub.__module__ == 'pypy.module.cpyext.test.test_arraymodule' + assert str(Sub) == "" + diff --git a/pypy/module/cpyext/test/test_setobject.py b/pypy/module/cpyext/test/test_setobject.py --- a/pypy/module/cpyext/test/test_setobject.py +++ b/pypy/module/cpyext/test/test_setobject.py @@ -28,7 +28,11 @@ assert api.PySet_Size(w_set) == 4 api.PySet_Add(w_set, space.wrap(6)) assert api.PySet_Size(w_set) == 5 - api.PySet_Discard(w_set, space.wrap(6)) + res = api.PySet_Discard(w_set, space.wrap(6)) + assert res == 1 + assert api.PySet_Size(w_set) == 4 + res = api.PySet_Discard(w_set, space.wrap(6)) + assert res == 0 assert api.PySet_Size(w_set) == 4 def test_set_contains(self, space, api): diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py --- a/pypy/module/cpyext/test/test_tupleobject.py +++ b/pypy/module/cpyext/test/test_tupleobject.py @@ -67,6 +67,18 @@ assert space.int_w(space.getitem(w_tuple, space.wrap(i))) == 42 + i decref(space, ar[0]) + py_tuple = state.ccall("PyTuple_New", 1) + ar[0] = py_tuple + api._PyTuple_Resize(ar, 1) + assert api.PyTuple_Size(ar[0]) == 1 + decref(space, ar[0]) + + py_tuple = state.ccall("PyTuple_New", 1) + ar[0] = py_tuple + api._PyTuple_Resize(ar, 5) + assert api.PyTuple_Size(ar[0]) == 5 + decref(space, ar[0]) + lltype.free(ar, flavor='raw') def test_setitem(self, space, api): diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -5,7 +5,7 @@ from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.api import generic_cpy_call from pypy.module.cpyext.pyobject import make_ref, from_ref, decref, as_pyobj -from pypy.module.cpyext.typeobject import PyTypeObjectPtr +from pypy.module.cpyext.typeobject import PyTypeObjectPtr, W_PyCTypeObject class AppTestTypeObject(AppTestCpythonExtensionBase): @@ -412,33 +412,42 @@ def test_type_dict(self): foo = self.import_module("foo") module = self.import_extension('test', [ - ("hack_tp_dict", "METH_O", + ("hack_tp_dict", "METH_VARARGS", ''' - PyTypeObject *type = args->ob_type; + PyTypeObject *type, *obj; PyObject *a1 = PyLong_FromLong(1); PyObject *a2 = PyLong_FromLong(2); PyObject *value; + PyObject * key; + if (!PyArg_ParseTuple(args, "OO", &obj, &key)) + return NULL; + type = obj->ob_type; - if (PyDict_SetItemString(type->tp_dict, "a", + if (PyDict_SetItem(type->tp_dict, key, a1) < 0) return NULL; Py_DECREF(a1); PyType_Modified(type); - value = PyObject_GetAttrString((PyObject *)type, "a"); + value = PyObject_GetAttr((PyObject *)type, key); Py_DECREF(value); - if (PyDict_SetItemString(type->tp_dict, "a", + if (PyDict_SetItem(type->tp_dict, key, a2) < 0) return NULL; Py_DECREF(a2); PyType_Modified(type); - value = PyObject_GetAttrString((PyObject *)type, "a"); + value = PyObject_GetAttr((PyObject *)type, key); return value; ''' ) ]) obj = foo.new() - assert module.hack_tp_dict(obj) == 2 + assert module.hack_tp_dict(obj, "a") == 2 + class Sub(foo.fooType): + pass + obj = Sub() + assert module.hack_tp_dict(obj, "b") == 2 + def test_tp_descr_get(self): module = self.import_extension('foo', [ @@ -560,6 +569,23 @@ assert w_obj is None assert api.PyErr_Occurred() is None + def test_subclass_not_PyCTypeObject(self, space, api): + pyobj = make_ref(space, api.PyLong_Type) + py_type = rffi.cast(PyTypeObjectPtr, pyobj) + w_pyclass = W_PyCTypeObject(space, py_type) + w_class = space.appexec([w_pyclass], """(base): + class Sub(base): + def addattrib(self, value): + self.attrib = value + return Sub + """) + assert w_pyclass in w_class.mro_w + assert isinstance(w_pyclass, W_PyCTypeObject) + assert not isinstance(w_class, W_PyCTypeObject) + assert w_pyclass.is_cpytype() + # XXX document the current status, not clear if this is desirable + assert w_class.is_cpytype() + class AppTestSlots(AppTestCpythonExtensionBase): def setup_class(cls): @@ -1600,6 +1626,65 @@ pass C(42) # assert is not aborting + def test_getset(self): + module = self.import_extension('foo', [ + ("get_instance", "METH_NOARGS", + ''' + return PyObject_New(PyObject, &Foo_Type); + ''' + ), ("get_number", "METH_NOARGS", + ''' + return PyInt_FromLong(my_global_number); + ''' + )], prologue=''' + #if PY_MAJOR_VERSION > 2 + #define PyInt_FromLong PyLong_FromLong + #define PyInt_AsLong PyLong_AsLong + #endif + static long my_global_number; + static PyTypeObject Foo_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "foo.foo", + }; + static PyObject *bar_get(PyObject *foo, void *closure) + { + return PyInt_FromLong(1000 + (long)closure); + } + static PyObject *baz_get(PyObject *foo, void *closure) + { + return PyInt_FromLong(2000 + (long)closure); + } + static int baz_set(PyObject *foo, PyObject *x, void *closure) + { + if (x != NULL) + my_global_number = 3000 + (long)closure + PyInt_AsLong(x); + else + my_global_number = 4000 + (long)closure; + return 0; + } + static PyGetSetDef foo_getset[] = { + { "bar", bar_get, NULL, "mybardoc", (void *)42 }, + { "baz", baz_get, baz_set, "mybazdoc", (void *)43 }, + { NULL } + }; + ''', more_init = ''' + Foo_Type.tp_getset = foo_getset; + Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT; + if (PyType_Ready(&Foo_Type) < 0) INITERROR; + ''') + foo = module.get_instance() + assert foo.bar == 1042 + assert foo.bar == 1042 + assert foo.baz == 2043 + foo.baz = 50000 + assert module.get_number() == 53043 + e = raises(AttributeError, "foo.bar = 0") + assert str(e.value).startswith("attribute 'bar' of '") + assert str(e.value).endswith("foo' objects is not writable") + del foo.baz + assert module.get_number() == 4043 + raises(AttributeError, "del foo.bar") + class AppTestHashable(AppTestCpythonExtensionBase): def test_unhashable(self): diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py --- a/pypy/module/cpyext/tupleobject.py +++ b/pypy/module/cpyext/tupleobject.py @@ -187,6 +187,8 @@ PyErr_BadInternalCall(space) oldref = rffi.cast(PyTupleObject, ref) oldsize = oldref.c_ob_size + if oldsize == newsize: + return 0 ptup = state.ccall("PyTuple_New", newsize) if not ptup: state.check_and_raise_exception(always=True) @@ -199,8 +201,9 @@ to_cp = newsize for i in range(to_cp): ob = oldref.c_ob_item[i] - incref(space, ob) - newref.c_ob_item[i] = ob + if ob: + incref(space, ob) + newref.c_ob_item[i] = ob except: decref(space, p_ref[0]) p_ref[0] = lltype.nullptr(PyObject.TO) diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -1,6 +1,6 @@ import os -from rpython.rlib import jit +from rpython.rlib import jit, rawrefcount from rpython.rlib.objectmodel import specialize, we_are_translated from rpython.rtyper.lltypesystem import rffi, lltype @@ -54,19 +54,26 @@ class W_GetSetPropertyEx(GetSetProperty): def __init__(self, getset, w_type): self.getset = getset - self.name = rffi.charp2str(getset.c_name) self.w_type = w_type - doc = set = get = None + doc = fset = fget = fdel = None if doc: # XXX dead code? doc = rffi.charp2str(getset.c_doc) if getset.c_get: - get = GettersAndSetters.getter.im_func + fget = GettersAndSetters.getter.im_func if getset.c_set: - set = GettersAndSetters.setter.im_func - GetSetProperty.__init__(self, get, set, None, doc, + fset = GettersAndSetters.setter.im_func + fdel = GettersAndSetters.deleter.im_func + GetSetProperty.__init__(self, fget, fset, fdel, doc, cls=None, use_closure=True, tag="cpyext_1") + self.name = rffi.charp2str(getset.c_name) + + def readonly_attribute(self, space): # overwritten + raise oefmt(space.w_AttributeError, + "attribute '%s' of '%N' objects is not writable", + self.name, self.w_type) + def PyDescr_NewGetSet(space, getset, w_type): return W_GetSetPropertyEx(getset, w_type) @@ -454,6 +461,16 @@ state = space.fromcache(State) state.check_and_raise_exception() + def deleter(self, space, w_self): + assert isinstance(self, W_GetSetPropertyEx) + check_descr(space, w_self, self.w_type) + res = generic_cpy_call( + space, self.getset.c_set, w_self, None, + self.getset.c_closure) + if rffi.cast(lltype.Signed, res) < 0: + state = space.fromcache(State) + state.check_and_raise_exception() + def member_getter(self, space, w_self): assert isinstance(self, W_MemberDescr) check_descr(space, w_self, self.w_type) @@ -517,6 +534,10 @@ self.w_doc = space.newtext( rffi.charp2str(cts.cast('char*', pto.c_tp_doc))) + def _cpyext_attach_pyobj(self, space, py_obj): + self._cpy_ref = py_obj + rawrefcount.create_link_pyobj(self, py_obj) + @bootstrap_function def init_typeobject(space): make_typedescr(space.w_type.layout.typedef, @@ -777,7 +798,6 @@ try: w_obj = _type_realize(space, py_obj) finally: - name = rffi.charp2str(cts.cast('char*', pto.c_tp_name)) pto.c_tp_flags &= ~Py_TPFLAGS_READYING pto.c_tp_flags |= Py_TPFLAGS_READY return w_obj @@ -884,7 +904,6 @@ base = pto.c_tp_base base_pyo = rffi.cast(PyObject, pto.c_tp_base) if base and not base.c_tp_flags & Py_TPFLAGS_READY: - name = rffi.charp2str(cts.cast('char*', base.c_tp_name)) type_realize(space, base_pyo) if base and not pto.c_ob_type: # will be filled later pto.c_ob_type = base.c_ob_type diff --git a/pypy/tool/release/force-builds.py b/pypy/tool/release/force-builds.py --- a/pypy/tool/release/force-builds.py +++ b/pypy/tool/release/force-builds.py @@ -31,6 +31,9 @@ 'pypy-c-jit-linux-s390x', 'build-pypy-c-jit-linux-armhf-raspbian', 'build-pypy-c-jit-linux-armel', + 'rpython-linux-x86-32', + 'rpython-linux-x86-64' + 'rpython-win-x86-32' ] def get_user(): diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -644,7 +644,10 @@ def get_rpy_type_index(gcref): from rpython.rlib.rarithmetic import intmask Class = gcref._x.__class__ - return intmask(id(Class)) + i = intmask(id(Class)) + if i < 0: + i = ~i # always return a positive number, at least + return i def cast_gcref_to_int(gcref): # This is meant to be used on cast_instance_to_gcref results. diff --git a/rpython/rlib/rvmprof/src/shared/_vmprof.c b/rpython/rlib/rvmprof/src/shared/_vmprof.c --- a/rpython/rlib/rvmprof/src/shared/_vmprof.c +++ b/rpython/rlib/rvmprof/src/shared/_vmprof.c @@ -36,6 +36,8 @@ register PY_STACK_FRAME_T * callee_saved asm("rbx"); #elif defined(X86_32) register PY_STACK_FRAME_T * callee_saved asm("edi"); +#elif defined(__arm__) + register PY_STACK_FRAME_T * callee_saved asm("r4"); #else # error "platform not supported" #endif @@ -45,6 +47,8 @@ "movq %1, %0\t\n" #elif defined(X86_32) "mov %1, %0\t\n" +#elif defined(__arm__) + "mov %1, %0\t\n" #else # error "platform not supported" #endif diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.c b/rpython/rlib/rvmprof/src/shared/vmp_stack.c --- a/rpython/rlib/rvmprof/src/shared/vmp_stack.c +++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.c @@ -16,7 +16,7 @@ #ifdef VMP_SUPPORTS_NATIVE_PROFILING -#ifdef VMPROF_LINUX +#if defined(VMPROF_LINUX) || defined(VMPROF_BSD) #include "unwind/vmprof_unwind.h" typedef mcontext_t unw_context_t; @@ -510,13 +510,15 @@ static const char * vmprof_error = NULL; static void * libhandle = NULL; - #ifdef VMPROF_LINUX +#include #define LIBUNWIND "libunwind.so" #ifdef __i386__ #define PREFIX "x86" +#define LIBUNWIND_SUFFIX "" #elif __x86_64__ #define PREFIX "x86_64" +#define LIBUNWIND_SUFFIX "-x86_64" #endif #define U_PREFIX "_U" #define UL_PREFIX "_UL" @@ -524,10 +526,41 @@ int vmp_native_enable(void) { #ifdef VMPROF_LINUX + void * oldhandle = NULL; + struct link_map * map = NULL; if (libhandle == NULL) { + // on linux, the wheel includes the libunwind shared object. + libhandle = dlopen(NULL, RTLD_NOW); + if (libhandle != NULL) { + // load the link map, it will contain an entry to + // .libs_vmprof/libunwind-...so, this is the file that is + // distributed with the wheel. + if (dlinfo(libhandle, RTLD_DI_LINKMAP, &map) != 0) { + (void)dlclose(libhandle); + libhandle = NULL; + goto bail_out; + } + // grab the new handle + do { + if (strstr(map->l_name, ".libs_vmprof/libunwind" LIBUNWIND_SUFFIX) != NULL) { + oldhandle = libhandle; + libhandle = dlopen(map->l_name, RTLD_LAZY|RTLD_LOCAL); + (void)dlclose(oldhandle); + oldhandle = NULL; + goto loaded_libunwind; + } + map = map->l_next; + } while (map != NULL); + // did not find .libs_vmprof/libunwind... + (void)dlclose(libhandle); + libhandle = NULL; + } + + // fallback! try to load the system's libunwind.so if ((libhandle = dlopen(LIBUNWIND, RTLD_LAZY | RTLD_LOCAL)) == NULL) { goto bail_out; } +loaded_libunwind: if ((unw_get_reg = dlsym(libhandle, UL_PREFIX PREFIX "_get_reg")) == NULL) { goto bail_out; } diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_common.h b/rpython/rlib/rvmprof/src/shared/vmprof_common.h --- a/rpython/rlib/rvmprof/src/shared/vmprof_common.h +++ b/rpython/rlib/rvmprof/src/shared/vmprof_common.h @@ -23,6 +23,10 @@ #include #endif +#ifdef VMPROF_BSD +#include +#endif + #define MAX_FUNC_NAME 1024 #ifdef VMPROF_UNIX From pypy.commits at gmail.com Sat Apr 14 14:56:28 2018 From: pypy.commits at gmail.com (arigo) Date: Sat, 14 Apr 2018 11:56:28 -0700 (PDT) Subject: [pypy-commit] pypy issue2752: fix 2ef33ef89243 Message-ID: <5ad24edc.6ca0df0a.801f5.6105@mx.google.com> Author: Armin Rigo Branch: issue2752 Changeset: r94327:f54cc653fb4a Date: 2018-04-14 20:55 +0200 http://bitbucket.org/pypy/pypy/changeset/f54cc653fb4a/ Log: fix 2ef33ef89243 diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -736,9 +736,14 @@ shadow = self.nursery_objects_shadows.get(obj) ll_assert(shadow != llmemory.NULL, "GCFLAG_HAS_SHADOW but no shadow found") - return shadow + else: + shadow = self._allocate_shadow(obj) - return self._allocate_shadow(obj, copy=True) + if (self.header(obj).tid & GCFLAG_SHADOW_INITIALIZED) == 0: + self.header(obj).tid |= GCFLAG_SHADOW_INITIALIZED + totalsize = self.get_size(obj) + llmemory.raw_memcopy(obj, shadow, totalsize) + return shadow def collect(self, gen=2): """Do a minor (gen=0), start a major (gen=1), or do a full @@ -2645,7 +2650,7 @@ # id() and identityhash() support @specialize.arg(2) - def _allocate_shadow(self, obj, copy=False): + def _allocate_shadow(self, obj): size_gc_header = self.gcheaderbuilder.size_gc_header size = self.get_size(obj) shadowhdr = self._malloc_out_of_nursery(size_gc_header + @@ -2667,12 +2672,6 @@ # self.header(obj).tid |= GCFLAG_HAS_SHADOW self.nursery_objects_shadows.setitem(obj, shadow) - - if copy: - self.header(obj).tid |= GCFLAG_SHADOW_INITIALIZED - totalsize = self.get_size(obj) - llmemory.raw_memcopy(obj, shadow, totalsize) - return shadow def _find_shadow(self, obj): From pypy.commits at gmail.com Sat Apr 14 15:00:37 2018 From: pypy.commits at gmail.com (arigo) Date: Sat, 14 Apr 2018 12:00:37 -0700 (PDT) Subject: [pypy-commit] pypy issue2752: translation fix Message-ID: <5ad24fd5.166a1c0a.806bd.8ecc@mx.google.com> Author: Armin Rigo Branch: issue2752 Changeset: r94328:0699dc997d8a Date: 2018-04-14 20:58 +0200 http://bitbucket.org/pypy/pypy/changeset/0699dc997d8a/ Log: translation fix diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -2649,7 +2649,6 @@ # ---------- # id() and identityhash() support - @specialize.arg(2) def _allocate_shadow(self, obj): size_gc_header = self.gcheaderbuilder.size_gc_header size = self.get_size(obj) From pypy.commits at gmail.com Sat Apr 14 15:57:55 2018 From: pypy.commits at gmail.com (arigo) Date: Sat, 14 Apr 2018 12:57:55 -0700 (PDT) Subject: [pypy-commit] pypy default: hg merge issue2752 Message-ID: <5ad25d43.b7b0df0a.86fcb.6c0c@mx.google.com> Author: Armin Rigo Branch: Changeset: r94330:91ffd8f1e3f5 Date: 2018-04-14 21:56 +0200 http://bitbucket.org/pypy/pypy/changeset/91ffd8f1e3f5/ Log: hg merge issue2752 diff --git a/pypy/module/cpyext/test/test_bufferobject.py b/pypy/module/cpyext/test/test_bufferobject.py --- a/pypy/module/cpyext/test/test_bufferobject.py +++ b/pypy/module/cpyext/test/test_bufferobject.py @@ -4,6 +4,7 @@ from pypy.module.cpyext.api import PyObject class AppTestBufferObject(AppTestCpythonExtensionBase): + def test_FromMemory(self): module = self.import_extension('foo', [ ("get_FromMemory", "METH_NOARGS", @@ -62,3 +63,61 @@ a = array.array('c', 'text') b = buffer(a) assert module.roundtrip(b) == 'text' + + + def test_issue2752(self): + iterations = 10 + if self.runappdirect: + iterations = 2000 + module = self.import_extension('foo', [ + ("test_mod", 'METH_VARARGS', + """ + PyObject *obj; + Py_buffer bp; + if (!PyArg_ParseTuple(args, "O", &obj)) + return NULL; + + if (PyObject_GetBuffer(obj, &bp, PyBUF_SIMPLE) == -1) + return NULL; + + if (((unsigned char*)bp.buf)[0] != '0') { + void * buf = (void*)bp.buf; + unsigned char val[4]; + unsigned char * s = PyString_AsString(obj); + memcpy(val, bp.buf, 4); + PyBuffer_Release(&bp); + if (PyObject_GetBuffer(obj, &bp, PyBUF_SIMPLE) == -1) + return NULL; + PyErr_Format(PyExc_ValueError, + "mismatch: %p [%x %x %x %x...] now %p [%x %x %x %x...] as str '%s'", + buf, val[0], val[1], val[2], val[3], + (void *)bp.buf, + ((unsigned char*)bp.buf)[0], + ((unsigned char*)bp.buf)[1], + ((unsigned char*)bp.buf)[2], + ((unsigned char*)bp.buf)[3], + s); + PyBuffer_Release(&bp); + return NULL; + } + + PyBuffer_Release(&bp); + Py_RETURN_NONE; + """), + ]) + bufsize = 4096 + def getdata(bufsize): + data = b'01234567' + for x in range(18): + data += data + if len(data) >= bufsize: + break + return data + for j in range(iterations): + block = getdata(bufsize) + assert block[:8] == '01234567' + try: + module.test_mod(block) + except ValueError as e: + print("%s at it=%d" % (e, j)) + assert False diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -736,9 +736,14 @@ shadow = self.nursery_objects_shadows.get(obj) ll_assert(shadow != llmemory.NULL, "GCFLAG_HAS_SHADOW but no shadow found") - return shadow + else: + shadow = self._allocate_shadow(obj) - return self._allocate_shadow(obj, copy=True) + if (self.header(obj).tid & GCFLAG_SHADOW_INITIALIZED) == 0: + self.header(obj).tid |= GCFLAG_SHADOW_INITIALIZED + totalsize = self.get_size(obj) + llmemory.raw_memcopy(obj, shadow, totalsize) + return shadow def collect(self, gen=2): """Do a minor (gen=0), start a major (gen=1), or do a full @@ -2644,8 +2649,7 @@ # ---------- # id() and identityhash() support - @specialize.arg(2) - def _allocate_shadow(self, obj, copy=False): + def _allocate_shadow(self, obj): size_gc_header = self.gcheaderbuilder.size_gc_header size = self.get_size(obj) shadowhdr = self._malloc_out_of_nursery(size_gc_header + @@ -2667,12 +2671,6 @@ # self.header(obj).tid |= GCFLAG_HAS_SHADOW self.nursery_objects_shadows.setitem(obj, shadow) - - if copy: - self.header(obj).tid |= GCFLAG_SHADOW_INITIALIZED - totalsize = size_gc_header + self.get_size(obj) - llmemory.raw_memcopy(obj - size_gc_header, shadow, totalsize) - return shadow def _find_shadow(self, obj): diff --git a/rpython/memory/gc/test/test_direct.py b/rpython/memory/gc/test/test_direct.py --- a/rpython/memory/gc/test/test_direct.py +++ b/rpython/memory/gc/test/test_direct.py @@ -671,6 +671,25 @@ self.gc.debug_gc_step_until(incminimark.STATE_SCANNING) assert self.stackroots[1].x == 13 + def test_move_out_of_nursery(self): + obj0 = self.malloc(S) + obj0.x = 123 + adr1 = self.gc.move_out_of_nursery(llmemory.cast_ptr_to_adr(obj0)) + obj1 = llmemory.cast_adr_to_ptr(adr1, lltype.Ptr(S)) + assert obj1.x == 123 + # + import pytest + obj2 = self.malloc(S) + obj2.x = 456 + adr3 = self.gc._find_shadow(llmemory.cast_ptr_to_adr(obj2)) + obj3 = llmemory.cast_adr_to_ptr(adr3, lltype.Ptr(S)) + with pytest.raises(lltype.UninitializedMemoryAccess): + obj3.x # the shadow is not populated yet + adr4 = self.gc.move_out_of_nursery(llmemory.cast_ptr_to_adr(obj2)) + assert adr4 == adr3 + assert obj3.x == 456 # it is populated now + + class TestIncrementalMiniMarkGCFull(DirectGCTest): from rpython.memory.gc.incminimark import IncrementalMiniMarkGC as GCClass def test_malloc_fixedsize_no_cleanup(self): From pypy.commits at gmail.com Sat Apr 14 15:57:53 2018 From: pypy.commits at gmail.com (arigo) Date: Sat, 14 Apr 2018 12:57:53 -0700 (PDT) Subject: [pypy-commit] pypy issue2752: close branch, ready to merge Message-ID: <5ad25d41.1ca4df0a.e9a30.3678@mx.google.com> Author: Armin Rigo Branch: issue2752 Changeset: r94329:fee8b503c7e1 Date: 2018-04-14 21:56 +0200 http://bitbucket.org/pypy/pypy/changeset/fee8b503c7e1/ Log: close branch, ready to merge From pypy.commits at gmail.com Sat Apr 14 17:20:50 2018 From: pypy.commits at gmail.com (arigo) Date: Sat, 14 Apr 2018 14:20:50 -0700 (PDT) Subject: [pypy-commit] pypy default: Small tweaks and documentation for issue2752 Message-ID: <5ad270b2.e8361c0a.57bca.3a94@mx.google.com> Author: Armin Rigo Branch: Changeset: r94331:f154dd05a3a5 Date: 2018-04-14 23:13 +0200 http://bitbucket.org/pypy/pypy/changeset/f154dd05a3a5/ Log: Small tweaks and documentation for issue2752 diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -6,3 +6,7 @@ .. startrev: f22145c34985 +.. branch: issue2752 + +Fix a rare GC bug that was introduced more than one year ago, but was +not diagnosed before issue #2752. diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -731,14 +731,11 @@ def move_out_of_nursery(self, obj): # called twice, it should return the same shadow object, - # and not creating another shadow object - if self.header(obj).tid & GCFLAG_HAS_SHADOW: - shadow = self.nursery_objects_shadows.get(obj) - ll_assert(shadow != llmemory.NULL, - "GCFLAG_HAS_SHADOW but no shadow found") - else: - shadow = self._allocate_shadow(obj) - + # and not creating another shadow object. As a safety feature, + # when called on a non-nursery object, do nothing. + if not self.is_in_nursery(obj): + return obj + shadow = self._find_shadow(obj) if (self.header(obj).tid & GCFLAG_SHADOW_INITIALIZED) == 0: self.header(obj).tid |= GCFLAG_SHADOW_INITIALIZED totalsize = self.get_size(obj) @@ -2079,13 +2076,12 @@ ll_assert(newobj != llmemory.NULL, "GCFLAG_HAS_SHADOW but no shadow found") newhdr = newobj - size_gc_header # - # Remove the flag GCFLAG_HAS_SHADOW, so that it doesn't get - # copied to the shadow itself. - self.header(obj).tid &= ~GCFLAG_HAS_SHADOW + # The flags GCFLAG_HAS_SHADOW and GCFLAG_SHADOW_INITIALIZED + # have no meaning in non-nursery objects. We don't need to + # remove them explicitly here before doing the copy. tid = self.header(obj).tid if (tid & GCFLAG_SHADOW_INITIALIZED) != 0: copy = False - self.header(obj).tid &= ~GCFLAG_SHADOW_INITIALIZED # totalsize = size_gc_header + self.get_size(obj) self.nursery_surviving_size += raw_malloc_usage(totalsize) From pypy.commits at gmail.com Sat Apr 14 20:43:36 2018 From: pypy.commits at gmail.com (rlamy) Date: Sat, 14 Apr 2018 17:43:36 -0700 (PDT) Subject: [pypy-commit] pypy apptest-file: Adding some tests (WIP) Message-ID: <5ad2a038.563f1c0a.bf1d6.9dd9@mx.google.com> Author: Ronan Lamy Branch: apptest-file Changeset: r94332:e09619779251 Date: 2018-04-15 01:42 +0100 http://bitbucket.org/pypy/pypy/changeset/e09619779251/ Log: Adding some tests (WIP) diff --git a/pypy/tool/pytest/test/apptest_xx.py b/pypy/tool/pytest/test/apptest_xx.py new file mode 100644 --- /dev/null +++ b/pypy/tool/pytest/test/apptest_xx.py @@ -0,0 +1,3 @@ +def test_fail(): + x = 'foo' + assert x == 'bar' diff --git a/pypy/tool/pytest/test/test_appsupport.py b/pypy/tool/pytest/test/test_appsupport.py --- a/pypy/tool/pytest/test/test_appsupport.py +++ b/pypy/tool/pytest/test/test_appsupport.py @@ -23,10 +23,16 @@ def test_method(self): pass """) + testdir.makepyfile(apptest_collection=""" + def test_app(): + pass + """) setpypyconftest(testdir) result = testdir.runpytest("--collectonly") assert result.ret == 0 result.stdout.fnmatch_lines([ + "*AppTestModule*apptest_collection*", + "*AppTestFunction*test_app*", "*Function*test_func*", "*Class*TestClassInt*", "*Function*test_method*", @@ -126,6 +132,45 @@ "*E*application-level*KeyError*42*", ]) +def test_apptest_raise(testdir): + setpypyconftest(testdir) + p = testdir.makepyfile(apptest_raise=""" + def test_raise(): + raise KeyError(42) + """) + result = testdir.runpytest(p) + assert result.ret == 1 + result.stdout.fnmatch_lines([ + "*E*application-level*KeyError*42*", + ]) + +def test_apptest_fail_plain(testdir): + setpypyconftest(testdir) + p = testdir.makepyfile(apptest_fail=""" + def test_fail(): + x = 'foo' + assert x == 'bar' + """) + result = testdir.runpytest(p) + assert result.ret == 1 + result.stdout.fnmatch_lines([ + "*E*application-level*KeyError*42*", + ]) + +def test_apptest_fail_rewrite(testdir): + setpypyconftest(testdir) + p = testdir.makepyfile(apptest_fail_rewrite=""" + def test_fail(): + x = 'foo' + assert x == 'bar' + """) + result = testdir.runpytest(p, "--applevel-rewrite") + assert result.ret == 1 + result.stdout.fnmatch_lines([ + "*E*application-level*KeyError*42*", + ]) + + def app_test_raises(): info = raises(TypeError, id) assert info.type is TypeError From pypy.commits at gmail.com Sun Apr 15 02:00:52 2018 From: pypy.commits at gmail.com (mattip) Date: Sat, 14 Apr 2018 23:00:52 -0700 (PDT) Subject: [pypy-commit] pypy default: fix python2-only test Message-ID: <5ad2ea94.130d1c0a.93aaf.d0a6@mx.google.com> Author: Matti Picus Branch: Changeset: r94334:4d2fb1545820 Date: 2018-04-15 08:59 +0300 http://bitbucket.org/pypy/pypy/changeset/4d2fb1545820/ Log: fix python2-only test diff --git a/pypy/module/cpyext/test/test_bufferobject.py b/pypy/module/cpyext/test/test_bufferobject.py --- a/pypy/module/cpyext/test/test_bufferobject.py +++ b/pypy/module/cpyext/test/test_bufferobject.py @@ -83,7 +83,7 @@ if (((unsigned char*)bp.buf)[0] != '0') { void * buf = (void*)bp.buf; unsigned char val[4]; - unsigned char * s = PyString_AsString(obj); + char * s = PyString_AsString(obj); memcpy(val, bp.buf, 4); PyBuffer_Release(&bp); if (PyObject_GetBuffer(obj, &bp, PyBUF_SIMPLE) == -1) From pypy.commits at gmail.com Sun Apr 15 02:06:32 2018 From: pypy.commits at gmail.com (mattip) Date: Sat, 14 Apr 2018 23:06:32 -0700 (PDT) Subject: [pypy-commit] buildbot default: depricate building on tannit32 Message-ID: <5ad2ebe8.41621c0a.2c9ec.eb32@mx.google.com> Author: Matti Picus Branch: Changeset: r1063:0bfbe41f7868 Date: 2018-04-15 09:05 +0300 http://bitbucket.org/pypy/buildbot/changeset/0bfbe41f7868/ Log: depricate building on tannit32 diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -14,7 +14,7 @@ # buildbot supports SlaveLocks, which can be used to limit the amout of builds # to be run on each slave in parallel. However, they assume that each # buildslave is on a differen physical machine, which is not the case for -# tannit32 and tannit64. As a result, we have to use a global lock, and +# bencher4 and bencher4_32. As a result, we have to use a global lock, and # manually tell each builder that uses tannit to acquire it. # # Look at the various "locks" session in master.py/BuildmasterConfig. For diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -290,12 +290,12 @@ # All the other linux tests run on bencher4.soft-dev.org. Nightly("nightly-0-00", [ # linux tests - LINUX32OWN, # on tannit32, uses all cores + LINUX32OWN, # on bencher4_32, uses all cores LINUX64OWN, # on bencher4, uses all cores WIN32OWN, # on allegro_win32, SalsaSalsa - JITLINUX32, # on tannit32, uses 1 core + JITLINUX32, # on bencher4_32, uses 1 core JITLINUX64, # on bencher4, uses 1 core - #APPLVLLINUX32, # on tannit32, uses 1 core + #APPLVLLINUX32, # on bencher4_32, uses 1 core #APPLVLLINUX64, # on bencher4, uses 1 core # other platforms #MACOSX32, # on minime @@ -311,7 +311,7 @@ ), Nightly("nightly-0-01", [ - LINUX32RPYTHON, # on tannit32, uses all cores + LINUX32RPYTHON, # on bencher4_32, uses all cores LINUX64RPYTHON, # on bencher4, uses all cores WIN32RPYTHON, # on allegro_win32, SalsaSalsa LINUX_S390XRPYTHON, @@ -342,8 +342,8 @@ # branch="py3.5", hour=3, minute=0), Nightly("nightly-3-00-py3.5", [ - LINUX32OWN, # on tannit32, uses all cores - JITLINUX32, # on tannit32, uses 1 core + LINUX32OWN, # on bencher4_32, uses all cores + JITLINUX32, # on bencher4_32, uses 1 core LINUX64OWN, # on bencher4, uses all cores JITLINUX64, # on bencher4, uses 1 core JITMACOSX64, # on xerxes @@ -418,14 +418,14 @@ 'builders': [ {"name": LINUX32OWN, - "slavenames": ["tannit32", "bencher4_32"], + "slavenames": ["bencher4_32"], "builddir": LINUX32OWN, "factory": pypyOwnTestFactory, "category": 'linux32', "locks": [TannitCPU.access('counting')], }, {"name": LINUX32RPYTHON, - "slavenames": ["tannit32", "bencher4_32"], + "slavenames": ["bencher4_32"], "builddir": LINUX32RPYTHON, "factory": pypyRPythonTestFactory, "category": 'linux32', @@ -449,7 +449,7 @@ }, {"name": APPLVLLINUX32, #"slavenames": ["allegro32"], - "slavenames": ["tannit32", "bencher4_32"], + "slavenames": ["bencher4_32"], "builddir": APPLVLLINUX32, "factory": pypyTranslatedAppLevelTestFactory, 'category': 'linux32', @@ -464,7 +464,7 @@ "locks": [Bencher4Lock.access('counting')], }, {"name": LIBPYTHON_LINUX32, - "slavenames": ["tannit32", "bencher4_32"], + "slavenames": ["bencher4_32"], #"slavenames": ["allegro32"], "builddir": LIBPYTHON_LINUX32, "factory": pypyTranslatedLibPythonTestFactory, @@ -481,7 +481,7 @@ }, {"name" : JITLINUX32, #"slavenames": ["allegro32"], - "slavenames": ["tannit32", "bencher4_32"], + "slavenames": ["bencher4_32"], 'builddir' : JITLINUX32, 'factory' : pypyJITTranslatedTestFactory, 'category' : 'linux32', @@ -577,7 +577,6 @@ }, {'name': NUMPY_64, 'slavenames': ["bencher4"], - #'slavenames': ["tannit64"], 'builddir': NUMPY_64, 'factory': pypyNumpyCompatability, 'category': 'numpy', From pypy.commits at gmail.com Sun Apr 15 02:00:49 2018 From: pypy.commits at gmail.com (mattip) Date: Sat, 14 Apr 2018 23:00:49 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: merge default into py3.5 Message-ID: <5ad2ea91.2f80df0a.3a7fd.0f44@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r94333:599b7470b778 Date: 2018-04-15 08:48 +0300 http://bitbucket.org/pypy/pypy/changeset/599b7470b778/ Log: merge default into py3.5 diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -6,3 +6,7 @@ .. startrev: f22145c34985 +.. branch: issue2752 + +Fix a rare GC bug that was introduced more than one year ago, but was +not diagnosed before issue #2752. diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -731,14 +731,16 @@ def move_out_of_nursery(self, obj): # called twice, it should return the same shadow object, - # and not creating another shadow object - if self.header(obj).tid & GCFLAG_HAS_SHADOW: - shadow = self.nursery_objects_shadows.get(obj) - ll_assert(shadow != llmemory.NULL, - "GCFLAG_HAS_SHADOW but no shadow found") - return shadow - - return self._allocate_shadow(obj, copy=True) + # and not creating another shadow object. As a safety feature, + # when called on a non-nursery object, do nothing. + if not self.is_in_nursery(obj): + return obj + shadow = self._find_shadow(obj) + if (self.header(obj).tid & GCFLAG_SHADOW_INITIALIZED) == 0: + self.header(obj).tid |= GCFLAG_SHADOW_INITIALIZED + totalsize = self.get_size(obj) + llmemory.raw_memcopy(obj, shadow, totalsize) + return shadow def collect(self, gen=2): """Do a minor (gen=0), start a major (gen=1), or do a full @@ -2074,13 +2076,12 @@ ll_assert(newobj != llmemory.NULL, "GCFLAG_HAS_SHADOW but no shadow found") newhdr = newobj - size_gc_header # - # Remove the flag GCFLAG_HAS_SHADOW, so that it doesn't get - # copied to the shadow itself. - self.header(obj).tid &= ~GCFLAG_HAS_SHADOW + # The flags GCFLAG_HAS_SHADOW and GCFLAG_SHADOW_INITIALIZED + # have no meaning in non-nursery objects. We don't need to + # remove them explicitly here before doing the copy. tid = self.header(obj).tid if (tid & GCFLAG_SHADOW_INITIALIZED) != 0: copy = False - self.header(obj).tid &= ~GCFLAG_SHADOW_INITIALIZED # totalsize = size_gc_header + self.get_size(obj) self.nursery_surviving_size += raw_malloc_usage(totalsize) @@ -2644,8 +2645,7 @@ # ---------- # id() and identityhash() support - @specialize.arg(2) - def _allocate_shadow(self, obj, copy=False): + def _allocate_shadow(self, obj): size_gc_header = self.gcheaderbuilder.size_gc_header size = self.get_size(obj) shadowhdr = self._malloc_out_of_nursery(size_gc_header + @@ -2667,12 +2667,6 @@ # self.header(obj).tid |= GCFLAG_HAS_SHADOW self.nursery_objects_shadows.setitem(obj, shadow) - - if copy: - self.header(obj).tid |= GCFLAG_SHADOW_INITIALIZED - totalsize = size_gc_header + self.get_size(obj) - llmemory.raw_memcopy(obj - size_gc_header, shadow, totalsize) - return shadow def _find_shadow(self, obj): diff --git a/rpython/memory/gc/test/test_direct.py b/rpython/memory/gc/test/test_direct.py --- a/rpython/memory/gc/test/test_direct.py +++ b/rpython/memory/gc/test/test_direct.py @@ -671,6 +671,25 @@ self.gc.debug_gc_step_until(incminimark.STATE_SCANNING) assert self.stackroots[1].x == 13 + def test_move_out_of_nursery(self): + obj0 = self.malloc(S) + obj0.x = 123 + adr1 = self.gc.move_out_of_nursery(llmemory.cast_ptr_to_adr(obj0)) + obj1 = llmemory.cast_adr_to_ptr(adr1, lltype.Ptr(S)) + assert obj1.x == 123 + # + import pytest + obj2 = self.malloc(S) + obj2.x = 456 + adr3 = self.gc._find_shadow(llmemory.cast_ptr_to_adr(obj2)) + obj3 = llmemory.cast_adr_to_ptr(adr3, lltype.Ptr(S)) + with pytest.raises(lltype.UninitializedMemoryAccess): + obj3.x # the shadow is not populated yet + adr4 = self.gc.move_out_of_nursery(llmemory.cast_ptr_to_adr(obj2)) + assert adr4 == adr3 + assert obj3.x == 456 # it is populated now + + class TestIncrementalMiniMarkGCFull(DirectGCTest): from rpython.memory.gc.incminimark import IncrementalMiniMarkGC as GCClass def test_malloc_fixedsize_no_cleanup(self): From pypy.commits at gmail.com Sun Apr 15 10:33:02 2018 From: pypy.commits at gmail.com (rlamy) Date: Sun, 15 Apr 2018 07:33:02 -0700 (PDT) Subject: [pypy-commit] pypy default: Delete obsolete code Message-ID: <5ad3629e.05f71c0a.e40fc.db7f@mx.google.com> Author: Ronan Lamy Branch: Changeset: r94335:5c8076e87315 Date: 2018-04-15 15:32 +0100 http://bitbucket.org/pypy/pypy/changeset/5c8076e87315/ Log: Delete obsolete code diff --git a/pypy/tool/pytest/genreportdata.py b/pypy/tool/pytest/genreportdata.py deleted file mode 100755 --- a/pypy/tool/pytest/genreportdata.py +++ /dev/null @@ -1,30 +0,0 @@ -#! /usr/bin/env python -import py -import sys - -mydir = py.path.local(__file__).dirpath().realpath() -from pypy.tool.pytest import htmlreport -from pypy.tool.pytest import confpath - -if __name__ == '__main__': - if len(sys.argv) > 1: - testresultdir = py.path.local(sys.argv[1]) - assert testresultdir.check(dir=1) - else: - testresultdir = confpath.testresultdir - assert testresultdir.check(dir=1) - try: - resultwc = py.path.svnwc(testresultdir) - print "updating", resultwc - resultwc.update() - except (KeyboardInterrupt, RuntimeError): - raise - except Exception as e: #py.process.ExecutionFailed,e: - print >> sys.stderr, "Warning: ",e #Subversion update failed" - - print "traversing", mydir - rep = htmlreport.HtmlReport(testresultdir) - rep.parselatest() - - print "making html files" - rep.makeindex(testresultdir.join('index.html')) diff --git a/pypy/tool/pytest/htmlreport.py b/pypy/tool/pytest/htmlreport.py deleted file mode 100644 --- a/pypy/tool/pytest/htmlreport.py +++ /dev/null @@ -1,239 +0,0 @@ -#! /usr/bin/env python - -""" -the html test reporter - -""" -import sys, os, re -import pprint -import py -from pypy.tool.pytest import result -from pypy.tool.pytest.overview import ResultCache - -# -# various interesting path objects -# - -html = py.xml.html -NBSP = py.xml.raw(" ") - -class HtmlReport(object): - def __init__(self, resultdir): - self.resultcache = ResultCache(resultdir) - - def parselatest(self): - self.resultcache.parselatest() - - # - # rendering - # - - def render_latest_table(self, results): - table = html.table( - [html.th(x, align='left') - for x in ("failure", "filename", "revision", - "user", "platform", "elapsed", - "options", "last error line" - )], - ) - r = results[:] - def f(x, y): - xnum = x.isok() and 1 or (x.istimeout() and 2 or 3) - ynum = y.isok() and 1 or (y.istimeout() and 2 or 3) - res = -cmp(xnum, ynum) - if res == 0: - return cmp(x['execution-time'], y['execution-time']) - return res - r.sort(f) - for result in r: - table.append(self.render_result_row(result)) - return table - - def render_result_row(self, result): - dp = py.path.local(result['fspath']) - - options = " ".join([x for x in result.get('options', []) if x!= 'core']) - if not options: - options = NBSP - - failureratio = 100 * (1.0 - result.ratio_of_passed()) - self.data[result.testname] = failureratio - return html.tr( - html.td("%.2f%%" % failureratio, - style = "background-color: %s" % (getresultcolor(result),)), - html.td(self.render_test_references(result)), - html.td(result['pypy-revision']), - html.td(result['userhost'][:15]), - html.td(result['platform']), - html.td("%.2fs" % result['execution-time']), - html.td(options), - html.td(result.repr_short_error() or NBSP) - ) - - def getrelpath(self, p): - return p.relto(self.indexpath.dirpath()) - - def render_test_references(self, result): - dest = self.make_single_test_result(result) - #XXX: ask hg for differences between test and vendor branch - modified = result.ismodifiedtest() and " [mod]" or "" - return html.div(html.a(result.path.purebasename + modified, - href=self.getrelpath(dest)), - style="background-color: transparent") - - def make_single_test_result(self, result): - cache = self.indexpath.dirpath('.cache', result['userhost'][:15]) - cache.ensure(dir=1) - dest = cache.join(result.path.basename).new(ext='.html') - doc = ViewResult(result) - doc.writetopath(dest) - return dest - - def getcorelists(self): - def iscore(result): - return 'core' in result.get('options', []) - coretests = [] - noncoretests = [] - for name in self.resultcache.getnames(): - result = self.resultcache.getlatestrelevant(name) - if iscore(result): - coretests.append(result) - else: - noncoretests.append(result) - return coretests, noncoretests - - # generate html files - # - def makeindex(self, indexpath, detail="PyPy - latest"): - self.indexpath = indexpath - self.data = {} - doc = Document(title='pypy test results') - body = doc.body - coretests, noncoretests = self.getcorelists() - body.append(html.h2("%s compliance test results - " - "core tests" % detail)) - - body.append(self.render_test_summary('core', coretests)) - body.append(self.render_latest_table(coretests)) - body.append( - html.h2("%s compliance test results - non-core tests" % detail)) - body.append(self.render_test_summary('noncore', noncoretests)) - body.append(self.render_latest_table(noncoretests)) - doc.writetopath(indexpath) - datapath = indexpath.dirpath().join('data') - d = datapath.open('w') - print >>d, "data = ", - pprint.pprint(self.data, stream=d) - d.close() - self.data = None - - def render_test_summary(self, tag, tests): - ok = len([x for x in tests if x.isok()]) - err = len([x for x in tests if x.iserror()]) - to = len([x for x in tests if x.istimeout()]) - numtests = ok + err + to - assert numtests == len(tests) - assert numtests - - t = html.table() - sum100 = numtests / 100.0 - def row(*args): - return html.tr(*[html.td(arg) for arg in args]) - - sum_passed = sum([x.ratio_of_passed() for x in tests]) - compliancy = sum_passed/sum100 - self.data['%s-compliancy' % tag] = compliancy - t.append(row(html.b("tests compliancy"), - html.b("%.2f%%" % (compliancy,)))) - - passed = ok/sum100 - self.data['%s-passed' % tag] = passed - t.append(row("testmodules passed completely", "%.2f%%" % passed)) - failed = err/sum100 - self.data['%s-failed' % tag] = failed - t.append(row("testmodules (partially) failed", "%.2f%%" % failed)) - timedout = to/sum100 - self.data['%s-timedout' % tag] = timedout - t.append(row("testmodules timeout", "%.2f%%" % timedout)) - return t - -class Document(object): - def __init__(self, title=None): - self.body = html.body() - self.head = html.head() - self.doc = html.html(self.head, self.body) - if title is not None: - self.head.append( - html.meta(name="title", content=title)) - self.head.append( - html.link(rel="Stylesheet", type="text/css", href="/pypy/default.css")) - - def writetopath(self, p): - assert p.ext == '.html' - self.head.append( - html.meta(name="Content-Type", content="text/html;charset=UTF-8") - ) - s = self.doc.unicode().encode('utf-8') - p.write(s) - -def getresultcolor(result): - if result.isok(): - color = "#00ee00" - elif result.iserror(): - color = "#ee0000" - elif result.istimeout: - color = "#0000ee" - else: - color = "#444444" - return color - -class ViewResult(Document): - def __init__(self, result): - title = "%s testresult" % (result.path.purebasename,) - super(ViewResult, self).__init__(title=title) - color = getresultcolor(result) - self.body.append(html.h2(title, - style="background-color: %s" % color)) - self.body.append(self.render_meta_info(result)) - - for name in ('reportdiff', 'stdout', 'stderr'): - try: - text = result.getnamedtext(name) - except KeyError: - continue - self.body.append(html.h3(name)) - self.body.append(html.pre(text)) - - def render_meta_info(self, result): - t = html.table() - items = result.items() - items.sort() - for name, value in items: - if name.lower() == name: - t.append(html.tr( - html.td(name), html.td(value))) - return t - -class TestOfHtmlReportClass: - def setup_class(cls): - py.test.skip('needs move to own test file') - cls.testresultdir = confpath.testresultdir - cls.rep = rep = HtmlReport() - rep.parse_all(cls.testresultdir) - - def test_pickling(self): - # test pickling of report - tempdir = py.test.ensuretemp('reportpickle') - picklepath = tempdir.join('report.pickle') - picklepath.dump(self.rep) - x = picklepath.load() - assert len(x.results) == len(self.rep.results) - - def test_render_latest(self): - t = self.rep.render_latest_table(self.rep.results) - assert unicode(t) - -mydir = py.path.local(__file__).dirpath() - -def getpicklepath(): - return mydir.join('.htmlreport.pickle') diff --git a/pypy/tool/pytest/overview.py b/pypy/tool/pytest/overview.py deleted file mode 100644 --- a/pypy/tool/pytest/overview.py +++ /dev/null @@ -1,56 +0,0 @@ -from pypy.tool.pytest import result -import sys - -class ResultCache: - def __init__(self, resultdir): - self.resultdir = resultdir - self.name2result = {} - - def parselatest(self): - def filefilter(p): - return p.check(fnmatch='test_*.txt', file=1) - def rec(p): - return p.check(dotfile=0) - for x in self.resultdir.visit(filefilter, rec): - self.parse_one(x) - - def parse_one(self, resultpath): - try: - res = result.ResultFromMime(resultpath) - ver = res['testreport-version'] - if ver != "1.1" and ver != "1.1.1": - raise TypeError - except TypeError: # xxx - print >>sys.stderr, "could not parse %s" % resultpath - return - name = res.testname - print name - self.name2result.setdefault(name, []).append(res) - return res - - def getnames(self): - return self.name2result.keys() - - def getlatest(self, name, timeout=0, error=0, ok=0): - l = [] - resultlist = self.name2result[name] - maxrev = 0 - maxresult = None - for res in resultlist: - resrev = res['pypy-revision'] - if resrev == 'unknown': - continue - if resrev <= maxrev: - continue - if timeout or error or ok: - if not (timeout and res.istimeout() or - error and res.iserror() or - ok and res.isok()): - continue - maxrev = resrev - maxresult = res - return maxresult - - def getlatestrelevant(self, name): - # get the latest revision that did not time out. - return self.getlatest(name, error=1, ok=1) or self.getlatest(name) diff --git a/pypy/tool/pytest/result.py b/pypy/tool/pytest/result.py deleted file mode 100644 --- a/pypy/tool/pytest/result.py +++ /dev/null @@ -1,215 +0,0 @@ -import sys -import py -import re - -class Result(object): - def __init__(self, init=True): - self._headers = {} - self._blocks = {} - self._blocknames = [] - if init: - stdinit(self) - - def __setitem__(self, name, value): - self._headers[name.lower()] = value - - def __getitem__(self, name): - return self._headers[name.lower()] - - def get(self, name, default): - return self._headers.get(name, default) - - def __delitem__(self, name): - del self._headers[name.lower()] - - def items(self): - return self._headers.items() - - def addnamedtext(self, name, text): - assert isinstance(text, basestring) - assert isinstance(name, str) - self._blocknames.append(name) - self._blocks[name] = text - - def getnamedtext(self, name): - return self._blocks[name] - - def repr_short_error(self): - if not self.isok(): - if 'reportdiff' in self._blocks: - return "output comparison failed, see reportdiff" - else: - text = self.getnamedtext('stderr') - lines = text.strip().split('\n') - if lines: - return lines[-1] - - def repr_mimemessage(self): - from email.MIMEMultipart import MIMEMultipart - from email.MIMEText import MIMEText - - outer = MIMEMultipart() - items = self._headers.items() - items.sort() - reprs = {} - for name, value in items: - assert ':' not in name - chars = map(ord, name) - assert min(chars) >= 33 and max(chars) <= 126 - outer[name] = str(value) - if not isinstance(value, str): - typename = type(value).__name__ - assert typename in vars(py.std.__builtin__) - reprs[name] = typename - - outer['_reprs'] = repr(reprs) - - for name in self._blocknames: - text = self._blocks[name] - m = MIMEText(text) - m.add_header('Content-Disposition', 'attachment', filename=name) - outer.attach(m) - return outer - - def grep_nr(self,text,section='stdout'): - stdout = self._blocks[section] - find = re.search('%s(?P\d+)'%text,stdout) - if find: - return float(find.group('nr')) - return 0. - - def ratio_of_passed(self): - if self.isok(): - return 1. - elif self.istimeout(): - return 0. - else: - nr = self.grep_nr('Ran ') - if nr > 0: - return (nr - (self.grep_nr('errors=') + self.grep_nr('failures=')))/nr - else: - passed = self.grep_nr('TestFailed: ',section='stderr') - run = self.grep_nr('TestFailed: \d+/',section='stderr') - if run > 0: - return passed/run - else: - run = self.grep_nr('TestFailed: \d+ of ',section='stderr') - if run > 0 : - return (run-passed)/run - else: - return 0.0 - - def isok(self): - return self['outcome'].lower() == 'ok' - - def iserror(self): - return self['outcome'].lower()[:3] == 'err' or self['outcome'].lower() == 'fail' - - def istimeout(self): - return self['outcome'].lower() == 't/o' - -# XXX backward compatibility -def sanitize(msg, path): - if 'exit-status' in msg.keys(): - return msg - f = open(str(path), 'r') - msg = f.read() - f.close() - for broken in ('exit status', 'cpu model', 'cpu mhz'): - valid = broken.replace(' ','-') - invalid = msg.find(broken+':') - msg = (msg[:invalid] + valid + - msg[invalid+len(valid):]) - from email import message_from_string - msg = message_from_string(msg) - return msg - -def sanitize_reprs(reprs): - if 'exit status' in reprs: - reprs['exit-status'] = reprs.pop('exit status') - -class ResultFromMime(Result): - def __init__(self, path): - super(ResultFromMime, self).__init__(init=False) - f = open(str(path), 'r') - from email import message_from_file - msg = message_from_file(f) - f.close() - msg = sanitize(msg, path) - # XXX security wise evil (keep in mind once we accept reporsts - # from anonymous - #print msg['_reprs'] - self._reprs = eval(msg['_reprs']) - del msg['_reprs'] - sanitize_reprs(self._reprs) - for name, value in msg.items(): - if name in self._reprs: - value = eval(value) # XXX security - self._headers[name] = value - self.fspath = self['fspath'] - if self['platform'] == 'win32' and '\\' in self.fspath: - self.testname = self.fspath.split('\\')[-1] - else: - self.testname = self.fspath.split('/')[-1] - #if sys.platform != 'win32' and '\\' in self.fspath: - # self.fspath = py.path.local(self['fspath'].replace('\\' - self.path = path - - payload = msg.get_payload() - if payload: - for submsg in payload: - assert submsg.get_content_type() == 'text/plain' - fn = submsg.get_filename() - assert fn - # XXX we need to deal better with encodings to - # begin with - content = submsg.get_payload() - for candidate in 'utf8', 'latin1': - try: - text = unicode(content, candidate) - except UnicodeDecodeError: - continue - else: - unicode(content, candidate) - self.addnamedtext(fn, text) - - def ismodifiedtest(self): - # XXX we need proper cross-platform paths! - return 'modified' in self.fspath - - def __repr__(self): - return '<%s (%s) %r rev=%s>' %(self.__class__.__name__, - self['outcome'], - self.fspath, - self['pypy-revision']) - -def stdinit(result): - import getpass - import socket - try: - username = getpass.getuser() - except: - username = 'unknown' - userhost = '%s@%s' % (username, socket.gethostname()) - result['testreport-version'] = "1.1.1" - result['userhost'] = userhost - result['platform'] = sys.platform - result['python-version-info'] = sys.version_info - info = try_getcpuinfo() - if info is not None: - result['cpu-model'] = info.get('model name', "unknown") - result['cpu-mhz'] = info.get('cpu mhz', 'unknown') -# -# -# -def try_getcpuinfo(): - if sys.platform.startswith('linux'): - cpuinfopath = py.path.local('/proc/cpuinfo') - if cpuinfopath.check(file=1): - d = {} - for line in cpuinfopath.readlines(): - if line.strip(): - name, value = line.split(':', 1) - name = name.strip().lower() - d[name] = value.strip() - return d diff --git a/pypy/tool/pytest/test/test_new_count.py b/pypy/tool/pytest/test/test_new_count.py deleted file mode 100644 --- a/pypy/tool/pytest/test/test_new_count.py +++ /dev/null @@ -1,32 +0,0 @@ - -import py -#from pypy.tool.pytest.confpath import testresultdir -from pypy.tool.pytest.result import ResultFromMime -testpath = py.path.local(__file__).dirpath('data') - -class TestResultCache: - - def test_timeout(self): - test = ResultFromMime(testpath.join('test___all__.txt')) - assert test.ratio_of_passed() == 0. - - def test_passed(self): - test = ResultFromMime(testpath.join('test_sys.txt')) - assert test.ratio_of_passed() == 1. - - def test_unittest_partial(self): - test = ResultFromMime(testpath.join('test_compile.txt')) - assert test.ratio_of_passed() == 10./15 - - def test_doctest_of(self): - test = ResultFromMime(testpath.join('test_generators.txt')) - assert test.ratio_of_passed() == 133./154 - - def test_doctest_slash(self): - test = ResultFromMime(testpath.join('test_descr.txt')) - assert test.ratio_of_passed() == 65./92 - - def test_fail(self): - test = ResultFromMime(testpath.join('test_global.txt')) - assert test.ratio_of_passed() == 0. - diff --git a/pypy/tool/pytest/test/test_overview.py b/pypy/tool/pytest/test/test_overview.py deleted file mode 100644 --- a/pypy/tool/pytest/test/test_overview.py +++ /dev/null @@ -1,23 +0,0 @@ - -import py -from pypy.tool.pytest.confpath import testresultdir -from pypy.tool.pytest.overview import ResultCache - -class TestResultCache: - def setup_class(cls): - if not testresultdir.check(dir=1): - py.test.skip("testresult directory not checked out") - cls.rc = ResultCache(testresultdir) - cls.rc.parselatest() - - def test_getlatest_all(self): - for type in 'error', 'timeout', 'ok': - for name in self.rc.getnames(): - result = self.rc.getlatest(name, **{type:1}) - if result: - meth = getattr(result, 'is'+type) - assert meth() - - #def test_getlatest_datetime(self): - # result = self.rc.getlatest('test_datetime', ok=1) - # assert result From pypy.commits at gmail.com Sun Apr 15 10:46:14 2018 From: pypy.commits at gmail.com (rlamy) Date: Sun, 15 Apr 2018 07:46:14 -0700 (PDT) Subject: [pypy-commit] pypy default: Remove data used by deleted tests as well Message-ID: <5ad365b6.ce851c0a.df2f8.abcf@mx.google.com> Author: Ronan Lamy Branch: Changeset: r94336:6e679a6792a4 Date: 2018-04-15 15:45 +0100 http://bitbucket.org/pypy/pypy/changeset/6e679a6792a4/ Log: Remove data used by deleted tests as well diff --git a/pypy/tool/pytest/test/data/test___all__.txt b/pypy/tool/pytest/test/data/test___all__.txt deleted file mode 100644 --- a/pypy/tool/pytest/test/data/test___all__.txt +++ /dev/null @@ -1,94 +0,0 @@ -Content-Type: multipart/mixed; boundary="===============0790678169==" -MIME-Version: 1.0 -execution-time: 1445.14346004 -exit status: 1 -fspath: /Users/anderslehmann/pypy/lib-python/2.4.1/test/test___all__.py -options: ['core', '_sre'] -outcome: T/O -platform: darwin -pypy-revision: 16114 -python-version-info: (2, 4, 1, 'final', 0) -startdate: Wed Aug 17 23:51:59 2005 -testreport-version: 1.1 -timeout: 1369.0 -userhost: anderslehmann at anders-lehmanns-15-powerbook-g4.local -_reprs: {'execution-time': 'float', 'python-version-info': 'tuple', - 'options': 'list', 'timeout': 'float', 'pypy-revision': 'int', - 'exit status': 'int'} - ---===============0790678169== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stdout" - -test_all (__main__.AllTest) ... ERROR - -====================================================================== -ERROR: test_all (__main__.AllTest) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test___all__.py", line 163, in test_all - self.check_all("tty") - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test___all__.py", line 26, in check_all - "%s has no __all__ attribute" % modname) - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_support.py", line 208, in verify - raise TestFailed(reason) -TestFailed: tty has no __all__ attribute - ----------------------------------------------------------------------- - ---===============0790678169== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stderr" - -faking -Loading grammar /Users/anderslehmann/pypy/pypy/interpreter/pyparser/data/Grammar2.4 -faking -faking -faking -fake-wrapping interp file ', mode 'w' at 0x12068> -fake-wrapping interp file ', mode 'w' at 0x120b0> -fake-wrapping interp file ', mode 'r' at 0x12020> -faking -faking -faking -faking -==========================timedout========================== -Traceback (application-level): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test___all__.py", line 192 in - test_main() - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test___all__.py", line 189 in test_main - test_support.run_unittest(AllTest) - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_support.py", line 290 in run_unittest - run_suite(suite, testclass) - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_support.py", line 262 in run_suite - result = runner.run(suite) -Traceback (application-level): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/atexit.py", line 29 in _run_exitfuncs - print >> sys.stderr, "Error in atexit._run_exitfuncs:" -KeyboardInterrupt -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/pypy/tool/alarm.py", line 43, in ? - execfile(_main_with_alarm(finished)) - File "/Users/anderslehmann/pypy/pypy/bin/py.py", line 206, in ? - sys.exit(main_(sys.argv)) - File "/Users/anderslehmann/pypy/pypy/bin/py.py", line 115, in main_ - if not main.run_toplevel(space, doit, verbose=Options.verbose): - File "/Users/anderslehmann/pypy/pypy/interpreter/main.py", line 150, in run_toplevel - operationerr.print_application_traceback(space) - File "/Users/anderslehmann/pypy/pypy/interpreter/error.py", line 83, in print_application_traceback - self.print_app_tb_only(file) - File "/Users/anderslehmann/pypy/pypy/interpreter/error.py", line 104, in print_app_tb_only - l = linecache.getline(fname, lineno) - File "/Library/Frameworks/Python.framework/Versions/2.4/lib/python2.4/linecache.py", line 14, in getline - lines = getlines(filename) - File "/Library/Frameworks/Python.framework/Versions/2.4/lib/python2.4/linecache.py", line 40, in getlines - return updatecache(filename) - File "/Library/Frameworks/Python.framework/Versions/2.4/lib/python2.4/linecache.py", line 101, in updatecache - lines = fp.readlines() -KeyboardInterrupt - ---===============0790678169==-- \ No newline at end of file diff --git a/pypy/tool/pytest/test/data/test_compile.txt b/pypy/tool/pytest/test/data/test_compile.txt deleted file mode 100644 --- a/pypy/tool/pytest/test/data/test_compile.txt +++ /dev/null @@ -1,111 +0,0 @@ -Content-Type: multipart/mixed; boundary="===============2137793924==" -MIME-Version: 1.0 -execution-time: 34.8464071751 -exit status: 1 -fspath: /Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py -options: ['core', '_sre'] -outcome: ERR -platform: darwin -pypy-revision: 16114 -python-version-info: (2, 4, 1, 'final', 0) -startdate: Thu Aug 18 03:08:18 2005 -testreport-version: 1.1 -timeout: 1521.0 -userhost: anderslehmann at anders-lehmanns-15-powerbook-g4.local -_reprs: {'execution-time': 'float', 'python-version-info': 'tuple', - 'options': 'list', 'timeout': 'float', 'pypy-revision': 'int', - 'exit status': 'int'} - ---===============2137793924== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stdout" - -test_argument_handling (__main__.TestSpecifics) ... FAIL -test_argument_order (__main__.TestSpecifics) ... FAIL -test_complex_args (__main__.TestSpecifics) ... ok -test_debug_assignment (__main__.TestSpecifics) ... FAIL -test_duplicate_global_local (__main__.TestSpecifics) ... ok -test_exec_with_general_mapping_for_locals (__main__.TestSpecifics) ... ok -test_float_literals (__main__.TestSpecifics) ... ok -test_for_distinct_code_objects (__main__.TestSpecifics) ... ok -test_import (__main__.TestSpecifics) ... FAIL -test_indentation (__main__.TestSpecifics) ... ok -test_literals_with_leading_zeroes (__main__.TestSpecifics) ... ok -test_none_assignment (__main__.TestSpecifics) ... FAIL -test_sequence_unpacking_error (__main__.TestSpecifics) ... ok -test_syntax_error (__main__.TestSpecifics) ... ok -test_unary_minus (__main__.TestSpecifics) ... ok - -====================================================================== -FAIL: test_argument_handling (__main__.TestSpecifics) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 18, in test_argument_handling - self.assertRaises(SyntaxError, eval, 'lambda a,a:0') -AssertionError: SyntaxError not raised - -====================================================================== -FAIL: test_argument_order (__main__.TestSpecifics) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 127, in test_argument_order - self.fail("non-default args after default") -AssertionError: non-default args after default - -====================================================================== -FAIL: test_debug_assignment (__main__.TestSpecifics) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 10, in test_debug_assignment - self.assertRaises(SyntaxError, compile, '__debug__ = 1', '?', 'single') -AssertionError: SyntaxError not raised - -====================================================================== -FAIL: test_import (__main__.TestSpecifics) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 253, in test_import - self.assertRaises(SyntaxError, compile, stmt, 'tmp', 'exec') -AssertionError: SyntaxError not raised - -====================================================================== -FAIL: test_none_assignment (__main__.TestSpecifics) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 211, in test_none_assignment - self.assertRaises(SyntaxError, compile, stmt, 'tmp', 'single') -AssertionError: SyntaxError not raised - ----------------------------------------------------------------------- -Ran 15 tests in 14.363s - -FAILED (failures=5) - ---===============2137793924== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stderr" - -faking -Loading grammar /Users/anderslehmann/pypy/pypy/interpreter/pyparser/data/Grammar2.4 -faking -faking -faking -fake-wrapping interp file ', mode 'w' at 0x12068> -fake-wrapping interp file ', mode 'w' at 0x120b0> -fake-wrapping interp file ', mode 'r' at 0x12020> -Traceback (application-level): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 268 in - test_main() - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 265 in test_main - test_support.run_unittest(TestSpecifics) - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_support.py", line 290 in run_unittest - run_suite(suite, testclass) - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_support.py", line 274 in run_suite - raise TestFailed(msg) -TestFailed: errors occurred in __main__.TestSpecifics - ---===============2137793924==-- \ No newline at end of file diff --git a/pypy/tool/pytest/test/data/test_descr.txt b/pypy/tool/pytest/test/data/test_descr.txt deleted file mode 100644 --- a/pypy/tool/pytest/test/data/test_descr.txt +++ /dev/null @@ -1,233 +0,0 @@ -Content-Type: multipart/mixed; boundary="===============1265023865==" -MIME-Version: 1.0 -execution-time: 4098.8407588 -exit status: 1 -fspath: /Users/anderslehmann/pypy/lib-python/modified-2.4.1/test/test_descr.py -options: ['oldstyle', 'core'] -outcome: ERR -platform: darwin -pypy-revision: 16388 -python-version-info: (2, 4, 1, 'final', 0) -startdate: Wed Aug 24 16:54:12 2005 -testreport-version: 1.1 -timeout: 10000.0 -userhost: anderslehmann at anders-lehmanns-15-powerbook-g4.local -_reprs: {'execution-time': 'float', 'python-version-info': 'tuple', - 'options': 'list', 'timeout': 'float', 'pypy-revision': 'int', - 'exit status': 'int'} - ---===============1265023865== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stdout" - -**************************************** ---> weakref_segfault FAILURE(0/92) _weakref -**************************************** ---> do_this_first OK(1/92) -**************************************** ---> class_docstrings OK(2/92) -**************************************** ---> lists FAILURE(2/92) type object 'list' has no attribute '__getslice__' -**************************************** ---> dicts FAILURE(2/92) type object 'dict' has no attribute '__cmp__' -**************************************** ---> dict_constructor OK(3/92) -**************************************** ---> test_dir OK(4/92) -**************************************** ---> ints OK(5/92) -**************************************** ---> longs OK(6/92) -**************************************** ---> floats OK(7/92) -**************************************** ---> complexes OK(8/92) -**************************************** ---> spamlists FAILURE(8/92) xxsubtype -**************************************** ---> spamdicts FAILURE(8/92) xxsubtype -**************************************** ---> pydicts OK(9/92) -**************************************** ---> pylists OK(10/92) -**************************************** ---> metaclass OK(11/92) -**************************************** ---> pymods OK(12/92) -**************************************** ---> multi OK(13/92) -**************************************** ---> mro_disagreement FAILURE(13/92) Message 'cycle among base classes: A < B < A', expected 'Cannot create a consistent method resolution\norder (MRO) for bases ' -**************************************** ---> diamond OK(14/92) -**************************************** ---> ex5 OK(15/92) -**************************************** ---> monotonicity OK(16/92) -**************************************** ---> consistency_with_epg OK(17/92) -**************************************** ---> objects OK(18/92) -**************************************** ---> slots FAILURE(18/92) ['foo bar'] slots not caught -**************************************** ---> slotspecials FAILURE(18/92) test failed -**************************************** ---> dynamics OK(19/92) -**************************************** ---> errors FAILURE(19/92) inheritance from CFunction should be illegal -**************************************** ---> classmethods OK(20/92) -**************************************** ---> classmethods_in_c FAILURE(20/92) xxsubtype -**************************************** ---> staticmethods OK(21/92) -**************************************** ---> staticmethods_in_c FAILURE(21/92) xxsubtype -**************************************** ---> classic OK(22/92) -**************************************** ---> compattr OK(23/92) -**************************************** ---> newslot OK(24/92) -**************************************** ---> altmro OK(25/92) -**************************************** ---> overloading OK(26/92) -**************************************** ---> methods FAILURE(26/92) test failed -**************************************** ---> specials FAILURE(26/92) shouldn't allow .__cmp__(u'123', '123') -**************************************** ---> weakrefs FAILURE(26/92) 'module' object has no attribute 'ref' -**************************************** ---> properties FAILURE(26/92) expected TypeError from trying to set readonly '__doc__' attr on a property -**************************************** ---> supers OK(27/92) -**************************************** ---> inherits FAILURE(27/92) test failed -**************************************** ---> keywords FAILURE(27/92) descr__new__() got an unexpected keyword argument 'string' -**************************************** ---> restricted OK(28/92) -**************************************** ---> str_subclass_as_dict_key OK(29/92) -**************************************** ---> classic_comparisons OK(30/92) -**************************************** ---> rich_comparisons OK(31/92) -**************************************** ---> coercions OK(32/92) -**************************************** ---> descrdoc FAILURE(32/92) 'The most base type' == 'True if the file is closed' -**************************************** ---> setclass OK(33/92) -**************************************** ---> setdict OK(34/92) -**************************************** ---> pickles OK(35/92) -**************************************** ---> copies OK(36/92) -**************************************** ---> binopoverride OK(37/92) -**************************************** ---> subclasspropagation OK(38/92) -**************************************** ---> buffer_inherit OK(39/92) -**************************************** ---> str_of_str_subclass OK(40/92) -**************************************** ---> kwdargs OK(41/92) -**************************************** ---> delhook OK(42/92) -**************************************** ---> hashinherit OK(43/92) -**************************************** ---> strops OK(44/92) -**************************************** ---> deepcopyrecursive OK(45/92) -**************************************** ---> modules OK(46/92) -**************************************** ---> dictproxyiterkeys FAILURE(46/92) ['__dict__', '__module__', 'meth'] == ['__dict__', '__doc__', '__module__', '__weakref__', 'meth'] -**************************************** ---> dictproxyitervalues FAILURE(46/92) 3 == 5 -**************************************** ---> dictproxyiteritems FAILURE(46/92) ['__dict__', '__module__', 'meth'] == ['__dict__', '__doc__', '__module__', '__weakref__', 'meth'] -**************************************** ---> pickleslots OK(47/92) -**************************************** ---> funnynew OK(48/92) -**************************************** ---> imulbug OK(49/92) -**************************************** ---> docdescriptor OK(50/92) -**************************************** ---> string_exceptions FAILURE(50/92) string subclass allowed as exception -**************************************** ---> copy_setstate OK(51/92) -**************************************** ---> slices OK(52/92) -**************************************** ---> subtype_resurrection OK(53/92) -**************************************** ---> slottrash OK(54/92) -**************************************** ---> slotmultipleinheritance FAILURE(54/92) type object 'C' has no attribute '__basicsize__' -**************************************** ---> testrmul OK(55/92) -**************************************** ---> testipow OK(56/92) -**************************************** ---> test_mutable_bases FAILURE(56/92) readonly attribute -**************************************** ---> test_mutable_bases_with_failing_mro FAILURE(56/92) readonly attribute -**************************************** ---> test_mutable_bases_catch_mro_conflict OK(57/92) -**************************************** ---> mutable_names OK(58/92) -**************************************** ---> subclass_right_op OK(59/92) -**************************************** ---> dict_type_with_metaclass OK(60/92) -**************************************** ---> meth_class_get FAILURE(60/92) shouldn't have allowed descr.__get__(None, None) -**************************************** ---> isinst_isclass OK(61/92) -**************************************** ---> proxysuper OK(62/92) -**************************************** ---> carloverre OK(63/92) -**************************************** ---> filefault OK(64/92) -**************************************** ---> vicious_descriptor_nonsense OK(65/92) -**************************************** ---> test_init FAILURE(65/92) did not test __init__() for None return - ---===============1265023865== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stderr" - -faking -Loading grammar /Users/anderslehmann/pypy/pypy/interpreter/pyparser/data/Grammar2.4 -faking -faking -faking -fake-wrapping interp file ', mode 'w' at 0x12068> -fake-wrapping interp file ', mode 'w' at 0x120b0> -fake-wrapping interp file ', mode 'r' at 0x12020> -faking -faking -Traceback (application-level): - File "/Users/anderslehmann/pypy/pypy/tool/pytest/regrverbose.py", line 12 in - indirect_test() - File "/Users/anderslehmann/pypy/lib-python/modified-2.4.1/test/test_descr.py", line 4102 in test_main - raise TestFailed, "%d/%d" % (success, n) -TestFailed: 65/92 - ---===============1265023865==-- \ No newline at end of file diff --git a/pypy/tool/pytest/test/data/test_generators.txt b/pypy/tool/pytest/test/data/test_generators.txt deleted file mode 100644 --- a/pypy/tool/pytest/test/data/test_generators.txt +++ /dev/null @@ -1,317 +0,0 @@ -Content-Type: multipart/mixed; boundary="===============1565511160==" -MIME-Version: 1.0 -cpu mhz: unknown -cpu model: unknown -execution-time: 671.678878069 -exit status: 1 -fspath: /home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py -options: ['core'] -outcome: ERR -platform: linux2 -pypy-revision: 16123 -python-version-info: (2, 4, 1, 'final', 0) -startdate: Thu Aug 18 02:08:13 2005 -testreport-version: 1.1 -timeout: 3136.0 -userhost: xoraxax at tick -_reprs: {'execution-time': 'float', 'python-version-info': 'tuple', - 'options': 'list', 'timeout': 'float', 'pypy-revision': 'int', - 'exit status': 'int'} - ---===============1565511160== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stdout" - -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.weakref -Failed example: - import weakref -Exception raised: - Traceback (most recent call last): - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/doctest.py", line 1243, in __run - compileflags, 1) in test.globs - File "", line 1, in - import weakref - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/weakref.py", line 14, in - from _weakref import ( - ImportError: _weakref -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.weakref -Failed example: - wr = weakref.ref(gen) -Exception raised: - Traceback (most recent call last): - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/doctest.py", line 1243, in __run - compileflags, 1) in test.globs - File "", line 1, in - wr = weakref.ref(gen) - NameError: global name 'weakref' is not defined -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.weakref -Failed example: - wr() is gen -Exception raised: - Traceback (most recent call last): - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/doctest.py", line 1243, in __run - compileflags, 1) in test.globs - File "", line 1, in - wr() is gen - NameError: global name 'wr' is not defined -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.weakref -Failed example: - p = weakref.proxy(gen) -Exception raised: - Traceback (most recent call last): - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/doctest.py", line 1243, in __run - compileflags, 1) in test.globs - File "", line 1, in - p = weakref.proxy(gen) - NameError: global name 'weakref' is not defined -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.weakref -Failed example: - wr = weakref.ref(gi) -Exception raised: - Traceback (most recent call last): - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/doctest.py", line 1243, in __run - compileflags, 1) in test.globs - File "", line 1, in - wr = weakref.ref(gi) - NameError: global name 'weakref' is not defined -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.weakref -Failed example: - wr() is gi -Exception raised: - Traceback (most recent call last): - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/doctest.py", line 1243, in __run - compileflags, 1) in test.globs - File "", line 1, in - wr() is gi - NameError: global name 'wr' is not defined -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.weakref -Failed example: - p = weakref.proxy(gi) -Exception raised: - Traceback (most recent call last): - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/doctest.py", line 1243, in __run - compileflags, 1) in test.globs - File "", line 1, in - p = weakref.proxy(gi) - NameError: global name 'weakref' is not defined -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.weakref -Failed example: - list(p) -Exception raised: - Traceback (most recent call last): - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/doctest.py", line 1243, in __run - compileflags, 1) in test.globs - File "", line 1, in - list(p) - NameError: global name 'p' is not defined -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.pep -Failed example: - k.next() -Expected: - Traceback (most recent call last): - File "", line 1, in ? - File "", line 2, in g - File "", line 2, in f - ZeroDivisionError: integer division or modulo by zero -Got: - Traceback (most recent call last): - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/doctest.py", line 1243, in __run - compileflags, 1) in test.globs - File "", line 1, in - k.next() - File "", line 2, in g - yield f() # the zero division exception propagates - File "", line 2, in f - return 1//0 - ZeroDivisionError: integer division by zero -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.syntax -Failed example: - def f(): - return 22 - yield 1 -Expected: - Traceback (most recent call last): - .. - SyntaxError: 'return' with argument inside generator (, line 2) -Got nothing -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.syntax -Failed example: - def f(): - yield 1 - return 22 -Expected: - Traceback (most recent call last): - .. - SyntaxError: 'return' with argument inside generator (, line 3) -Got nothing -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.syntax -Failed example: - def f(): - yield 1 - return None -Expected: - Traceback (most recent call last): - .. - SyntaxError: 'return' with argument inside generator (, line 3) -Got nothing -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.syntax -Failed example: - def f(): - try: - yield 1 - finally: - pass -Expected: - Traceback (most recent call last): - .. - SyntaxError: 'yield' not allowed in a 'try' block with a 'finally' clause (, line 3) -Got nothing -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.syntax -Failed example: - def f(): - try: - try: - 1//0 - except ZeroDivisionError: - yield 666 # bad because *outer* try has finally - except: - pass - finally: - pass -Expected: - Traceback (most recent call last): - ... - SyntaxError: 'yield' not allowed in a 'try' block with a 'finally' clause (, line 6) -Got nothing -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.syntax -Failed example: - def f(): - yield -Expected: - Traceback (most recent call last): - SyntaxError: invalid syntax -Got: - Traceback (most recent call last): - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/doctest.py", line 1243, in __run - compileflags, 1) in test.globs - File "", line 2 - def f(): - ^ - SyntaxError: error -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.syntax -Failed example: - def f(): - if 0: - yield -Expected: - Traceback (most recent call last): - SyntaxError: invalid syntax -Got: - Traceback (most recent call last): - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/doctest.py", line 1243, in __run - compileflags, 1) in test.globs - File "", line 3 - def f(): - ^ - SyntaxError: error -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.syntax -Failed example: - type(f()) -Expected: - -Got: - -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.syntax -Failed example: - type(f()) -Expected: - -Got: - -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.syntax -Failed example: - type(f()) -Expected: - -Got: - -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.syntax -Failed example: - type(f()) -Expected: - -Got: - -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.syntax -Failed example: - def f(): - if 0: - lambda x: x # shouldn't trigger here - return # or here - def f(i): - return 2*i # or here - if 0: - return 3 # but *this* sucks (line 8) - if 0: - yield 2 # because it's a generator -Expected: - Traceback (most recent call last): - SyntaxError: 'return' with argument inside generator (, line 8) -Got nothing -********************************************************************** -3 items had failures: - 1 of 22 in test.test_generators.__test__.pep - 12 of 29 in test.test_generators.__test__.syntax - 8 of 10 in test.test_generators.__test__.weakref -***Test Failed*** 21 failures. - ---===============1565511160== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stderr" - -faking -Loading grammar /home/contest/xoraxax/pypy-dist/pypy/interpreter/pyparser/data/Grammar2.4 -faking -faking -faking -fake-wrapping interp file ', mode 'w' at 0xf7fa3068> -fake-wrapping interp file ', mode 'w' at 0xf7fa30b0> -fake-wrapping interp file ', mode 'r' at 0xf7fa3020> -faking -faking -faking -Traceback (application-level): - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line 1405 in - test_main(1) - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line 1401 in test_main - test_support.run_doctest(test_generators, verbose) - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_support.py", line 319 in run_doctest - finally: -TestFailed: 21 of 154 doctests failed - ---===============1565511160==-- \ No newline at end of file diff --git a/pypy/tool/pytest/test/data/test_global.txt b/pypy/tool/pytest/test/data/test_global.txt deleted file mode 100644 --- a/pypy/tool/pytest/test/data/test_global.txt +++ /dev/null @@ -1,94 +0,0 @@ -Content-Type: multipart/mixed; boundary="===============1988336873==" -MIME-Version: 1.0 -cpu mhz: unknown -cpu model: unknown -execution-time: 12.6530230045 -exit status: 2 -fspath: /home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_global.py -options: ['core'] -outcome: ERROUT -platform: linux2 -pypy-revision: 16123 -python-version-info: (2, 4, 1, 'final', 0) -startdate: Thu Aug 18 02:12:59 2005 -testreport-version: 1.1 -timeout: 3844.0 -userhost: xoraxax at tick -_reprs: {'execution-time': 'float', 'python-version-info': 'tuple', - 'options': 'list', 'timeout': 'float', 'pypy-revision': 'int', - 'exit status': 'int'} - ---===============1988336873== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stdout" - -should have raised SyntaxError: -def wrong1(): - a = 1 - b = 2 - global a - global b - -should have raised SyntaxError: -def wrong2(): - print x - global x - -should have raised SyntaxError: -def wrong3(): - print x - x = 2 - global x - -as expected, no SyntaxError - ---===============1988336873== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stderr" - -faking -Loading grammar /home/contest/xoraxax/pypy-dist/pypy/interpreter/pyparser/data/Grammar2.4 -faking -faking -faking -fake-wrapping interp file ', mode 'w' at 0xf7fa3068> -fake-wrapping interp file ', mode 'w' at 0xf7fa30b0> -fake-wrapping interp file ', mode 'r' at 0xf7fa3020> -faking - ---===============1988336873== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="reportdiff" - -********************************************************************** -*** mismatch between lines 2-4 of expected output and lines 2-19 of actual output: -- got SyntaxError as expected -- got SyntaxError as expected -- got SyntaxError as expected -+ should have raised SyntaxError: -+ def wrong1(): -+ a = 1 -+ b = 2 -+ global a -+ global b -+ -+ should have raised SyntaxError: -+ def wrong2(): -+ print x -+ global x -+ -+ should have raised SyntaxError: -+ def wrong3(): -+ print x -+ x = 2 -+ global x -+ -********************************************************************** - ---===============1988336873==-- \ No newline at end of file diff --git a/pypy/tool/pytest/test/data/test_sys.txt b/pypy/tool/pytest/test/data/test_sys.txt deleted file mode 100644 --- a/pypy/tool/pytest/test/data/test_sys.txt +++ /dev/null @@ -1,61 +0,0 @@ -Content-Type: multipart/mixed; boundary="===============1380540766==" -MIME-Version: 1.0 -cpu mhz: unknown -cpu model: unknown -execution-time: 22.5916008949 -exit status: 0 -fspath: /home/contest/xoraxax/pypy-dist/lib-python/modified-2.4.1/test/test_sys.py -options: ['core'] -outcome: OK -platform: linux2 -pypy-revision: 16123 -python-version-info: (2, 4, 1, 'final', 0) -startdate: Thu Aug 18 04:16:48 2005 -testreport-version: 1.1 -timeout: 3364.0 -userhost: xoraxax at tick -_reprs: {'execution-time': 'float', 'python-version-info': 'tuple', - 'options': 'list', 'timeout': 'float', 'pypy-revision': 'int', - 'exit status': 'int'} - ---===============1380540766== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stdout" - -test_attributes (__main__.SysModuleTest) ... ok -test_custom_displayhook (__main__.SysModuleTest) ... ok -test_dlopenflags (__main__.SysModuleTest) ... ok -test_exc_clear (__main__.SysModuleTest) ... ok -test_exit (__main__.SysModuleTest) ... ok -test_getdefaultencoding (__main__.SysModuleTest) ... ok -test_getframe (__main__.SysModuleTest) ... ok -test_getwindowsversion (__main__.SysModuleTest) ... ok -test_lost_displayhook (__main__.SysModuleTest) ... ok -test_original_displayhook (__main__.SysModuleTest) ... ok -test_original_excepthook (__main__.SysModuleTest) ... ok -test_recursionlimit (__main__.SysModuleTest) ... ok -test_setcheckinterval (__main__.SysModuleTest) ... ok - ----------------------------------------------------------------------- -Ran 13 tests in 7.241s - -OK - ---===============1380540766== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stderr" - -faking -Loading grammar /home/contest/xoraxax/pypy-dist/pypy/interpreter/pyparser/data/Grammar2.4 -faking -faking -faking -fake-wrapping interp file ', mode 'w' at 0xf7fa3068> -fake-wrapping interp file ', mode 'w' at 0xf7fa30b0> -fake-wrapping interp file ', mode 'r' at 0xf7fa3020> - ---===============1380540766==-- \ No newline at end of file From pypy.commits at gmail.com Sun Apr 15 12:08:18 2018 From: pypy.commits at gmail.com (rlamy) Date: Sun, 15 Apr 2018 09:08:18 -0700 (PDT) Subject: [pypy-commit] pypy apptest-file: Do assert reinterpretation only inside old-style apptests; fix tests. Message-ID: <5ad378f2.029bdf0a.726b3.1421@mx.google.com> Author: Ronan Lamy Branch: apptest-file Changeset: r94337:d719e76c1bc7 Date: 2018-04-15 17:07 +0100 http://bitbucket.org/pypy/pypy/changeset/d719e76c1bc7/ Log: Do assert reinterpretation only inside old-style apptests; fix tests. diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -247,6 +247,10 @@ BoolOption("newshortcut", "cache and shortcut calling __new__ from builtin types", default=False), + BoolOption("reinterpretasserts", + "Perform reinterpretation when an assert fails " + "(only relevant for tests)", + default=False), ]), ]) diff --git a/pypy/tool/pytest/apptest.py b/pypy/tool/pytest/apptest.py --- a/pypy/tool/pytest/apptest.py +++ b/pypy/tool/pytest/apptest.py @@ -44,7 +44,7 @@ target = self.obj if self.config.option.runappdirect: return target() - space = gettestobjspace() + space = gettestobjspace(**{'objspace.std.reinterpretasserts': True}) filename = self._getdynfilename(target) func = app2interp_temp(target, filename=filename) print "executing", func diff --git a/pypy/tool/pytest/objspace.py b/pypy/tool/pytest/objspace.py --- a/pypy/tool/pytest/objspace.py +++ b/pypy/tool/pytest/objspace.py @@ -7,10 +7,10 @@ _SPACECACHE={} def gettestobjspace(**kwds): - """ helper for instantiating and caching space's for testing. + """ helper for instantiating and caching spaces for testing. """ try: - config = make_config(option,**kwds) + config = make_config(option, **kwds) except ConflictConfigError as e: # this exception is typically only raised if a module is not available. # in this case the test should be skipped @@ -33,8 +33,9 @@ config.objspace.extmodules = 'pypy.tool.pytest.fake_pytest' space = make_objspace(config) space.startup() # Initialize all builtin modules - space.setitem(space.builtin.w_dict, space.wrap('AssertionError'), - appsupport.build_pytest_assertion(space)) + if config.objspace.std.reinterpretasserts: + space.setitem(space.builtin.w_dict, space.wrap('AssertionError'), + appsupport.build_pytest_assertion(space)) space.setitem(space.builtin.w_dict, space.wrap('raises'), space.wrap(appsupport.app_raises)) space.setitem(space.builtin.w_dict, space.wrap('skip'), diff --git a/pypy/tool/pytest/test/apptest_xx.py b/pypy/tool/pytest/test/apptest_xx.py deleted file mode 100644 --- a/pypy/tool/pytest/test/apptest_xx.py +++ /dev/null @@ -1,3 +0,0 @@ -def test_fail(): - x = 'foo' - assert x == 'bar' diff --git a/pypy/tool/pytest/test/test_appsupport.py b/pypy/tool/pytest/test/test_appsupport.py --- a/pypy/tool/pytest/test/test_appsupport.py +++ b/pypy/tool/pytest/test/test_appsupport.py @@ -154,7 +154,7 @@ result = testdir.runpytest(p) assert result.ret == 1 result.stdout.fnmatch_lines([ - "*E*application-level*KeyError*42*", + "*E*(application-level) AssertionError", ]) def test_apptest_fail_rewrite(testdir): @@ -167,7 +167,9 @@ result = testdir.runpytest(p, "--applevel-rewrite") assert result.ret == 1 result.stdout.fnmatch_lines([ - "*E*application-level*KeyError*42*", + "*E*application-level*AssertionError: assert 'foo' == 'bar'", + "*E*- foo*", + "*E*+ bar*", ]) From pypy.commits at gmail.com Sun Apr 15 12:51:55 2018 From: pypy.commits at gmail.com (rlamy) Date: Sun, 15 Apr 2018 09:51:55 -0700 (PDT) Subject: [pypy-commit] pypy apptest-file: there are no app_test directories now Message-ID: <5ad3832b.89c4df0a.182ee.9b2f@mx.google.com> Author: Ronan Lamy Branch: apptest-file Changeset: r94338:dd33811ad1b4 Date: 2018-04-15 17:51 +0100 http://bitbucket.org/pypy/pypy/changeset/dd33811ad1b4/ Log: there are no app_test directories now diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -109,10 +109,9 @@ """ def accept_regular_test(self): if self.config.option.runappdirect: - # only collect regular tests if we are in an 'app_test' directory, - # or in test_lib_pypy + # only collect regular tests if we are in test_lib_pypy for name in self.listnames(): - if "app_test" in name or "test_lib_pypy" in name: + if "test_lib_pypy" in name: return True return False return True From pypy.commits at gmail.com Sun Apr 15 12:57:30 2018 From: pypy.commits at gmail.com (rlamy) Date: Sun, 15 Apr 2018 09:57:30 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: hg merge default Message-ID: <5ad3847a.dcb1df0a.2e37d.468b@mx.google.com> Author: Ronan Lamy Branch: py3.5 Changeset: r94339:7967a85cf805 Date: 2018-04-15 17:56 +0100 http://bitbucket.org/pypy/pypy/changeset/7967a85cf805/ Log: hg merge default diff --git a/pypy/tool/pytest/genreportdata.py b/pypy/tool/pytest/genreportdata.py deleted file mode 100755 --- a/pypy/tool/pytest/genreportdata.py +++ /dev/null @@ -1,30 +0,0 @@ -#! /usr/bin/env python -import py -import sys - -mydir = py.path.local(__file__).dirpath().realpath() -from pypy.tool.pytest import htmlreport -from pypy.tool.pytest import confpath - -if __name__ == '__main__': - if len(sys.argv) > 1: - testresultdir = py.path.local(sys.argv[1]) - assert testresultdir.check(dir=1) - else: - testresultdir = confpath.testresultdir - assert testresultdir.check(dir=1) - try: - resultwc = py.path.svnwc(testresultdir) - print "updating", resultwc - resultwc.update() - except (KeyboardInterrupt, RuntimeError): - raise - except Exception as e: #py.process.ExecutionFailed,e: - print >> sys.stderr, "Warning: ",e #Subversion update failed" - - print "traversing", mydir - rep = htmlreport.HtmlReport(testresultdir) - rep.parselatest() - - print "making html files" - rep.makeindex(testresultdir.join('index.html')) diff --git a/pypy/tool/pytest/htmlreport.py b/pypy/tool/pytest/htmlreport.py deleted file mode 100644 --- a/pypy/tool/pytest/htmlreport.py +++ /dev/null @@ -1,239 +0,0 @@ -#! /usr/bin/env python - -""" -the html test reporter - -""" -import sys, os, re -import pprint -import py -from pypy.tool.pytest import result -from pypy.tool.pytest.overview import ResultCache - -# -# various interesting path objects -# - -html = py.xml.html -NBSP = py.xml.raw(" ") - -class HtmlReport(object): - def __init__(self, resultdir): - self.resultcache = ResultCache(resultdir) - - def parselatest(self): - self.resultcache.parselatest() - - # - # rendering - # - - def render_latest_table(self, results): - table = html.table( - [html.th(x, align='left') - for x in ("failure", "filename", "revision", - "user", "platform", "elapsed", - "options", "last error line" - )], - ) - r = results[:] - def f(x, y): - xnum = x.isok() and 1 or (x.istimeout() and 2 or 3) - ynum = y.isok() and 1 or (y.istimeout() and 2 or 3) - res = -cmp(xnum, ynum) - if res == 0: - return cmp(x['execution-time'], y['execution-time']) - return res - r.sort(f) - for result in r: - table.append(self.render_result_row(result)) - return table - - def render_result_row(self, result): - dp = py.path.local(result['fspath']) - - options = " ".join([x for x in result.get('options', []) if x!= 'core']) - if not options: - options = NBSP - - failureratio = 100 * (1.0 - result.ratio_of_passed()) - self.data[result.testname] = failureratio - return html.tr( - html.td("%.2f%%" % failureratio, - style = "background-color: %s" % (getresultcolor(result),)), - html.td(self.render_test_references(result)), - html.td(result['pypy-revision']), - html.td(result['userhost'][:15]), - html.td(result['platform']), - html.td("%.2fs" % result['execution-time']), - html.td(options), - html.td(result.repr_short_error() or NBSP) - ) - - def getrelpath(self, p): - return p.relto(self.indexpath.dirpath()) - - def render_test_references(self, result): - dest = self.make_single_test_result(result) - #XXX: ask hg for differences between test and vendor branch - modified = result.ismodifiedtest() and " [mod]" or "" - return html.div(html.a(result.path.purebasename + modified, - href=self.getrelpath(dest)), - style="background-color: transparent") - - def make_single_test_result(self, result): - cache = self.indexpath.dirpath('.cache', result['userhost'][:15]) - cache.ensure(dir=1) - dest = cache.join(result.path.basename).new(ext='.html') - doc = ViewResult(result) - doc.writetopath(dest) - return dest - - def getcorelists(self): - def iscore(result): - return 'core' in result.get('options', []) - coretests = [] - noncoretests = [] - for name in self.resultcache.getnames(): - result = self.resultcache.getlatestrelevant(name) - if iscore(result): - coretests.append(result) - else: - noncoretests.append(result) - return coretests, noncoretests - - # generate html files - # - def makeindex(self, indexpath, detail="PyPy - latest"): - self.indexpath = indexpath - self.data = {} - doc = Document(title='pypy test results') - body = doc.body - coretests, noncoretests = self.getcorelists() - body.append(html.h2("%s compliance test results - " - "core tests" % detail)) - - body.append(self.render_test_summary('core', coretests)) - body.append(self.render_latest_table(coretests)) - body.append( - html.h2("%s compliance test results - non-core tests" % detail)) - body.append(self.render_test_summary('noncore', noncoretests)) - body.append(self.render_latest_table(noncoretests)) - doc.writetopath(indexpath) - datapath = indexpath.dirpath().join('data') - d = datapath.open('w') - print >>d, "data = ", - pprint.pprint(self.data, stream=d) - d.close() - self.data = None - - def render_test_summary(self, tag, tests): - ok = len([x for x in tests if x.isok()]) - err = len([x for x in tests if x.iserror()]) - to = len([x for x in tests if x.istimeout()]) - numtests = ok + err + to - assert numtests == len(tests) - assert numtests - - t = html.table() - sum100 = numtests / 100.0 - def row(*args): - return html.tr(*[html.td(arg) for arg in args]) - - sum_passed = sum([x.ratio_of_passed() for x in tests]) - compliancy = sum_passed/sum100 - self.data['%s-compliancy' % tag] = compliancy - t.append(row(html.b("tests compliancy"), - html.b("%.2f%%" % (compliancy,)))) - - passed = ok/sum100 - self.data['%s-passed' % tag] = passed - t.append(row("testmodules passed completely", "%.2f%%" % passed)) - failed = err/sum100 - self.data['%s-failed' % tag] = failed - t.append(row("testmodules (partially) failed", "%.2f%%" % failed)) - timedout = to/sum100 - self.data['%s-timedout' % tag] = timedout - t.append(row("testmodules timeout", "%.2f%%" % timedout)) - return t - -class Document(object): - def __init__(self, title=None): - self.body = html.body() - self.head = html.head() - self.doc = html.html(self.head, self.body) - if title is not None: - self.head.append( - html.meta(name="title", content=title)) - self.head.append( - html.link(rel="Stylesheet", type="text/css", href="/pypy/default.css")) - - def writetopath(self, p): - assert p.ext == '.html' - self.head.append( - html.meta(name="Content-Type", content="text/html;charset=UTF-8") - ) - s = self.doc.unicode().encode('utf-8') - p.write(s) - -def getresultcolor(result): - if result.isok(): - color = "#00ee00" - elif result.iserror(): - color = "#ee0000" - elif result.istimeout: - color = "#0000ee" - else: - color = "#444444" - return color - -class ViewResult(Document): - def __init__(self, result): - title = "%s testresult" % (result.path.purebasename,) - super(ViewResult, self).__init__(title=title) - color = getresultcolor(result) - self.body.append(html.h2(title, - style="background-color: %s" % color)) - self.body.append(self.render_meta_info(result)) - - for name in ('reportdiff', 'stdout', 'stderr'): - try: - text = result.getnamedtext(name) - except KeyError: - continue - self.body.append(html.h3(name)) - self.body.append(html.pre(text)) - - def render_meta_info(self, result): - t = html.table() - items = result.items() - items.sort() - for name, value in items: - if name.lower() == name: - t.append(html.tr( - html.td(name), html.td(value))) - return t - -class TestOfHtmlReportClass: - def setup_class(cls): - py.test.skip('needs move to own test file') - cls.testresultdir = confpath.testresultdir - cls.rep = rep = HtmlReport() - rep.parse_all(cls.testresultdir) - - def test_pickling(self): - # test pickling of report - tempdir = py.test.ensuretemp('reportpickle') - picklepath = tempdir.join('report.pickle') - picklepath.dump(self.rep) - x = picklepath.load() - assert len(x.results) == len(self.rep.results) - - def test_render_latest(self): - t = self.rep.render_latest_table(self.rep.results) - assert unicode(t) - -mydir = py.path.local(__file__).dirpath() - -def getpicklepath(): - return mydir.join('.htmlreport.pickle') diff --git a/pypy/tool/pytest/overview.py b/pypy/tool/pytest/overview.py deleted file mode 100644 --- a/pypy/tool/pytest/overview.py +++ /dev/null @@ -1,56 +0,0 @@ -from pypy.tool.pytest import result -import sys - -class ResultCache: - def __init__(self, resultdir): - self.resultdir = resultdir - self.name2result = {} - - def parselatest(self): - def filefilter(p): - return p.check(fnmatch='test_*.txt', file=1) - def rec(p): - return p.check(dotfile=0) - for x in self.resultdir.visit(filefilter, rec): - self.parse_one(x) - - def parse_one(self, resultpath): - try: - res = result.ResultFromMime(resultpath) - ver = res['testreport-version'] - if ver != "1.1" and ver != "1.1.1": - raise TypeError - except TypeError: # xxx - print >>sys.stderr, "could not parse %s" % resultpath - return - name = res.testname - print name - self.name2result.setdefault(name, []).append(res) - return res - - def getnames(self): - return self.name2result.keys() - - def getlatest(self, name, timeout=0, error=0, ok=0): - l = [] - resultlist = self.name2result[name] - maxrev = 0 - maxresult = None - for res in resultlist: - resrev = res['pypy-revision'] - if resrev == 'unknown': - continue - if resrev <= maxrev: - continue - if timeout or error or ok: - if not (timeout and res.istimeout() or - error and res.iserror() or - ok and res.isok()): - continue - maxrev = resrev - maxresult = res - return maxresult - - def getlatestrelevant(self, name): - # get the latest revision that did not time out. - return self.getlatest(name, error=1, ok=1) or self.getlatest(name) diff --git a/pypy/tool/pytest/result.py b/pypy/tool/pytest/result.py deleted file mode 100644 --- a/pypy/tool/pytest/result.py +++ /dev/null @@ -1,215 +0,0 @@ -import sys -import py -import re - -class Result(object): - def __init__(self, init=True): - self._headers = {} - self._blocks = {} - self._blocknames = [] - if init: - stdinit(self) - - def __setitem__(self, name, value): - self._headers[name.lower()] = value - - def __getitem__(self, name): - return self._headers[name.lower()] - - def get(self, name, default): - return self._headers.get(name, default) - - def __delitem__(self, name): - del self._headers[name.lower()] - - def items(self): - return self._headers.items() - - def addnamedtext(self, name, text): - assert isinstance(text, basestring) - assert isinstance(name, str) - self._blocknames.append(name) - self._blocks[name] = text - - def getnamedtext(self, name): - return self._blocks[name] - - def repr_short_error(self): - if not self.isok(): - if 'reportdiff' in self._blocks: - return "output comparison failed, see reportdiff" - else: - text = self.getnamedtext('stderr') - lines = text.strip().split('\n') - if lines: - return lines[-1] - - def repr_mimemessage(self): - from email.MIMEMultipart import MIMEMultipart - from email.MIMEText import MIMEText - - outer = MIMEMultipart() - items = self._headers.items() - items.sort() - reprs = {} - for name, value in items: - assert ':' not in name - chars = map(ord, name) - assert min(chars) >= 33 and max(chars) <= 126 - outer[name] = str(value) - if not isinstance(value, str): - typename = type(value).__name__ - assert typename in vars(py.std.__builtin__) - reprs[name] = typename - - outer['_reprs'] = repr(reprs) - - for name in self._blocknames: - text = self._blocks[name] - m = MIMEText(text) - m.add_header('Content-Disposition', 'attachment', filename=name) - outer.attach(m) - return outer - - def grep_nr(self,text,section='stdout'): - stdout = self._blocks[section] - find = re.search('%s(?P\d+)'%text,stdout) - if find: - return float(find.group('nr')) - return 0. - - def ratio_of_passed(self): - if self.isok(): - return 1. - elif self.istimeout(): - return 0. - else: - nr = self.grep_nr('Ran ') - if nr > 0: - return (nr - (self.grep_nr('errors=') + self.grep_nr('failures=')))/nr - else: - passed = self.grep_nr('TestFailed: ',section='stderr') - run = self.grep_nr('TestFailed: \d+/',section='stderr') - if run > 0: - return passed/run - else: - run = self.grep_nr('TestFailed: \d+ of ',section='stderr') - if run > 0 : - return (run-passed)/run - else: - return 0.0 - - def isok(self): - return self['outcome'].lower() == 'ok' - - def iserror(self): - return self['outcome'].lower()[:3] == 'err' or self['outcome'].lower() == 'fail' - - def istimeout(self): - return self['outcome'].lower() == 't/o' - -# XXX backward compatibility -def sanitize(msg, path): - if 'exit-status' in msg.keys(): - return msg - f = open(str(path), 'r') - msg = f.read() - f.close() - for broken in ('exit status', 'cpu model', 'cpu mhz'): - valid = broken.replace(' ','-') - invalid = msg.find(broken+':') - msg = (msg[:invalid] + valid + - msg[invalid+len(valid):]) - from email import message_from_string - msg = message_from_string(msg) - return msg - -def sanitize_reprs(reprs): - if 'exit status' in reprs: - reprs['exit-status'] = reprs.pop('exit status') - -class ResultFromMime(Result): - def __init__(self, path): - super(ResultFromMime, self).__init__(init=False) - f = open(str(path), 'r') - from email import message_from_file - msg = message_from_file(f) - f.close() - msg = sanitize(msg, path) - # XXX security wise evil (keep in mind once we accept reporsts - # from anonymous - #print msg['_reprs'] - self._reprs = eval(msg['_reprs']) - del msg['_reprs'] - sanitize_reprs(self._reprs) - for name, value in msg.items(): - if name in self._reprs: - value = eval(value) # XXX security - self._headers[name] = value - self.fspath = self['fspath'] - if self['platform'] == 'win32' and '\\' in self.fspath: - self.testname = self.fspath.split('\\')[-1] - else: - self.testname = self.fspath.split('/')[-1] - #if sys.platform != 'win32' and '\\' in self.fspath: - # self.fspath = py.path.local(self['fspath'].replace('\\' - self.path = path - - payload = msg.get_payload() - if payload: - for submsg in payload: - assert submsg.get_content_type() == 'text/plain' - fn = submsg.get_filename() - assert fn - # XXX we need to deal better with encodings to - # begin with - content = submsg.get_payload() - for candidate in 'utf8', 'latin1': - try: - text = unicode(content, candidate) - except UnicodeDecodeError: - continue - else: - unicode(content, candidate) - self.addnamedtext(fn, text) - - def ismodifiedtest(self): - # XXX we need proper cross-platform paths! - return 'modified' in self.fspath - - def __repr__(self): - return '<%s (%s) %r rev=%s>' %(self.__class__.__name__, - self['outcome'], - self.fspath, - self['pypy-revision']) - -def stdinit(result): - import getpass - import socket - try: - username = getpass.getuser() - except: - username = 'unknown' - userhost = '%s@%s' % (username, socket.gethostname()) - result['testreport-version'] = "1.1.1" - result['userhost'] = userhost - result['platform'] = sys.platform - result['python-version-info'] = sys.version_info - info = try_getcpuinfo() - if info is not None: - result['cpu-model'] = info.get('model name', "unknown") - result['cpu-mhz'] = info.get('cpu mhz', 'unknown') -# -# -# -def try_getcpuinfo(): - if sys.platform.startswith('linux'): - cpuinfopath = py.path.local('/proc/cpuinfo') - if cpuinfopath.check(file=1): - d = {} - for line in cpuinfopath.readlines(): - if line.strip(): - name, value = line.split(':', 1) - name = name.strip().lower() - d[name] = value.strip() - return d diff --git a/pypy/tool/pytest/test/data/test___all__.txt b/pypy/tool/pytest/test/data/test___all__.txt deleted file mode 100644 --- a/pypy/tool/pytest/test/data/test___all__.txt +++ /dev/null @@ -1,94 +0,0 @@ -Content-Type: multipart/mixed; boundary="===============0790678169==" -MIME-Version: 1.0 -execution-time: 1445.14346004 -exit status: 1 -fspath: /Users/anderslehmann/pypy/lib-python/2.4.1/test/test___all__.py -options: ['core', '_sre'] -outcome: T/O -platform: darwin -pypy-revision: 16114 -python-version-info: (2, 4, 1, 'final', 0) -startdate: Wed Aug 17 23:51:59 2005 -testreport-version: 1.1 -timeout: 1369.0 -userhost: anderslehmann at anders-lehmanns-15-powerbook-g4.local -_reprs: {'execution-time': 'float', 'python-version-info': 'tuple', - 'options': 'list', 'timeout': 'float', 'pypy-revision': 'int', - 'exit status': 'int'} - ---===============0790678169== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stdout" - -test_all (__main__.AllTest) ... ERROR - -====================================================================== -ERROR: test_all (__main__.AllTest) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test___all__.py", line 163, in test_all - self.check_all("tty") - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test___all__.py", line 26, in check_all - "%s has no __all__ attribute" % modname) - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_support.py", line 208, in verify - raise TestFailed(reason) -TestFailed: tty has no __all__ attribute - ----------------------------------------------------------------------- - ---===============0790678169== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stderr" - -faking -Loading grammar /Users/anderslehmann/pypy/pypy/interpreter/pyparser/data/Grammar2.4 -faking -faking -faking -fake-wrapping interp file ', mode 'w' at 0x12068> -fake-wrapping interp file ', mode 'w' at 0x120b0> -fake-wrapping interp file ', mode 'r' at 0x12020> -faking -faking -faking -faking -==========================timedout========================== -Traceback (application-level): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test___all__.py", line 192 in - test_main() - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test___all__.py", line 189 in test_main - test_support.run_unittest(AllTest) - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_support.py", line 290 in run_unittest - run_suite(suite, testclass) - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_support.py", line 262 in run_suite - result = runner.run(suite) -Traceback (application-level): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/atexit.py", line 29 in _run_exitfuncs - print >> sys.stderr, "Error in atexit._run_exitfuncs:" -KeyboardInterrupt -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/pypy/tool/alarm.py", line 43, in ? - execfile(_main_with_alarm(finished)) - File "/Users/anderslehmann/pypy/pypy/bin/py.py", line 206, in ? - sys.exit(main_(sys.argv)) - File "/Users/anderslehmann/pypy/pypy/bin/py.py", line 115, in main_ - if not main.run_toplevel(space, doit, verbose=Options.verbose): - File "/Users/anderslehmann/pypy/pypy/interpreter/main.py", line 150, in run_toplevel - operationerr.print_application_traceback(space) - File "/Users/anderslehmann/pypy/pypy/interpreter/error.py", line 83, in print_application_traceback - self.print_app_tb_only(file) - File "/Users/anderslehmann/pypy/pypy/interpreter/error.py", line 104, in print_app_tb_only - l = linecache.getline(fname, lineno) - File "/Library/Frameworks/Python.framework/Versions/2.4/lib/python2.4/linecache.py", line 14, in getline - lines = getlines(filename) - File "/Library/Frameworks/Python.framework/Versions/2.4/lib/python2.4/linecache.py", line 40, in getlines - return updatecache(filename) - File "/Library/Frameworks/Python.framework/Versions/2.4/lib/python2.4/linecache.py", line 101, in updatecache - lines = fp.readlines() -KeyboardInterrupt - ---===============0790678169==-- \ No newline at end of file diff --git a/pypy/tool/pytest/test/data/test_compile.txt b/pypy/tool/pytest/test/data/test_compile.txt deleted file mode 100644 --- a/pypy/tool/pytest/test/data/test_compile.txt +++ /dev/null @@ -1,111 +0,0 @@ -Content-Type: multipart/mixed; boundary="===============2137793924==" -MIME-Version: 1.0 -execution-time: 34.8464071751 -exit status: 1 -fspath: /Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py -options: ['core', '_sre'] -outcome: ERR -platform: darwin -pypy-revision: 16114 -python-version-info: (2, 4, 1, 'final', 0) -startdate: Thu Aug 18 03:08:18 2005 -testreport-version: 1.1 -timeout: 1521.0 -userhost: anderslehmann at anders-lehmanns-15-powerbook-g4.local -_reprs: {'execution-time': 'float', 'python-version-info': 'tuple', - 'options': 'list', 'timeout': 'float', 'pypy-revision': 'int', - 'exit status': 'int'} - ---===============2137793924== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stdout" - -test_argument_handling (__main__.TestSpecifics) ... FAIL -test_argument_order (__main__.TestSpecifics) ... FAIL -test_complex_args (__main__.TestSpecifics) ... ok -test_debug_assignment (__main__.TestSpecifics) ... FAIL -test_duplicate_global_local (__main__.TestSpecifics) ... ok -test_exec_with_general_mapping_for_locals (__main__.TestSpecifics) ... ok -test_float_literals (__main__.TestSpecifics) ... ok -test_for_distinct_code_objects (__main__.TestSpecifics) ... ok -test_import (__main__.TestSpecifics) ... FAIL -test_indentation (__main__.TestSpecifics) ... ok -test_literals_with_leading_zeroes (__main__.TestSpecifics) ... ok -test_none_assignment (__main__.TestSpecifics) ... FAIL -test_sequence_unpacking_error (__main__.TestSpecifics) ... ok -test_syntax_error (__main__.TestSpecifics) ... ok -test_unary_minus (__main__.TestSpecifics) ... ok - -====================================================================== -FAIL: test_argument_handling (__main__.TestSpecifics) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 18, in test_argument_handling - self.assertRaises(SyntaxError, eval, 'lambda a,a:0') -AssertionError: SyntaxError not raised - -====================================================================== -FAIL: test_argument_order (__main__.TestSpecifics) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 127, in test_argument_order - self.fail("non-default args after default") -AssertionError: non-default args after default - -====================================================================== -FAIL: test_debug_assignment (__main__.TestSpecifics) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 10, in test_debug_assignment - self.assertRaises(SyntaxError, compile, '__debug__ = 1', '?', 'single') -AssertionError: SyntaxError not raised - -====================================================================== -FAIL: test_import (__main__.TestSpecifics) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 253, in test_import - self.assertRaises(SyntaxError, compile, stmt, 'tmp', 'exec') -AssertionError: SyntaxError not raised - -====================================================================== -FAIL: test_none_assignment (__main__.TestSpecifics) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 211, in test_none_assignment - self.assertRaises(SyntaxError, compile, stmt, 'tmp', 'single') -AssertionError: SyntaxError not raised - ----------------------------------------------------------------------- -Ran 15 tests in 14.363s - -FAILED (failures=5) - ---===============2137793924== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stderr" - -faking -Loading grammar /Users/anderslehmann/pypy/pypy/interpreter/pyparser/data/Grammar2.4 -faking -faking -faking -fake-wrapping interp file ', mode 'w' at 0x12068> -fake-wrapping interp file ', mode 'w' at 0x120b0> -fake-wrapping interp file ', mode 'r' at 0x12020> -Traceback (application-level): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 268 in - test_main() - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 265 in test_main - test_support.run_unittest(TestSpecifics) - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_support.py", line 290 in run_unittest - run_suite(suite, testclass) - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_support.py", line 274 in run_suite - raise TestFailed(msg) -TestFailed: errors occurred in __main__.TestSpecifics - ---===============2137793924==-- \ No newline at end of file diff --git a/pypy/tool/pytest/test/data/test_descr.txt b/pypy/tool/pytest/test/data/test_descr.txt deleted file mode 100644 --- a/pypy/tool/pytest/test/data/test_descr.txt +++ /dev/null @@ -1,233 +0,0 @@ -Content-Type: multipart/mixed; boundary="===============1265023865==" -MIME-Version: 1.0 -execution-time: 4098.8407588 -exit status: 1 -fspath: /Users/anderslehmann/pypy/lib-python/modified-2.4.1/test/test_descr.py -options: ['oldstyle', 'core'] -outcome: ERR -platform: darwin -pypy-revision: 16388 -python-version-info: (2, 4, 1, 'final', 0) -startdate: Wed Aug 24 16:54:12 2005 -testreport-version: 1.1 -timeout: 10000.0 -userhost: anderslehmann at anders-lehmanns-15-powerbook-g4.local -_reprs: {'execution-time': 'float', 'python-version-info': 'tuple', - 'options': 'list', 'timeout': 'float', 'pypy-revision': 'int', - 'exit status': 'int'} - ---===============1265023865== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stdout" - -**************************************** ---> weakref_segfault FAILURE(0/92) _weakref -**************************************** ---> do_this_first OK(1/92) -**************************************** ---> class_docstrings OK(2/92) -**************************************** ---> lists FAILURE(2/92) type object 'list' has no attribute '__getslice__' -**************************************** ---> dicts FAILURE(2/92) type object 'dict' has no attribute '__cmp__' -**************************************** ---> dict_constructor OK(3/92) -**************************************** ---> test_dir OK(4/92) -**************************************** ---> ints OK(5/92) -**************************************** ---> longs OK(6/92) -**************************************** ---> floats OK(7/92) -**************************************** ---> complexes OK(8/92) -**************************************** ---> spamlists FAILURE(8/92) xxsubtype -**************************************** ---> spamdicts FAILURE(8/92) xxsubtype -**************************************** ---> pydicts OK(9/92) -**************************************** ---> pylists OK(10/92) -**************************************** ---> metaclass OK(11/92) -**************************************** ---> pymods OK(12/92) -**************************************** ---> multi OK(13/92) -**************************************** ---> mro_disagreement FAILURE(13/92) Message 'cycle among base classes: A < B < A', expected 'Cannot create a consistent method resolution\norder (MRO) for bases ' -**************************************** ---> diamond OK(14/92) -**************************************** ---> ex5 OK(15/92) -**************************************** ---> monotonicity OK(16/92) -**************************************** ---> consistency_with_epg OK(17/92) -**************************************** ---> objects OK(18/92) -**************************************** ---> slots FAILURE(18/92) ['foo bar'] slots not caught -**************************************** ---> slotspecials FAILURE(18/92) test failed -**************************************** ---> dynamics OK(19/92) -**************************************** ---> errors FAILURE(19/92) inheritance from CFunction should be illegal -**************************************** ---> classmethods OK(20/92) -**************************************** ---> classmethods_in_c FAILURE(20/92) xxsubtype -**************************************** ---> staticmethods OK(21/92) -**************************************** ---> staticmethods_in_c FAILURE(21/92) xxsubtype -**************************************** ---> classic OK(22/92) -**************************************** ---> compattr OK(23/92) -**************************************** ---> newslot OK(24/92) -**************************************** ---> altmro OK(25/92) -**************************************** ---> overloading OK(26/92) -**************************************** ---> methods FAILURE(26/92) test failed -**************************************** ---> specials FAILURE(26/92) shouldn't allow .__cmp__(u'123', '123') -**************************************** ---> weakrefs FAILURE(26/92) 'module' object has no attribute 'ref' -**************************************** ---> properties FAILURE(26/92) expected TypeError from trying to set readonly '__doc__' attr on a property -**************************************** ---> supers OK(27/92) -**************************************** ---> inherits FAILURE(27/92) test failed -**************************************** ---> keywords FAILURE(27/92) descr__new__() got an unexpected keyword argument 'string' -**************************************** ---> restricted OK(28/92) -**************************************** ---> str_subclass_as_dict_key OK(29/92) -**************************************** ---> classic_comparisons OK(30/92) -**************************************** ---> rich_comparisons OK(31/92) -**************************************** ---> coercions OK(32/92) -**************************************** ---> descrdoc FAILURE(32/92) 'The most base type' == 'True if the file is closed' -**************************************** ---> setclass OK(33/92) -**************************************** ---> setdict OK(34/92) -**************************************** ---> pickles OK(35/92) -**************************************** ---> copies OK(36/92) -**************************************** ---> binopoverride OK(37/92) -**************************************** ---> subclasspropagation OK(38/92) -**************************************** ---> buffer_inherit OK(39/92) -**************************************** ---> str_of_str_subclass OK(40/92) -**************************************** ---> kwdargs OK(41/92) -**************************************** ---> delhook OK(42/92) -**************************************** ---> hashinherit OK(43/92) -**************************************** ---> strops OK(44/92) -**************************************** ---> deepcopyrecursive OK(45/92) -**************************************** ---> modules OK(46/92) -**************************************** ---> dictproxyiterkeys FAILURE(46/92) ['__dict__', '__module__', 'meth'] == ['__dict__', '__doc__', '__module__', '__weakref__', 'meth'] -**************************************** ---> dictproxyitervalues FAILURE(46/92) 3 == 5 -**************************************** ---> dictproxyiteritems FAILURE(46/92) ['__dict__', '__module__', 'meth'] == ['__dict__', '__doc__', '__module__', '__weakref__', 'meth'] -**************************************** ---> pickleslots OK(47/92) -**************************************** ---> funnynew OK(48/92) -**************************************** ---> imulbug OK(49/92) -**************************************** ---> docdescriptor OK(50/92) -**************************************** ---> string_exceptions FAILURE(50/92) string subclass allowed as exception -**************************************** ---> copy_setstate OK(51/92) -**************************************** ---> slices OK(52/92) -**************************************** ---> subtype_resurrection OK(53/92) -**************************************** ---> slottrash OK(54/92) -**************************************** ---> slotmultipleinheritance FAILURE(54/92) type object 'C' has no attribute '__basicsize__' -**************************************** ---> testrmul OK(55/92) -**************************************** ---> testipow OK(56/92) -**************************************** ---> test_mutable_bases FAILURE(56/92) readonly attribute -**************************************** ---> test_mutable_bases_with_failing_mro FAILURE(56/92) readonly attribute -**************************************** ---> test_mutable_bases_catch_mro_conflict OK(57/92) -**************************************** ---> mutable_names OK(58/92) -**************************************** ---> subclass_right_op OK(59/92) -**************************************** ---> dict_type_with_metaclass OK(60/92) -**************************************** ---> meth_class_get FAILURE(60/92) shouldn't have allowed descr.__get__(None, None) -**************************************** ---> isinst_isclass OK(61/92) -**************************************** ---> proxysuper OK(62/92) -**************************************** ---> carloverre OK(63/92) -**************************************** ---> filefault OK(64/92) -**************************************** ---> vicious_descriptor_nonsense OK(65/92) -**************************************** ---> test_init FAILURE(65/92) did not test __init__() for None return - ---===============1265023865== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stderr" - -faking -Loading grammar /Users/anderslehmann/pypy/pypy/interpreter/pyparser/data/Grammar2.4 -faking -faking -faking -fake-wrapping interp file ', mode 'w' at 0x12068> -fake-wrapping interp file ', mode 'w' at 0x120b0> -fake-wrapping interp file ', mode 'r' at 0x12020> -faking -faking -Traceback (application-level): - File "/Users/anderslehmann/pypy/pypy/tool/pytest/regrverbose.py", line 12 in - indirect_test() - File "/Users/anderslehmann/pypy/lib-python/modified-2.4.1/test/test_descr.py", line 4102 in test_main - raise TestFailed, "%d/%d" % (success, n) -TestFailed: 65/92 - ---===============1265023865==-- \ No newline at end of file diff --git a/pypy/tool/pytest/test/data/test_generators.txt b/pypy/tool/pytest/test/data/test_generators.txt deleted file mode 100644 --- a/pypy/tool/pytest/test/data/test_generators.txt +++ /dev/null @@ -1,317 +0,0 @@ -Content-Type: multipart/mixed; boundary="===============1565511160==" -MIME-Version: 1.0 -cpu mhz: unknown -cpu model: unknown -execution-time: 671.678878069 -exit status: 1 -fspath: /home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py -options: ['core'] -outcome: ERR -platform: linux2 -pypy-revision: 16123 -python-version-info: (2, 4, 1, 'final', 0) -startdate: Thu Aug 18 02:08:13 2005 -testreport-version: 1.1 -timeout: 3136.0 -userhost: xoraxax at tick -_reprs: {'execution-time': 'float', 'python-version-info': 'tuple', - 'options': 'list', 'timeout': 'float', 'pypy-revision': 'int', - 'exit status': 'int'} - ---===============1565511160== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stdout" - -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.weakref -Failed example: - import weakref -Exception raised: - Traceback (most recent call last): - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/doctest.py", line 1243, in __run - compileflags, 1) in test.globs - File "", line 1, in - import weakref - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/weakref.py", line 14, in - from _weakref import ( - ImportError: _weakref -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.weakref -Failed example: - wr = weakref.ref(gen) -Exception raised: - Traceback (most recent call last): - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/doctest.py", line 1243, in __run - compileflags, 1) in test.globs - File "", line 1, in - wr = weakref.ref(gen) - NameError: global name 'weakref' is not defined -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.weakref -Failed example: - wr() is gen -Exception raised: - Traceback (most recent call last): - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/doctest.py", line 1243, in __run - compileflags, 1) in test.globs - File "", line 1, in - wr() is gen - NameError: global name 'wr' is not defined -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.weakref -Failed example: - p = weakref.proxy(gen) -Exception raised: - Traceback (most recent call last): - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/doctest.py", line 1243, in __run - compileflags, 1) in test.globs - File "", line 1, in - p = weakref.proxy(gen) - NameError: global name 'weakref' is not defined -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.weakref -Failed example: - wr = weakref.ref(gi) -Exception raised: - Traceback (most recent call last): - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/doctest.py", line 1243, in __run - compileflags, 1) in test.globs - File "", line 1, in - wr = weakref.ref(gi) - NameError: global name 'weakref' is not defined -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.weakref -Failed example: - wr() is gi -Exception raised: - Traceback (most recent call last): - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/doctest.py", line 1243, in __run - compileflags, 1) in test.globs - File "", line 1, in - wr() is gi - NameError: global name 'wr' is not defined -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.weakref -Failed example: - p = weakref.proxy(gi) -Exception raised: - Traceback (most recent call last): - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/doctest.py", line 1243, in __run - compileflags, 1) in test.globs - File "", line 1, in - p = weakref.proxy(gi) - NameError: global name 'weakref' is not defined -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.weakref -Failed example: - list(p) -Exception raised: - Traceback (most recent call last): - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/doctest.py", line 1243, in __run - compileflags, 1) in test.globs - File "", line 1, in - list(p) - NameError: global name 'p' is not defined -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.pep -Failed example: - k.next() -Expected: - Traceback (most recent call last): - File "", line 1, in ? - File "", line 2, in g - File "", line 2, in f - ZeroDivisionError: integer division or modulo by zero -Got: - Traceback (most recent call last): - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/doctest.py", line 1243, in __run - compileflags, 1) in test.globs - File "", line 1, in - k.next() - File "", line 2, in g - yield f() # the zero division exception propagates - File "", line 2, in f - return 1//0 - ZeroDivisionError: integer division by zero -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.syntax -Failed example: - def f(): - return 22 - yield 1 -Expected: - Traceback (most recent call last): - .. - SyntaxError: 'return' with argument inside generator (, line 2) -Got nothing -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.syntax -Failed example: - def f(): - yield 1 - return 22 -Expected: - Traceback (most recent call last): - .. - SyntaxError: 'return' with argument inside generator (, line 3) -Got nothing -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.syntax -Failed example: - def f(): - yield 1 - return None -Expected: - Traceback (most recent call last): - .. - SyntaxError: 'return' with argument inside generator (, line 3) -Got nothing -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.syntax -Failed example: - def f(): - try: - yield 1 - finally: - pass -Expected: - Traceback (most recent call last): - .. - SyntaxError: 'yield' not allowed in a 'try' block with a 'finally' clause (, line 3) -Got nothing -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.syntax -Failed example: - def f(): - try: - try: - 1//0 - except ZeroDivisionError: - yield 666 # bad because *outer* try has finally - except: - pass - finally: - pass -Expected: - Traceback (most recent call last): - ... - SyntaxError: 'yield' not allowed in a 'try' block with a 'finally' clause (, line 6) -Got nothing -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.syntax -Failed example: - def f(): - yield -Expected: - Traceback (most recent call last): - SyntaxError: invalid syntax -Got: - Traceback (most recent call last): - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/doctest.py", line 1243, in __run - compileflags, 1) in test.globs - File "", line 2 - def f(): - ^ - SyntaxError: error -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.syntax -Failed example: - def f(): - if 0: - yield -Expected: - Traceback (most recent call last): - SyntaxError: invalid syntax -Got: - Traceback (most recent call last): - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/doctest.py", line 1243, in __run - compileflags, 1) in test.globs - File "", line 3 - def f(): - ^ - SyntaxError: error -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.syntax -Failed example: - type(f()) -Expected: - -Got: - -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.syntax -Failed example: - type(f()) -Expected: - -Got: - -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.syntax -Failed example: - type(f()) -Expected: - -Got: - -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.syntax -Failed example: - type(f()) -Expected: - -Got: - -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.syntax -Failed example: - def f(): - if 0: - lambda x: x # shouldn't trigger here - return # or here - def f(i): - return 2*i # or here - if 0: - return 3 # but *this* sucks (line 8) - if 0: - yield 2 # because it's a generator -Expected: - Traceback (most recent call last): - SyntaxError: 'return' with argument inside generator (, line 8) -Got nothing -********************************************************************** -3 items had failures: - 1 of 22 in test.test_generators.__test__.pep - 12 of 29 in test.test_generators.__test__.syntax - 8 of 10 in test.test_generators.__test__.weakref -***Test Failed*** 21 failures. - ---===============1565511160== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stderr" - -faking -Loading grammar /home/contest/xoraxax/pypy-dist/pypy/interpreter/pyparser/data/Grammar2.4 -faking -faking -faking -fake-wrapping interp file ', mode 'w' at 0xf7fa3068> -fake-wrapping interp file ', mode 'w' at 0xf7fa30b0> -fake-wrapping interp file ', mode 'r' at 0xf7fa3020> -faking -faking -faking -Traceback (application-level): - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line 1405 in - test_main(1) - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line 1401 in test_main - test_support.run_doctest(test_generators, verbose) - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_support.py", line 319 in run_doctest - finally: -TestFailed: 21 of 154 doctests failed - ---===============1565511160==-- \ No newline at end of file diff --git a/pypy/tool/pytest/test/data/test_global.txt b/pypy/tool/pytest/test/data/test_global.txt deleted file mode 100644 --- a/pypy/tool/pytest/test/data/test_global.txt +++ /dev/null @@ -1,94 +0,0 @@ -Content-Type: multipart/mixed; boundary="===============1988336873==" -MIME-Version: 1.0 -cpu mhz: unknown -cpu model: unknown -execution-time: 12.6530230045 -exit status: 2 -fspath: /home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_global.py -options: ['core'] -outcome: ERROUT -platform: linux2 -pypy-revision: 16123 -python-version-info: (2, 4, 1, 'final', 0) -startdate: Thu Aug 18 02:12:59 2005 -testreport-version: 1.1 -timeout: 3844.0 -userhost: xoraxax at tick -_reprs: {'execution-time': 'float', 'python-version-info': 'tuple', - 'options': 'list', 'timeout': 'float', 'pypy-revision': 'int', - 'exit status': 'int'} - ---===============1988336873== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stdout" - -should have raised SyntaxError: -def wrong1(): - a = 1 - b = 2 - global a - global b - -should have raised SyntaxError: -def wrong2(): - print x - global x - -should have raised SyntaxError: -def wrong3(): - print x - x = 2 - global x - -as expected, no SyntaxError - ---===============1988336873== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stderr" - -faking -Loading grammar /home/contest/xoraxax/pypy-dist/pypy/interpreter/pyparser/data/Grammar2.4 -faking -faking -faking -fake-wrapping interp file ', mode 'w' at 0xf7fa3068> -fake-wrapping interp file ', mode 'w' at 0xf7fa30b0> -fake-wrapping interp file ', mode 'r' at 0xf7fa3020> -faking - ---===============1988336873== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="reportdiff" - -********************************************************************** -*** mismatch between lines 2-4 of expected output and lines 2-19 of actual output: -- got SyntaxError as expected -- got SyntaxError as expected -- got SyntaxError as expected -+ should have raised SyntaxError: -+ def wrong1(): -+ a = 1 -+ b = 2 -+ global a -+ global b -+ -+ should have raised SyntaxError: -+ def wrong2(): -+ print x -+ global x -+ -+ should have raised SyntaxError: -+ def wrong3(): -+ print x -+ x = 2 -+ global x -+ -********************************************************************** - ---===============1988336873==-- \ No newline at end of file diff --git a/pypy/tool/pytest/test/data/test_sys.txt b/pypy/tool/pytest/test/data/test_sys.txt deleted file mode 100644 --- a/pypy/tool/pytest/test/data/test_sys.txt +++ /dev/null @@ -1,61 +0,0 @@ -Content-Type: multipart/mixed; boundary="===============1380540766==" -MIME-Version: 1.0 -cpu mhz: unknown -cpu model: unknown -execution-time: 22.5916008949 -exit status: 0 -fspath: /home/contest/xoraxax/pypy-dist/lib-python/modified-2.4.1/test/test_sys.py -options: ['core'] -outcome: OK -platform: linux2 -pypy-revision: 16123 -python-version-info: (2, 4, 1, 'final', 0) -startdate: Thu Aug 18 04:16:48 2005 -testreport-version: 1.1 -timeout: 3364.0 -userhost: xoraxax at tick -_reprs: {'execution-time': 'float', 'python-version-info': 'tuple', - 'options': 'list', 'timeout': 'float', 'pypy-revision': 'int', - 'exit status': 'int'} - ---===============1380540766== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stdout" - -test_attributes (__main__.SysModuleTest) ... ok -test_custom_displayhook (__main__.SysModuleTest) ... ok -test_dlopenflags (__main__.SysModuleTest) ... ok -test_exc_clear (__main__.SysModuleTest) ... ok -test_exit (__main__.SysModuleTest) ... ok -test_getdefaultencoding (__main__.SysModuleTest) ... ok -test_getframe (__main__.SysModuleTest) ... ok -test_getwindowsversion (__main__.SysModuleTest) ... ok -test_lost_displayhook (__main__.SysModuleTest) ... ok -test_original_displayhook (__main__.SysModuleTest) ... ok -test_original_excepthook (__main__.SysModuleTest) ... ok -test_recursionlimit (__main__.SysModuleTest) ... ok -test_setcheckinterval (__main__.SysModuleTest) ... ok - ----------------------------------------------------------------------- -Ran 13 tests in 7.241s - -OK - ---===============1380540766== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stderr" - -faking -Loading grammar /home/contest/xoraxax/pypy-dist/pypy/interpreter/pyparser/data/Grammar2.4 -faking -faking -faking -fake-wrapping interp file ', mode 'w' at 0xf7fa3068> -fake-wrapping interp file ', mode 'w' at 0xf7fa30b0> -fake-wrapping interp file ', mode 'r' at 0xf7fa3020> - ---===============1380540766==-- \ No newline at end of file diff --git a/pypy/tool/pytest/test/test_new_count.py b/pypy/tool/pytest/test/test_new_count.py deleted file mode 100644 --- a/pypy/tool/pytest/test/test_new_count.py +++ /dev/null @@ -1,32 +0,0 @@ - -import py -#from pypy.tool.pytest.confpath import testresultdir -from pypy.tool.pytest.result import ResultFromMime -testpath = py.path.local(__file__).dirpath('data') - -class TestResultCache: - - def test_timeout(self): - test = ResultFromMime(testpath.join('test___all__.txt')) - assert test.ratio_of_passed() == 0. - - def test_passed(self): - test = ResultFromMime(testpath.join('test_sys.txt')) - assert test.ratio_of_passed() == 1. - - def test_unittest_partial(self): - test = ResultFromMime(testpath.join('test_compile.txt')) - assert test.ratio_of_passed() == 10./15 - - def test_doctest_of(self): - test = ResultFromMime(testpath.join('test_generators.txt')) - assert test.ratio_of_passed() == 133./154 - - def test_doctest_slash(self): - test = ResultFromMime(testpath.join('test_descr.txt')) - assert test.ratio_of_passed() == 65./92 - - def test_fail(self): - test = ResultFromMime(testpath.join('test_global.txt')) - assert test.ratio_of_passed() == 0. - diff --git a/pypy/tool/pytest/test/test_overview.py b/pypy/tool/pytest/test/test_overview.py deleted file mode 100644 --- a/pypy/tool/pytest/test/test_overview.py +++ /dev/null @@ -1,23 +0,0 @@ - -import py -from pypy.tool.pytest.confpath import testresultdir -from pypy.tool.pytest.overview import ResultCache - -class TestResultCache: - def setup_class(cls): - if not testresultdir.check(dir=1): - py.test.skip("testresult directory not checked out") - cls.rc = ResultCache(testresultdir) - cls.rc.parselatest() - - def test_getlatest_all(self): - for type in 'error', 'timeout', 'ok': - for name in self.rc.getnames(): - result = self.rc.getlatest(name, **{type:1}) - if result: - meth = getattr(result, 'is'+type) - assert meth() - - #def test_getlatest_datetime(self): - # result = self.rc.getlatest('test_datetime', ok=1) - # assert result From pypy.commits at gmail.com Sun Apr 15 12:59:43 2018 From: pypy.commits at gmail.com (rlamy) Date: Sun, 15 Apr 2018 09:59:43 -0700 (PDT) Subject: [pypy-commit] pypy py3tests: hg merge apptest-file Message-ID: <5ad384ff.166a1c0a.806bd.185d@mx.google.com> Author: Ronan Lamy Branch: py3tests Changeset: r94340:8cfa4b755c8e Date: 2018-04-14 20:12 +0100 http://bitbucket.org/pypy/pypy/changeset/8cfa4b755c8e/ Log: hg merge apptest-file diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -52,3 +52,13 @@ .. branch: refactor-slots Refactor cpyext slots. + + +.. branch: call-loopinvariant-into-bridges + +Speed up branchy code that does a lot of function inlining by saving one call +to read the TLS in most bridges. + +.. branch: rpython-sprint + +Refactor in rpython signatures diff --git a/pypy/interpreter/test/apptest_pyframe.py b/pypy/interpreter/test/apptest_pyframe.py --- a/pypy/interpreter/test/apptest_pyframe.py +++ b/pypy/interpreter/test/apptest_pyframe.py @@ -13,7 +13,7 @@ import sys f = sys._getframe() assert f.f_globals is globals() - raises(AttributeError, "f.f_globals = globals()") + pytest.raises(TypeError, "f.f_globals = globals()") def test_f_builtins(): import sys, builtins diff --git a/pypy/interpreter/test/fixtures.py b/pypy/interpreter/test/fixtures.py new file mode 100644 --- /dev/null +++ b/pypy/interpreter/test/fixtures.py @@ -0,0 +1,5 @@ +from _pytest.tmpdir import TempdirFactory + +def tempfile(space, config): + tmpdir = TempdirFactory(config).getbasetemp() + return space.newtext(str(tmpdir / 'tempfile1')) diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py --- a/pypy/interpreter/test/test_pyframe.py +++ b/pypy/interpreter/test/test_pyframe.py @@ -1,4 +1,3 @@ -from rpython.tool import udir from pypy.conftest import option from pypy.interpreter.gateway import interp2app diff --git a/pypy/module/cpyext/frameobject.py b/pypy/module/cpyext/frameobject.py --- a/pypy/module/cpyext/frameobject.py +++ b/pypy/module/cpyext/frameobject.py @@ -82,10 +82,10 @@ def PyTraceBack_Here(space, w_frame): from pypy.interpreter.pytraceback import record_application_traceback state = space.fromcache(State) - if state.operror is None: + if state.get_exception() is None: return -1 frame = space.interp_w(PyFrame, w_frame) - record_application_traceback(space, state.operror, frame, 0) + record_application_traceback(space, state.get_exception(), frame, 0) return 0 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py --- a/pypy/module/cpyext/pyerrors.py +++ b/pypy/module/cpyext/pyerrors.py @@ -67,9 +67,10 @@ @cpython_api([], PyObject, result_borrowed=True) def PyErr_Occurred(space): state = space.fromcache(State) - if state.operror is None: + operror = state.get_exception() + if operror is None: return None - return state.operror.w_type # borrowed ref + return operror.w_type # borrowed ref @cpython_api([], lltype.Void) def PyErr_Clear(space): diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py --- a/pypy/module/cpyext/state.py +++ b/pypy/module/cpyext/state.py @@ -2,11 +2,18 @@ from rpython.rtyper.lltypesystem import rffi, lltype from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter import executioncontext +from pypy.interpreter.executioncontext import ExecutionContext from rpython.rtyper.annlowlevel import llhelper from rpython.rlib.rdynload import DLLHANDLE from rpython.rlib import rawrefcount import sys + +# Keep track of exceptions raised in cpyext for a particular execution +# context. +ExecutionContext.cpyext_operror = None + + class State: def __init__(self, space): self.space = space @@ -18,7 +25,8 @@ def reset(self): from pypy.module.cpyext.modsupport import PyMethodDef - self.operror = None + ec = self.space.getexecutioncontext() + ec.cpyext_operror = None self.new_method_def = lltype.nullptr(PyMethodDef) # When importing a package, use this to keep track @@ -37,17 +45,24 @@ def set_exception(self, operror): self.clear_exception() - self.operror = operror + ec = self.space.getexecutioncontext() + ec.cpyext_operror = operror def clear_exception(self): """Clear the current exception state, and return the operror.""" - operror = self.operror - self.operror = None + ec = self.space.getexecutioncontext() + operror = ec.cpyext_operror + ec.cpyext_operror = None return operror + def get_exception(self): + ec = self.space.getexecutioncontext() + return ec.cpyext_operror + @specialize.arg(1) def check_and_raise_exception(self, always=False): - operror = self.operror + ec = self.space.getexecutioncontext() + operror = ec.cpyext_operror if operror: self.clear_exception() raise operror diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py --- a/pypy/module/cpyext/test/test_api.py +++ b/pypy/module/cpyext/test/test_api.py @@ -39,7 +39,7 @@ raise Exception("%s is not callable" % (f,)) f(*args) state = space.fromcache(State) - operror = state.operror + operror = state.get_exception() if not operror: raise Exception("DID NOT RAISE") if getattr(space, 'w_' + expected_exc.__name__) is not operror.w_type: diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py --- a/pypy/module/cpyext/test/test_pyerrors.py +++ b/pypy/module/cpyext/test/test_pyerrors.py @@ -43,7 +43,8 @@ api.PyErr_SetObject(space.w_ValueError, space.wrap("a value")) assert api.PyErr_Occurred() is space.w_ValueError state = space.fromcache(State) - assert space.eq_w(state.operror.get_w_value(space), + operror = state.get_exception() + assert space.eq_w(operror.get_w_value(space), space.wrap("a value")) api.PyErr_Clear() @@ -51,12 +52,14 @@ def test_SetNone(self, space, api): api.PyErr_SetNone(space.w_KeyError) state = space.fromcache(State) - assert space.eq_w(state.operror.w_type, space.w_KeyError) - assert space.eq_w(state.operror.get_w_value(space), space.w_None) + operror = state.get_exception() + assert space.eq_w(operror.w_type, space.w_KeyError) + assert space.eq_w(operror.get_w_value(space), space.w_None) api.PyErr_Clear() api.PyErr_NoMemory() - assert space.eq_w(state.operror.w_type, space.w_MemoryError) + operror = state.get_exception() + assert space.eq_w(operror.w_type, space.w_MemoryError) api.PyErr_Clear() def test_Warning(self, space, api, capfd): @@ -475,3 +478,59 @@ '''), ]) raises(SystemError, module.oops) + + def test_error_thread_race(self): + # Check race condition: thread 0 returns from cpyext with error set, + # after thread 1 has set an error but before it returns. + module = self.import_extension('foo', [ + ("emit_error", "METH_VARARGS", + ''' + PyThreadState *save = NULL; + PyGILState_STATE gilsave; + + /* NB. synchronization due to GIL */ + static volatile int flag = 0; + int id; + + if (!PyArg_ParseTuple(args, "i", &id)) + return NULL; + + /* Proceed in thread 1 first */ + save = PyEval_SaveThread(); + while (id == 0 && flag == 0); + gilsave = PyGILState_Ensure(); + + PyErr_Format(PyExc_ValueError, "%d", id); + + /* Proceed in thread 0 first */ + if (id == 1) flag = 1; + PyGILState_Release(gilsave); + while (id == 1 && flag == 1); + PyEval_RestoreThread(save); + + if (id == 0) flag = 0; + return NULL; + ''' + ), + ]) + + import threading + + failures = [] + + def worker(arg): + try: + module.emit_error(arg) + failures.append(True) + except Exception as exc: + if str(exc) != str(arg): + failures.append(exc) + + threads = [threading.Thread(target=worker, args=(j,)) + for j in (0, 1)] + for t in threads: + t.start() + for t in threads: + t.join() + + assert not failures diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -1403,6 +1403,7 @@ if len(e.value.args) > 2: assert e.value.args[2] == "\\foo\\bar\\baz" + @py.test.mark.skipif("sys.platform != 'win32'") def test_rename(self): os = self.posix fname = self.path2 + 'rename.txt' diff --git a/pypy/module/unicodedata/test/test_hyp.py b/pypy/module/unicodedata/test/test_hyp.py --- a/pypy/module/unicodedata/test/test_hyp.py +++ b/pypy/module/unicodedata/test/test_hyp.py @@ -1,6 +1,7 @@ +import sys import pytest try: - from hypothesis import given, strategies as st, example, settings + from hypothesis import given, strategies as st, example, settings, assume except ImportError: pytest.skip("hypothesis required") @@ -40,9 +41,14 @@ @pytest.mark.parametrize('NF1, NF2, NF3', compositions) @example(s=u'---\uafb8\u11a7---') # issue 2289 - at example(s=u'\ufacf') @settings(max_examples=1000) @given(s=st.text()) def test_composition(s, space, NF1, NF2, NF3): + # 'chr(0xfacf) normalizes to chr(0x2284a), which is too big') + assume(not (s == u'\ufacf' and sys.maxunicode == 65535)) norm1, norm2, norm3 = [make_normalization(space, form) for form in [NF1, NF2, NF3]] assert norm2(norm1(s)) == norm3(s) + +if sys.maxunicode != 65535: + # conditionally generate the example via an unwrapped decorator + test_composition = example(s=u'\ufacf')(test_composition) diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py --- a/pypy/objspace/std/listobject.py +++ b/pypy/objspace/std/listobject.py @@ -15,6 +15,7 @@ from rpython.rlib.listsort import make_timsort_class from rpython.rlib.objectmodel import ( import_from_mixin, instantiate, newlist_hint, resizelist_hint, specialize) +from rpython.rlib.rarithmetic import ovfcheck from rpython.rlib import longlong2float from rpython.tool.sourcetools import func_with_new_name @@ -848,7 +849,12 @@ """Extend w_list from a generic iterable""" length_hint = self.space.length_hint(w_iterable, 0) if length_hint: - w_list._resize_hint(w_list.length() + length_hint) + try: + newsize_hint = ovfcheck(w_list.length() + length_hint) + except OverflowError: + pass + else: + w_list._resize_hint(newsize_hint) extended = _do_extend_from_iterable(self.space, w_list, w_iterable) diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py --- a/pypy/objspace/std/test/test_listobject.py +++ b/pypy/objspace/std/test/test_listobject.py @@ -618,6 +618,18 @@ assert l == [1.2, 2.3, 3.4, 4.5] assert l is l0 + def test_extend_iterable_length_hint_overflow(self): + import sys + class CustomIterable(object): + def __iter__(self): + if False: + yield + def __length_hint__(self): + return sys.maxsize + a = [1, 2, 3, 4] + a.extend(CustomIterable()) + assert a == [1, 2, 3, 4] + def test_sort(self): l = l0 = [1, 5, 3, 0] l.sort() diff --git a/pypy/tool/pytest/appsupport.py b/pypy/tool/pytest/appsupport.py --- a/pypy/tool/pytest/appsupport.py +++ b/pypy/tool/pytest/appsupport.py @@ -198,7 +198,7 @@ w_BuiltinAssertionError = space.getitem(space.builtin.w_dict, space.wrap('AssertionError')) w_metaclass = space.type(w_BuiltinAssertionError) - w_init = space.wrap(gateway.interp2app_temp(my_init)) + w_init = space.wrap(gateway.interp2app(my_init)) w_dict = space.getattr(w_BuiltinAssertionError, space.wrap('__dict__')) w_dict = space.call_method(w_dict, 'copy') # fixup __module__, since the new type will be is_heaptype() == True diff --git a/pypy/tool/pytest/apptest2.py b/pypy/tool/pytest/apptest2.py --- a/pypy/tool/pytest/apptest2.py +++ b/pypy/tool/pytest/apptest2.py @@ -23,20 +23,21 @@ os.path.join(pypydir, 'tool', 'pytest', 'ast-rewriter')) w_source = space.newtext(source) fname = str(self.fspath) + w_name = space.newtext(str(self.fspath.purebasename)) w_fname = space.newtext(fname) if self.rewrite_asserts: - w_mod = space.appexec([w_rootdir, w_source, w_fname], - """(rootdir, source, fname): + w_mod = space.appexec([w_rootdir, w_source, w_fname, w_name], + """(rootdir, source, fname, name): import sys sys.path.insert(0, rootdir) from ast_rewrite import rewrite_asserts, create_module co = rewrite_asserts(source, fname) - mod = create_module(fname, co) + mod = create_module(name, co) return mod """) else: - w_mod = create_module(space, w_fname, fname, source) + w_mod = create_module(space, w_name, fname, source) mod_dict = w_mod.getdict(space).unwrap(space) items = [] for name, w_obj in mod_dict.items(): diff --git a/pypy/tool/pytest/ast-rewriter/ast_rewrite.py b/pypy/tool/pytest/ast-rewriter/ast_rewrite.py --- a/pypy/tool/pytest/ast-rewriter/ast_rewrite.py +++ b/pypy/tool/pytest/ast-rewriter/ast_rewrite.py @@ -3,14 +3,21 @@ import ast import itertools import marshal -import struct import sys -from ast_util import assertrepr_compare, format_explanation as _format_explanation +from ast_util import callbinrepr, format_explanation as _format_explanation # pytest caches rewritten pycs in __pycache__. -PYTEST_TAG = sys.implementation.cache_tag + "-PYTEST" +if hasattr(sys, "pypy_version_info"): + impl = "pypy" +elif sys.platform == "java": + impl = "jython" +else: + impl = "cpython" +ver = sys.version_info +PYTEST_TAG = "%s-%s%s-PYTEST" % (impl, ver[0], ver[1]) +del ver, impl PYC_EXT = ".py" + (__debug__ and "c" or "o") PYC_TAIL = "." + PYTEST_TAG + PYC_EXT @@ -29,6 +36,7 @@ # import. However, there's little reason deviate, and I hope # sometime to be able to use imp.load_compiled to load them. (See # the comment in load_module above.) + import struct try: fp = open(pyc, "wb") except IOError: @@ -91,6 +99,7 @@ Return rewritten code if successful or None if not. """ + import struct try: fp = open(pyc, "rb") except IOError: @@ -161,7 +170,7 @@ done = True if done: break - custom = assertrepr_compare(ops[i], each_obj[i], each_obj[i + 1]) + custom = callbinrepr(ops[i], each_obj[i], each_obj[i + 1]) if custom is not None: return custom return expl diff --git a/pypy/tool/pytest/ast-rewriter/ast_util.py b/pypy/tool/pytest/ast-rewriter/ast_util.py --- a/pypy/tool/pytest/ast-rewriter/ast_util.py +++ b/pypy/tool/pytest/ast-rewriter/ast_util.py @@ -90,6 +90,13 @@ s = s[:maxsize] return s +def callbinrepr(op, left, right): + new_expl = assertrepr_compare(op, left, right) + new_expl = [line.replace("\n", "\\n") for line in new_expl] + res = "\n~".join(new_expl) + res = res.replace("%", "%%") + return res + def assertrepr_compare(op, left, right, verbose=False): """Return specialised explanations for some operators/operands""" diff --git a/rpython/annotator/signature.py b/rpython/annotator/signature.py --- a/rpython/annotator/signature.py +++ b/rpython/annotator/signature.py @@ -14,16 +14,16 @@ def _annotation_key(t): from rpython.rtyper import extregistry - if type(t) is list: + if isinstance(t, list): assert len(t) == 1 return ('list', _annotation_key(t[0])) - elif type(t) is dict: + elif isinstance(t, dict): assert len(t.keys()) == 1 return ('dict', _annotation_key(t.items()[0])) elif isinstance(t, tuple): return tuple([_annotation_key(i) for i in t]) elif extregistry.is_registered(t): - # XXX should it really be always different? + # XXX do we want to do something in this case? return t return t @@ -38,24 +38,36 @@ return t return _compute_annotation(t, bookkeeper) + +def _validate_annotation_size(t): + try: + _ = iter(t) + except TypeError: # if it's not an iterable, just return + return t # (size does not matter) + if isinstance(t, tuple): # we accept tuples with any length, because + return t # their in-memory representation is predictable + if len(t) > 1: + raise TypeError("Cannot specify multiple types in a %s (try using tuple)", type(t)) + + def _compute_annotation(t, bookkeeper=None): from rpython.rtyper.lltypesystem import lltype from rpython.rtyper.llannotation import lltype_to_annotation + _validate_annotation_size(t) if isinstance(t, SomeObject): return t elif isinstance(t, lltype.LowLevelType): return lltype_to_annotation(t) elif isinstance(t, list): - assert len(t) == 1, "We do not support type joining in list" - listdef = ListDef(bookkeeper, annotation(t[0]), mutated=True, resized=True) - return SomeList(listdef) + return SomeList( + ListDef(bookkeeper, annotation(t[0]), + mutated=True, resized=True)) elif isinstance(t, tuple): return SomeTuple(tuple([annotation(i) for i in t])) elif isinstance(t, dict): - assert len(t) == 1, "We do not support type joining in dict" - result = SomeDict(DictDef(bookkeeper, annotation(t.keys()[0]), - annotation(t.values()[0]))) - return result + return SomeDict( + DictDef(bookkeeper, + annotation(t.keys()[0]), annotation(t.values()[0]))) elif type(t) is types.NoneType: return s_None elif extregistry.is_registered(t): @@ -84,15 +96,14 @@ elif t is types.NoneType: return s_None elif bookkeeper and extregistry.is_registered_type(t): - entry = extregistry.lookup_type(t) - return entry.compute_annotation_bk(bookkeeper) + return (extregistry.lookup_type(t) + .compute_annotation_bk(bookkeeper)) elif t is type: return SomeType() elif bookkeeper and not hasattr(t, '_freeze_'): - classdef = bookkeeper.getuniqueclassdef(t) - return SomeInstance(classdef) + return SomeInstance(bookkeeper.getuniqueclassdef(t)) else: - raise AssertionError("annotationoftype(%r)" % (t,)) + raise TypeError("Annotation of type %r not supported" % (t,)) class Sig(object): diff --git a/rpython/doc/rpython.rst b/rpython/doc/rpython.rst --- a/rpython/doc/rpython.rst +++ b/rpython/doc/rpython.rst @@ -259,6 +259,26 @@ intmask(). +Type Enforcing and Checking +--------------------------- + +RPython provides a helper decorator to force RPython-level types on function +arguments. The decorator, called ``enforceargs()``, accepts as parameters the +types expected to match the arguments of the function. + +Functions decorated with ``enforceargs()`` have their function signature +analyzed and their RPython-level type inferred at import time (for further +details about the flavor of translation performed in RPython, see the +`Annotation pass documentation`_). Encountering types not supported by RPython +will raise a ``TypeError``. + +``enforceargs()`` by default also performs type checking of parameter types +each time the function is invoked. To disable this behavior, it's possible to +pass the ``typecheck=False`` parameter to the decorator. + +.. _Annotation pass documentation: http://rpython.readthedocs.io/en/latest/translation.html#annotator + + Exception rules --------------- diff --git a/rpython/doc/translation.rst b/rpython/doc/translation.rst --- a/rpython/doc/translation.rst +++ b/rpython/doc/translation.rst @@ -48,7 +48,7 @@ be present in memory as a form that is "static enough" in the sense of :doc:`RPython `. -2. The Annotator_ performs a global analysis starting from an specified +2. The Annotator_ performs a global analysis starting from a specified entry point to deduce type and other information about what each variable can contain at run-time, :ref:`building flow graphs ` as it encounters them. diff --git a/rpython/jit/metainterp/optimizeopt/bridgeopt.py b/rpython/jit/metainterp/optimizeopt/bridgeopt.py --- a/rpython/jit/metainterp/optimizeopt/bridgeopt.py +++ b/rpython/jit/metainterp/optimizeopt/bridgeopt.py @@ -17,11 +17,17 @@ # # ( ) length times, if getfield(box1, descr) == box2 # both boxes should be in the liveboxes +# (or constants) # # # ( ) length times, if getarrayitem_gc(box1, index, descr) == box2 # both boxes should be in the liveboxes +# (or constants) # +# ---- call_loopinvariant knowledge +# +# ( ) length times, if call_loopinvariant(const) == box2 +# box2 should be in liveboxes # ---- @@ -55,11 +61,11 @@ return box def serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, liveboxes_from_env, memo): + from rpython.jit.metainterp.history import ConstInt available_boxes = {} for box in liveboxes: if box is not None and box in liveboxes_from_env: available_boxes[box] = None - metainterp_sd = optimizer.metainterp_sd # class knowledge is stored as bits, true meaning the class is known, false # means unknown. on deserializing we look at the bits, and read the runtime @@ -106,7 +112,19 @@ numb_state.append_int(0) numb_state.append_int(0) + if optimizer.optrewrite: + tuples_loopinvariant = optimizer.optrewrite.serialize_optrewrite( + available_boxes) + numb_state.append_int(len(tuples_loopinvariant)) + for constarg0, box in tuples_loopinvariant: + numb_state.append_short( + tag_box(ConstInt(constarg0), liveboxes_from_env, memo)) + numb_state.append_short(tag_box(box, liveboxes_from_env, memo)) + else: + numb_state.append_int(0) + def deserialize_optimizer_knowledge(optimizer, resumestorage, frontend_boxes, liveboxes): + from rpython.jit.metainterp.history import ConstInt reader = resumecode.Reader(resumestorage.rd_numb) assert len(frontend_boxes) == len(liveboxes) metainterp_sd = optimizer.metainterp_sd @@ -131,8 +149,6 @@ optimizer.make_constant_class(box, cls) # heap knowledge - if not optimizer.optheap: - return length = reader.next_item() result_struct = [] for i in range(length): @@ -154,4 +170,19 @@ tagged = reader.next_item() box2 = decode_box(resumestorage, tagged, liveboxes, metainterp_sd.cpu) result_array.append((box1, index, descr, box2)) - optimizer.optheap.deserialize_optheap(result_struct, result_array) + if optimizer.optheap: + optimizer.optheap.deserialize_optheap(result_struct, result_array) + + # call_loopinvariant knowledge + length = reader.next_item() + result_loopinvariant = [] + for i in range(length): + tagged1 = reader.next_item() + const = decode_box(resumestorage, tagged1, liveboxes, metainterp_sd.cpu) + assert isinstance(const, ConstInt) + i = const.getint() + tagged2 = reader.next_item() + box = decode_box(resumestorage, tagged2, liveboxes, metainterp_sd.cpu) + result_loopinvariant.append((i, box)) + if optimizer.optrewrite: + optimizer.optrewrite.deserialize_optrewrite(result_loopinvariant) diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -877,6 +877,18 @@ optimize_SAME_AS_R = optimize_SAME_AS_I optimize_SAME_AS_F = optimize_SAME_AS_I + def serialize_optrewrite(self, available_boxes): + res = [] + for i, box in self.loop_invariant_results.iteritems(): + box = self.get_box_replacement(box) + if box in available_boxes: + res.append((i, box)) + return res + + def deserialize_optrewrite(self, tups): + for i, box in tups: + self.loop_invariant_results[i] = box + dispatch_opt = make_dispatcher_method(OptRewrite, 'optimize_', default=OptRewrite.emit) optimize_guards = _findall(OptRewrite, 'optimize_', 'GUARD') diff --git a/rpython/jit/metainterp/test/test_bridgeopt.py b/rpython/jit/metainterp/test/test_bridgeopt.py --- a/rpython/jit/metainterp/test/test_bridgeopt.py +++ b/rpython/jit/metainterp/test/test_bridgeopt.py @@ -1,6 +1,9 @@ # tests that check that information is fed from the optimizer into the bridges +import pytest + import math + from rpython.rlib import jit from rpython.jit.metainterp.test.support import LLJitMixin from rpython.jit.metainterp.optimizeopt.bridgeopt import serialize_optimizer_knowledge @@ -27,6 +30,7 @@ class FakeOptimizer(object): metainterp_sd = None optheap = None + optrewrite = None def __init__(self, dct={}, cpu=None): self.dct = dct @@ -61,7 +65,8 @@ serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, {}, None) - assert unpack_numbering(numb_state.create_numbering()) == [1, 0b010000, 0, 0] + assert unpack_numbering(numb_state.create_numbering()) == [ + 1, 0b010000, 0, 0, 0] rbox1 = InputArgRef() rbox2 = InputArgRef() @@ -100,7 +105,7 @@ serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, {}, None) - assert len(numb_state.create_numbering().code) == 3 + math.ceil(len(refboxes) / 6.0) + assert len(numb_state.create_numbering().code) == 4 + math.ceil(len(refboxes) / 6.0) dct = {box: cls for box, known_class in boxes_known_classes @@ -321,3 +326,74 @@ self.check_trace_count(3) self.check_resops(guard_value=1) self.check_resops(getarrayitem_gc_i=5) + + def test_bridge_call_loopinvariant(self): + class A(object): + pass + class B(object): + pass + + aholder = B() + aholder.a = A() + + @jit.loop_invariant + def get(): + return aholder.a + + myjitdriver = jit.JitDriver(greens=[], reds=['y', 'res', 'n']) + def f(x, y, n): + if x == 10001121: + aholder.a = A() + if x: + get().x = 1 + else: + get().x = 2 + res = 0 + while y > 0: + myjitdriver.jit_merge_point(y=y, n=n, res=res) + a = get() + a = get() + res += a.x + if y > n: + res += 1 + res += get().x + a.x + y -= 1 + return res + res = self.meta_interp(f, [6, 32, 16]) + self.check_trace_count(3) + self.check_resops(call_r=1) + + @pytest.mark.xfail() + def test_bridge_call_loopinvariant_2(self): + class A(object): + pass + class B(object): + pass + + aholder = B() + aholder.a = A() + + @jit.loop_invariant + def get(): + return aholder.a + + myjitdriver = jit.JitDriver(greens=[], reds=['y', 'res', 'n']) + def f(x, y, n): + if x == 10001121: + aholder.a = A() + if x: + get().x = 1 + else: + get().x = 2 + res = 0 + while y > 0: + myjitdriver.jit_merge_point(y=y, n=n, res=res) + if y > n: + res += get().x + res += 1 + res += get().x + y -= 1 + return res + res = self.meta_interp(f, [6, 32, 16]) + self.check_trace_count(3) + self.check_resops(call_r=1) diff --git a/rpython/jit/metainterp/test/test_resume.py b/rpython/jit/metainterp/test/test_resume.py --- a/rpython/jit/metainterp/test/test_resume.py +++ b/rpython/jit/metainterp/test/test_resume.py @@ -40,7 +40,7 @@ class FakeOptimizer(object): metainterp_sd = None - optheap = None + optheap = optrewrite = None def __init__(self, trace=None): self.trace = trace diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -1592,8 +1592,7 @@ index = self.get_finalizer_queue_index(hop) c_index = rmodel.inputconst(lltype.Signed, index) v_ptr = hop.spaceop.args[1] - v_ptr = hop.genop("cast_opaque_ptr", [v_ptr], - resulttype=llmemory.GCREF) + assert v_ptr.concretetype == llmemory.GCREF hop.genop("direct_call", [self.register_finalizer_ptr, self.c_const_gc, c_index, v_ptr]) diff --git a/rpython/memory/gcwrapper.py b/rpython/memory/gcwrapper.py --- a/rpython/memory/gcwrapper.py +++ b/rpython/memory/gcwrapper.py @@ -235,11 +235,11 @@ obj = deque.popleft() else: obj = llmemory.NULL - return llmemory.cast_adr_to_ptr(obj, rclass.OBJECTPTR) + return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF) def gc_fq_register(self, fq_tag, ptr): index = self.get_finalizer_queue_index(fq_tag) - ptr = lltype.cast_opaque_ptr(llmemory.GCREF, ptr) + assert lltype.typeOf(ptr) == llmemory.GCREF self.gc.register_finalizer(index, ptr) # ____________________________________________________________ diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py --- a/rpython/rlib/objectmodel.py +++ b/rpython/rlib/objectmodel.py @@ -120,7 +120,7 @@ """ Decorate a function with forcing of RPython-level types on arguments. None means no enforcing. - When not translated, the type of the actual arguments are checked against + When not translated, the type of the actual arguments is checked against the enforced types every time the function is called. You can disable the typechecking by passing ``typecheck=False`` to @enforceargs. """ @@ -147,8 +147,7 @@ # they are already homogeneous, so we only check the first # item. The case of empty list/dict is handled inside typecheck() if isinstance(arg, list): - item = arg[0] - return [get_type_descr_of_argument(item)] + return [get_type_descr_of_argument(arg[0])] elif isinstance(arg, dict): key, value = next(arg.iteritems()) return {get_type_descr_of_argument(key): get_type_descr_of_argument(value)} diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -378,13 +378,12 @@ class FinalizerQueue(object): """A finalizer queue. See pypy/doc/discussion/finalizer-order.rst. - Note: only works with the framework GCs (like minimark). It is - ignored with Boehm or with refcounting (used by tests). """ # Must be subclassed, and the subclass needs these attributes: # # Class: # the class (or base class) of finalized objects + # --or-- None to handle low-level GCREFs directly # # def finalizer_trigger(self): # called to notify that new items have been put in the queue @@ -397,11 +396,13 @@ def next_dead(self): if we_are_translated(): from rpython.rtyper.lltypesystem.lloperation import llop - from rpython.rtyper.rclass import OBJECTPTR - from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance + from rpython.rtyper.lltypesystem.llmemory import GCREF + from rpython.rtyper.annlowlevel import cast_gcref_to_instance tag = FinalizerQueue._get_tag(self) - ptr = llop.gc_fq_next_dead(OBJECTPTR, tag) - return cast_base_ptr_to_instance(self.Class, ptr) + ptr = llop.gc_fq_next_dead(GCREF, tag) + if self.Class is not None: + ptr = cast_gcref_to_instance(self.Class, ptr) + return ptr try: return self._queue.popleft() except (AttributeError, IndexError): @@ -410,14 +411,18 @@ @specialize.arg(0) @jit.dont_look_inside def register_finalizer(self, obj): - assert isinstance(obj, self.Class) + from rpython.rtyper.lltypesystem.llmemory import GCREF + if self.Class is None: + assert lltype.typeOf(obj) == GCREF + else: + assert isinstance(obj, self.Class) if we_are_translated(): from rpython.rtyper.lltypesystem.lloperation import llop - from rpython.rtyper.rclass import OBJECTPTR - from rpython.rtyper.annlowlevel import cast_instance_to_base_ptr + from rpython.rtyper.annlowlevel import cast_instance_to_gcref tag = FinalizerQueue._get_tag(self) - ptr = cast_instance_to_base_ptr(obj) - llop.gc_fq_register(lltype.Void, tag, ptr) + if self.Class is not None: + obj = cast_instance_to_gcref(obj) + llop.gc_fq_register(lltype.Void, tag, obj) return else: self._untranslated_register_finalizer(obj) diff --git a/rpython/rlib/rstruct/nativefmttable.py b/rpython/rlib/rstruct/nativefmttable.py --- a/rpython/rlib/rstruct/nativefmttable.py +++ b/rpython/rlib/rstruct/nativefmttable.py @@ -130,6 +130,13 @@ sizeof_double = native_fmttable['d']['size'] sizeof_float = native_fmttable['f']['size'] +# Copy CPython's behavior of using short's size and alignment for half-floats. +native_fmttable['e'] = {'size': native_fmttable['h']['size'], + 'alignment': native_fmttable['h']['alignment'], + 'pack': std.pack_halffloat, + 'unpack': std.unpack_halffloat, + } + # ____________________________________________________________ # # A PyPy extension: accepts the 'u' format character in native mode, diff --git a/rpython/rlib/rstruct/test/test_pack.py b/rpython/rlib/rstruct/test/test_pack.py --- a/rpython/rlib/rstruct/test/test_pack.py +++ b/rpython/rlib/rstruct/test/test_pack.py @@ -139,9 +139,6 @@ self.check('d', 123.456789) def test_pack_halffloat(self): - if self.fmttable is nativefmttable.native_fmttable: - # Host Python cannot handle half floats. - return size = 2 wbuf = MutableStringBuffer(size) self.mypack_into('e', wbuf, 6.5e+04) diff --git a/rpython/rlib/test/test_rgc.py b/rpython/rlib/test/test_rgc.py --- a/rpython/rlib/test/test_rgc.py +++ b/rpython/rlib/test/test_rgc.py @@ -599,3 +599,94 @@ e = py.test.raises(TyperError, gengraph, f, []) assert str(e.value).startswith('the RPython-level __del__() method in') + + def test_translated_boehm(self): + self._test_translated(use_gc="boehm", llcase=False) + + def test_translated_boehm_ll(self): + self._test_translated(use_gc="boehm", llcase=True) + + def test_translated_incminimark(self): + self._test_translated(use_gc="incminimark", llcase=False) + + def test_translated_incminimark_ll(self): + self._test_translated(use_gc="incminimark", llcase=True) + + def _test_translated(self, use_gc, llcase): + import subprocess + from rpython.rlib import objectmodel + from rpython.translator.interactive import Translation + # + class Seen: + count = 0 + class MySimpleFQ(rgc.FinalizerQueue): + if not llcase: + Class = T_Root + else: + Class = None + def finalizer_trigger(self): + seen.count += 1 + seen = Seen() + fq = MySimpleFQ() + if not llcase: + EMPTY = None + llbuilder = T_Int + else: + from rpython.rtyper.annlowlevel import llstr + EMPTY = lltype.nullptr(llmemory.GCREF.TO) + def llbuilder(n): + return lltype.cast_opaque_ptr(llmemory.GCREF, llstr(str(n))) + + def subfunc(): + w0 = llbuilder(40); fq.register_finalizer(w0) + w1 = llbuilder(41); fq.register_finalizer(w1) + w2 = llbuilder(42); fq.register_finalizer(w2) + w3 = llbuilder(43); fq.register_finalizer(w3) + w4 = llbuilder(44); fq.register_finalizer(w4) + w5 = llbuilder(45); fq.register_finalizer(w5) + w6 = llbuilder(46); fq.register_finalizer(w6) + w7 = llbuilder(47); fq.register_finalizer(w7) + w8 = llbuilder(48); fq.register_finalizer(w8) + w9 = llbuilder(49); fq.register_finalizer(w9) + gc.collect() + assert seen.count == 0 + assert fq.next_dead() is EMPTY + objectmodel.keepalive_until_here(w0) + objectmodel.keepalive_until_here(w1) + objectmodel.keepalive_until_here(w2) + objectmodel.keepalive_until_here(w3) + objectmodel.keepalive_until_here(w4) + objectmodel.keepalive_until_here(w5) + objectmodel.keepalive_until_here(w6) + objectmodel.keepalive_until_here(w7) + objectmodel.keepalive_until_here(w8) + objectmodel.keepalive_until_here(w9) + + def main(argv): + assert fq.next_dead() is EMPTY + subfunc() + gc.collect(); gc.collect(); gc.collect() + assert seen.count > 0 + n = fq.next_dead() + while True: + if not llcase: + assert type(n) is T_Int and 40 <= n.x <= 49 + else: + from rpython.rtyper.lltypesystem.rstr import STR + assert lltype.typeOf(n) is llmemory.GCREF + p = lltype.cast_opaque_ptr(lltype.Ptr(STR), n) + assert len(p.chars) == 2 + assert p.chars[0] == "4" + assert "0" <= p.chars[1] <= "9" + n = fq.next_dead() + if n is EMPTY: + break + print "OK!" + return 0 + # + t = Translation(main, gc=use_gc) + t.disable(['backendopt']) + t.set_backend_extra_options(c_debug_defines=True) + exename = t.compile() + data = subprocess.check_output([str(exename), '.', '.', '.']) + assert data.strip().endswith('OK!') From pypy.commits at gmail.com Sun Apr 15 13:01:48 2018 From: pypy.commits at gmail.com (rlamy) Date: Sun, 15 Apr 2018 10:01:48 -0700 (PDT) Subject: [pypy-commit] pypy apptest-file: hg merge default Message-ID: <5ad3857c.09e61c0a.dd7ca.576b@mx.google.com> Author: Ronan Lamy Branch: apptest-file Changeset: r94342:88315b215135 Date: 2018-04-15 17:55 +0100 http://bitbucket.org/pypy/pypy/changeset/88315b215135/ Log: hg merge default diff too long, truncating to 2000 out of 8599 lines diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -6,36 +6,36 @@ Except when otherwise stated (look for LICENSE files in directories or information at the beginning of each file) all software and documentation in the 'rpython', 'pypy', 'ctype_configure', 'dotviewer', 'demo', 'lib_pypy', -'py', and '_pytest' directories is licensed as follows: +'py', and '_pytest' directories is licensed as follows: The MIT License - Permission is hereby granted, free of charge, to any person - obtaining a copy of this software and associated documentation - files (the "Software"), to deal in the Software without - restriction, including without limitation the rights to use, - copy, modify, merge, publish, distribute, sublicense, and/or - sell copies of the Software, and to permit persons to whom the + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation + files (the "Software"), to deal in the Software without + restriction, including without limitation the rights to use, + copy, modify, merge, publish, distribute, sublicense, and/or + sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included + The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. PyPy Copyright holders 2003-2018 ------------------------------------ +-------------------------------- Except when otherwise stated (look for LICENSE files or information at the beginning of each file) the files in the 'pypy' directory are each -copyrighted by one or more of the following people and organizations: +copyrighted by one or more of the following people and organizations: Armin Rigo Maciej Fijalkowski @@ -89,13 +89,13 @@ Niko Matsakis Alexander Hesse Ludovic Aubry + stian Jacob Hallen Jason Creighton Mark Young Alex Martelli Spenser Bauman Michal Bendowski - stian Jan de Mooij Tyler Wade Vincent Legoll @@ -123,10 +123,10 @@ Wenzhu Man Konstantin Lopuhin John Witulski + Jeremy Thurgood Greg Price Ivan Sichmann Freitas Dario Bertini - Jeremy Thurgood Mark Pearse Simon Cross Tobias Pape @@ -145,18 +145,19 @@ Adrian Kuhn tav Georg Brandl + Joannah Nanjekye Bert Freudenberg Stian Andreassen Wanja Saatkamp Mike Blume - Joannah Nanjekye Gerald Klix Oscar Nierstrasz Rami Chowdhury Stefan H. Muller + Dodan Mihai Tim Felgentreff Eugene Oden - Dodan Mihai + Colin Valliant Jeff Terrace Henry Mason Vasily Kuznetsov @@ -225,12 +226,14 @@ Vaibhav Sood Reuben Cummings Attila Gobi + Floris Bruynooghe Christopher Pope Tristan Arthur Christian Tismer Dan Stromberg Carl Meyer Florin Papa + Arianna Avanzini Jens-Uwe Mager Valentina Mukhamedzhanova Stefano Parmesan @@ -250,9 +253,11 @@ Alejandro J. Cura Vladimir Kryachko Gabriel + Thomas Hisch Mark Williams Kunal Grover Nathan Taylor + Barry Hart Travis Francis Athougies Yasir Suhail Sergey Kishchenko @@ -260,6 +265,7 @@ Lutz Paelike Ian Foote Philipp Rustemeuer + Logan Chien Catalin Gabriel Manciu Jacob Oscarson Ryan Gonzalez @@ -295,7 +301,6 @@ Akira Li Gustavo Niemeyer Rafał Gałczyński - Logan Chien Lucas Stadler roberto at goyle Matt Bogosian @@ -308,6 +313,7 @@ Anna Katrina Dominguez Kim Jin Su Amber Brown + Miro Hrončok Anthony Sottile Nate Bragg Ben Darnell @@ -315,7 +321,6 @@ Godefroid Chappelle Julian Berman Michael Hudson-Doyle - Floris Bruynooghe Stephan Busemann Dan Colish timo @@ -357,6 +362,7 @@ Michael Chermside Anna Ravencroft remarkablerocket + Pauli Virtanen Petre Vijiac Berker Peksag Christian Muirhead @@ -381,6 +387,7 @@ Graham Markall Dan Loewenherz werat + Andrew Stepanov Niclas Olofsson Chris Pressey Tobias Diaz @@ -395,14 +402,14 @@ m at funkyhat.org Stefan Marr - Heinrich-Heine University, Germany + Heinrich-Heine University, Germany Open End AB (formerly AB Strakt), Sweden - merlinux GmbH, Germany - tismerysoft GmbH, Germany - Logilab Paris, France - DFKI GmbH, Germany + merlinux GmbH, Germany + tismerysoft GmbH, Germany + Logilab Paris, France + DFKI GmbH, Germany Impara, Germany - Change Maker, Sweden + Change Maker, Sweden University of California Berkeley, USA Google Inc. King's College London @@ -410,14 +417,14 @@ The PyPy Logo as used by http://speed.pypy.org and others was created by Samuel Reis and is distributed on terms of Creative Commons Share Alike License. - -License for 'lib-python/2.7' -============================ + +License for 'lib-python/2.7, lib-python/3' +========================================== Except when otherwise stated (look for LICENSE files or copyright/license -information at the beginning of each file) the files in the 'lib-python/2.7' +information at the beginning of each file) the files in the 'lib-python' directory are all copyrighted by the Python Software Foundation and licensed -under the terms that you can find here: https://docs.python.org/2/license.html +under the terms that you can find here: https://docs.python.org/3/license.html License for 'pypy/module/unicodedata/' ====================================== @@ -441,9 +448,9 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/README.rst b/README.rst --- a/README.rst +++ b/README.rst @@ -4,42 +4,40 @@ Welcome to PyPy! -PyPy is both an implementation of the Python programming language, and -an extensive compiler framework for dynamic language implementations. -You can build self-contained Python implementations which execute -independently from CPython. +PyPy is an interperter that implements the Python programming language, based +on the RPython compiler framework for dynamic language implementations. -The home page is: +The home page for the interpreter is: http://pypy.org/ -If you want to help developing PyPy, this document might help you: +If you want to help developing PyPy, this documentation might help you: http://doc.pypy.org/ -It will also point you to the rest of the documentation which is generated -from files in the pypy/doc directory within the source repositories. Enjoy -and send us feedback! +More documentation about the RPython framework can be found here - the pypy-dev team + http://rpython.readthedocs.io +The source for the documentation is in the pypy/doc directory + +Using PyPy instead of CPython +============================= + +Please read the information at http://pypy.org to find the correct way to +download and use PyPy as an alternative to CPython. Building ======== -First switch to or download the correct branch. The basic choices are -``default`` for Python 2.7 and, for Python 3.X, the corresponding py3.X -branch (e.g. ``py3.5``). +Building PyPy is not the recommended way to obtain the PyPy alternative python +interpreter. It is time-consuming and requires significant computing resources. +More information can be found here -Build with: + http://doc.pypy.org/en/latest/build.html -.. code-block:: console +Enjoy and send us feedback! - $ rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py + the pypy-dev team -This ends up with a ``pypy-c`` or ``pypy3-c`` binary in the main pypy -directory. We suggest to use virtualenv with the resulting -pypy-c/pypy3-c as the interpreter; you can find more details about -various installation schemes here: - http://doc.pypy.org/en/latest/install.html diff --git a/lib-python/2.7/re.py b/lib-python/2.7/re.py --- a/lib-python/2.7/re.py +++ b/lib-python/2.7/re.py @@ -225,7 +225,7 @@ _pattern_type = type(sre_compile.compile("", 0)) -_MAXCACHE = 100 +_MAXCACHE = 1000 def _compile(*key): # internal: compile pattern diff --git a/lib-python/2.7/test/test_eof.py b/lib-python/2.7/test/test_eof.py --- a/lib-python/2.7/test/test_eof.py +++ b/lib-python/2.7/test/test_eof.py @@ -5,7 +5,7 @@ class EOFTestCase(unittest.TestCase): def test_EOFC(self): - expect = "EOL while scanning string literal (, line 1)" + expect = "end of line (EOL) while scanning string literal (, line 1)" try: eval("""'this is a test\ """) @@ -15,7 +15,7 @@ raise test_support.TestFailed def test_EOFS(self): - expect = ("EOF while scanning triple-quoted string literal " + expect = ("end of file (EOF) while scanning triple-quoted string literal " "(, line 1)") try: eval("""'''this is a test""") diff --git a/lib-python/2.7/test/test_generators.py b/lib-python/2.7/test/test_generators.py --- a/lib-python/2.7/test/test_generators.py +++ b/lib-python/2.7/test/test_generators.py @@ -398,7 +398,10 @@ 0 >>> type(i.gi_frame) ->>> i.gi_running = 42 + +PyPy prints "readonly attribute 'gi_running'" so ignore the exception detail + +>>> i.gi_running = 42 # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ... TypeError: readonly attribute diff --git a/lib-python/2.7/test/test_genexps.py b/lib-python/2.7/test/test_genexps.py --- a/lib-python/2.7/test/test_genexps.py +++ b/lib-python/2.7/test/test_genexps.py @@ -87,7 +87,7 @@ >>> dict(a = i for i in xrange(10)) Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: invalid syntax (expected ')') Verify that parenthesis are required when used as a keyword argument value diff --git a/lib-python/2.7/test/test_traceback.py b/lib-python/2.7/test/test_traceback.py --- a/lib-python/2.7/test/test_traceback.py +++ b/lib-python/2.7/test/test_traceback.py @@ -123,10 +123,7 @@ self.assertEqual(len(err), 4) self.assertEqual(err[1].strip(), "print(2)") self.assertIn("^", err[2]) - if check_impl_detail(): - self.assertEqual(err[1].find("p"), err[2].find("^")) - if check_impl_detail(pypy=True): - self.assertEqual(err[1].find("2)") + 1, err[2].find("^")) + self.assertEqual(err[1].find("p"), err[2].find("^")) def test_base_exception(self): # Test that exceptions derived from BaseException are formatted right diff --git a/lib-python/2.7/threading.py b/lib-python/2.7/threading.py --- a/lib-python/2.7/threading.py +++ b/lib-python/2.7/threading.py @@ -351,6 +351,21 @@ # forward-compatibility reasons we do the same. waiter.acquire() gotit = True + except AttributeError: + # someone patched the 'waiter' class, probably. + # Fall back to the standard CPython logic. + # See the CPython lib for the comments about it... + endtime = _time() + timeout + delay = 0.0005 # 500 us -> initial delay of 1 ms + while True: + gotit = waiter.acquire(0) + if gotit: + break + remaining = endtime - _time() + if remaining <= 0: + break + delay = min(delay * 2, remaining, .05) + _sleep(delay) else: gotit = waiter.acquire(False) if not gotit: diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -56,13 +56,13 @@ Niko Matsakis Alexander Hesse Ludovic Aubry + stian Jacob Hallen Jason Creighton Mark Young Alex Martelli Spenser Bauman Michal Bendowski - stian Jan de Mooij Tyler Wade Vincent Legoll @@ -90,10 +90,10 @@ Wenzhu Man Konstantin Lopuhin John Witulski + Jeremy Thurgood Greg Price Ivan Sichmann Freitas Dario Bertini - Jeremy Thurgood Mark Pearse Simon Cross Tobias Pape @@ -112,18 +112,19 @@ Adrian Kuhn tav Georg Brandl + Joannah Nanjekye Bert Freudenberg Stian Andreassen Wanja Saatkamp Mike Blume - Joannah Nanjekye Gerald Klix Oscar Nierstrasz Rami Chowdhury Stefan H. Muller + Dodan Mihai Tim Felgentreff Eugene Oden - Dodan Mihai + Colin Valliant Jeff Terrace Henry Mason Vasily Kuznetsov @@ -192,12 +193,14 @@ Vaibhav Sood Reuben Cummings Attila Gobi + Floris Bruynooghe Christopher Pope Tristan Arthur Christian Tismer Dan Stromberg Carl Meyer Florin Papa + Arianna Avanzini Jens-Uwe Mager Valentina Mukhamedzhanova Stefano Parmesan @@ -221,6 +224,7 @@ Mark Williams Kunal Grover Nathan Taylor + Barry Hart Travis Francis Athougies Yasir Suhail Sergey Kishchenko @@ -228,6 +232,7 @@ Lutz Paelike Ian Foote Philipp Rustemeuer + Logan Chien Catalin Gabriel Manciu Jacob Oscarson Ryan Gonzalez @@ -263,7 +268,6 @@ Akira Li Gustavo Niemeyer Rafał Gałczyński - Logan Chien Lucas Stadler roberto at goyle Matt Bogosian @@ -276,6 +280,7 @@ Anna Katrina Dominguez Kim Jin Su Amber Brown + Miro Hrončok Anthony Sottile Nate Bragg Ben Darnell @@ -283,7 +288,6 @@ Godefroid Chappelle Julian Berman Michael Hudson-Doyle - Floris Bruynooghe Stephan Busemann Dan Colish timo @@ -325,6 +329,7 @@ Michael Chermside Anna Ravencroft remarkablerocket + Pauli Virtanen Petre Vijiac Berker Peksag Christian Muirhead @@ -349,6 +354,7 @@ Graham Markall Dan Loewenherz werat + Andrew Stepanov Niclas Olofsson Chris Pressey Tobias Diaz diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -10,89 +10,6 @@ PyPy. -.. _extension-modules: - -Extension modules ------------------ - -List of extension modules that we support: - -* Supported as built-in modules (in :source:`pypy/module/`): - - __builtin__ - :doc:`__pypy__ <__pypy__-module>` - _ast - _codecs - _collections - :doc:`_continuation ` - :doc:`_ffi ` - _hashlib - _io - _locale - _lsprof - _md5 - :doc:`_minimal_curses ` - _multiprocessing - _random - :doc:`_rawffi ` - _sha - _socket - _sre - _ssl - _warnings - _weakref - _winreg - array - binascii - bz2 - cStringIO - cmath - `cpyext`_ - crypt - errno - exceptions - fcntl - gc - imp - itertools - marshal - math - mmap - operator - parser - posix - pyexpat - select - signal - struct - symbol - sys - termios - thread - time - token - unicodedata - zipimport - zlib - - When translated on Windows, a few Unix-only modules are skipped, - and the following module is built instead: - - _winreg - -* Supported by being rewritten in pure Python (possibly using ``cffi``): - see the :source:`lib_pypy/` directory. Examples of modules that we - support this way: ``ctypes``, ``cPickle``, ``cmath``, ``dbm``, ``datetime``... - Note that some modules are both in there and in the list above; - by default, the built-in module is used (but can be disabled - at translation time). - -The extension modules (i.e. modules written in C, in the standard CPython) -that are neither mentioned above nor in :source:`lib_pypy/` are not available in PyPy. -(You may have a chance to use them anyway with `cpyext`_.) - -.. _cpyext: http://morepypy.blogspot.com/2010/04/using-cpython-extension-modules-with.html - Differences related to garbage collection strategies ---------------------------------------------------- @@ -559,7 +476,96 @@ environment variable. CPython searches for ``vcvarsall.bat`` somewhere **above** that value. +* SyntaxError_ s try harder to give details about the cause of the failure, so + the error messages are not the same as in CPython + + +.. _extension-modules: + +Extension modules +----------------- + +List of extension modules that we support: + +* Supported as built-in modules (in :source:`pypy/module/`): + + __builtin__ + :doc:`__pypy__ <__pypy__-module>` + _ast + _codecs + _collections + :doc:`_continuation ` + :doc:`_ffi ` + _hashlib + _io + _locale + _lsprof + _md5 + :doc:`_minimal_curses ` + _multiprocessing + _random + :doc:`_rawffi ` + _sha + _socket + _sre + _ssl + _warnings + _weakref + _winreg + array + binascii + bz2 + cStringIO + cmath + `cpyext`_ + crypt + errno + exceptions + fcntl + gc + imp + itertools + marshal + math + mmap + operator + parser + posix + pyexpat + select + signal + struct + symbol + sys + termios + thread + time + token + unicodedata + zipimport + zlib + + When translated on Windows, a few Unix-only modules are skipped, + and the following module is built instead: + + _winreg + +* Supported by being rewritten in pure Python (possibly using ``cffi``): + see the :source:`lib_pypy/` directory. Examples of modules that we + support this way: ``ctypes``, ``cPickle``, ``cmath``, ``dbm``, ``datetime``... + Note that some modules are both in there and in the list above; + by default, the built-in module is used (but can be disabled + at translation time). + +The extension modules (i.e. modules written in C, in the standard CPython) +that are neither mentioned above nor in :source:`lib_pypy/` are not available in PyPy. +(You may have a chance to use them anyway with `cpyext`_.) + +.. _cpyext: http://morepypy.blogspot.com/2010/04/using-cpython-extension-modules-with.html + + .. _`is ignored in PyPy`: http://bugs.python.org/issue14621 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/ .. _`issue #2653`: https://bitbucket.org/pypy/pypy/issues/2653/ +.. _SyntaxError: https://morepypy.blogspot.co.il/2018/04/improving-syntaxerror-in-pypy.html diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst --- a/pypy/doc/how-to-release.rst +++ b/pypy/doc/how-to-release.rst @@ -40,6 +40,8 @@ sure things are ported back to the trunk and to the branch as necessary. +* Make sure the RPython builds on the buildbot pass with no failures + * Maybe bump the SOABI number in module/imp/importing. This has many implications, so make sure the PyPy community agrees to the change. diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-v6.0.0.rst release-v5.10.1.rst release-v5.10.0.rst release-v5.9.0.rst diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -7,6 +7,8 @@ .. toctree:: whatsnew-head.rst + whatsnew-pypy2-6.0.0.rst + whatsnew-pypy2-5.10.0.rst whatsnew-pypy2-5.10.0.rst whatsnew-pypy2-5.9.0.rst whatsnew-pypy2-5.8.0.rst diff --git a/pypy/doc/install.rst b/pypy/doc/install.rst --- a/pypy/doc/install.rst +++ b/pypy/doc/install.rst @@ -17,13 +17,18 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~ The quickest way to start using PyPy is to download a prebuilt binary for your -OS and architecture. You can either use the `most recent release`_ or one of -our `development nightly build`_. Please note that the nightly builds are not +OS and architecture. You may be able to use either use the +`most recent release`_ or one of our `development nightly build`_. These +builds depend on dynamically linked libraries that may not be available on your +OS. See the section about `Linux binaries` for more info and alternatives that +may work on your system. + +Please note that the nightly builds are not guaranteed to be as stable as official releases, use them at your own risk. .. _most recent release: http://pypy.org/download.html .. _development nightly build: http://buildbot.pypy.org/nightly/trunk/ - +.. _Linux binaries: http://pypy.org/download.html#linux-binaries-and-common-distributions Installing PyPy ~~~~~~~~~~~~~~~ @@ -69,9 +74,9 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~ It is often convenient to run pypy inside a virtualenv. To do this -you need a recent version of virtualenv -- 1.6.1 or greater. You can +you need a version of virtualenv -- 1.6.1 or greater. You can then install PyPy both from a precompiled tarball or from a mercurial -checkout:: +checkout after translation:: # from a tarball $ virtualenv -p /opt/pypy-xxx/bin/pypy my-pypy-env diff --git a/pypy/doc/release-v6.0.0.rst b/pypy/doc/release-v6.0.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-v6.0.0.rst @@ -0,0 +1,109 @@ +====================================== +PyPy2.7 and PyPy3.5 v6.0 dual release +====================================== + +The PyPy team is proud to release both PyPy2.7 v6.0 (an interpreter supporting +Python 2.7 syntax), and a PyPy3.5 v6.0 (an interpreter supporting Python +3.5 syntax). The two releases are both based on much the same codebase, thus +the dual release. + +This release is a feature release following our previous 5.10 incremental +release in late December 2017. Our C-API compatability layer ``cpyext`` is +now much faster (see the `blog post`_) as well as more complete. We have made +many other improvements in speed and CPython compatibility. Since the changes +affect the included python development header files, all c-extension modules must +be recompiled for this version. + +First-time python users are often stumped by silly typos and emissions when +getting started writing code. We have improved our parser to emit more friendly +`syntax errors`_, making PyPy not only faster but more friendly. + +The Windows PyPy3.5 release is still considered beta-quality. There are open +issues with unicode handling especially around system calls and c-extensions. + +The Matplotlib TkAgg backend now works with PyPy, as do pygame and pygobject_. + +As always, this release is 100% compatible with the previous one and fixed +several issues and bugs raised by the growing community of PyPy users. +We strongly recommend updating. + +We updated the cffi module included in PyPy to version 1.11.5 + +The utf8 branch that changes internal representation of unicode to utf8 did not +make it into the release, so there is still more goodness coming. We also +began working on a Python3.6 implementation, help is welcome. + +You can download the v6.0 releases here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project. If PyPy is not quite good enough for your needs, we are available for +direct consulting work. + +We would also like to thank our contributors and encourage new people to join +the project. PyPy has many layers and we need help with all of them: `PyPy`_ +and `RPython`_ documentation improvements, tweaking popular `modules`_ to run +on pypy, or general `help`_ with making RPython's JIT even better. + +.. _`PyPy`: index.html +.. _`RPython`: https://rpython.readthedocs.org +.. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly +.. _`help`: project-ideas.html +.. _`blog post`: https://morepypy.blogspot.it/2017/10/cape-of-good-hope-for-pypy-hello-from.html +.. _pygobject: https://lazka.github.io/posts/2018-04_pypy-pygobject/index.html +.. _`syntax errors`: https://morepypy.blogspot.com/2018/04/improving-syntaxerror-in-pypy.html + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7 and CPython 3.5. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +The PyPy release supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html + +Changelog +========= + +* Speed up C-API method calls, and make most Py*_Check calls C macros +* Speed up C-API slot method calls +* Enable TkAgg backend support for matplotlib +* support ``hastzinfo`` and ``tzinfo`` in the C-API ``PyDateTime*`` structures +* datetime.h is now more similar to CPython +* We now support ``PyUnicode_AsUTF{16,32}String``, ``_PyLong_AsByteArray``, + ``_PyLong_AsByteArrayO``, +* PyPy3.5 on Windows is compiled with the Microsoft Visual Compiler v14, like + CPython +* Fix performance of attribute lookup when more than 80 attributes are used +* Improve performance on passing built-in types to C-API C code +* Improve the performance of datetime and timedelta by skipping the consistency + checks of the datetime values (they are correct by construction) +* Improve handling of ``bigint`` s, including fixing ``int_divmod`` +* Improve reporting of GC statistics +* Accept unicode filenames in ``dbm.open()`` +* Improve RPython support for half-floats +* Added missing attributes to C-API ``instancemethod`` on pypy3 +* Store error state in thread-local storage for C-API. +* Fix JIT bugs exposed in the sre module +* Improve speed of Python parser, improve ParseError messages slightly +* Handle JIT hooks more efficiently + +We also refactored many parts of the JIT bridge optimizations, as well as cpyext +internals, and together with new contributors fixed issues, added new +documentation, and cleaned up the codebase. diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -1,60 +1,12 @@ -=========================== -What's new in PyPy2.7 5.10+ -=========================== +========================== +What's new in PyPy2.7 6.0+ +========================== -.. this is a revision shortly after release-pypy2.7-v5.10.0 -.. startrev: 6b024edd9d12 +.. this is a revision shortly after release-pypy-6.0.0 +.. startrev: f22145c34985 -.. branch: cpyext-avoid-roundtrip -Big refactoring of some cpyext code, which avoids a lot of nonsense when -calling C from Python and vice-versa: the result is a big speedup in -function/method calls, up to 6 times faster. +.. branch: issue2752 -.. branch: cpyext-datetime2 - -Support ``tzinfo`` field on C-API datetime objects, fixes latest pandas HEAD - - -.. branch: mapdict-size-limit - -Fix a corner case of mapdict: When an instance is used like a dict (using -``setattr`` and ``getattr``, or ``.__dict__``) and a lot of attributes are -added, then the performance using mapdict is linear in the number of -attributes. This is now fixed (by switching to a regular dict after 80 -attributes). - - -.. branch: cpyext-faster-arg-passing - -When using cpyext, improve the speed of passing certain objects from PyPy to C -code, most notably None, True, False, types, all instances of C-defined types. -Before, a dict lookup was needed every time such an object crossed over, now it -is just a field read. - - -.. branch: 2634_datetime_timedelta_performance - -Improve datetime + timedelta performance. - -.. branch: memory-accounting - -Improve way to describe memory - -.. branch: msvc14 - -Allow compilaiton with Visual Studio 2017 compiler suite on windows - -.. branch: refactor-slots - -Refactor cpyext slots. - - -.. branch: call-loopinvariant-into-bridges - -Speed up branchy code that does a lot of function inlining by saving one call -to read the TLS in most bridges. - -.. branch: rpython-sprint - -Refactor in rpython signatures +Fix a rare GC bug that was introduced more than one year ago, but was +not diagnosed before issue #2752. diff --git a/pypy/doc/whatsnew-pypy2-6.0.0.rst b/pypy/doc/whatsnew-pypy2-6.0.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-pypy2-6.0.0.rst @@ -0,0 +1,111 @@ +=========================== +What's new in PyPy2.7 5.10+ +=========================== + +.. this is a revision shortly after release-pypy2.7-v5.10.0 +.. startrev: 6b024edd9d12 + +.. branch: cpyext-avoid-roundtrip + +Big refactoring of some cpyext code, which avoids a lot of nonsense when +calling C from Python and vice-versa: the result is a big speedup in +function/method calls, up to 6 times faster. + +.. branch: cpyext-datetime2 + +Support ``tzinfo`` field on C-API datetime objects, fixes latest pandas HEAD + + +.. branch: mapdict-size-limit + +Fix a corner case of mapdict: When an instance is used like a dict (using +``setattr`` and ``getattr``, or ``.__dict__``) and a lot of attributes are +added, then the performance using mapdict is linear in the number of +attributes. This is now fixed (by switching to a regular dict after 80 +attributes). + + +.. branch: cpyext-faster-arg-passing + +When using cpyext, improve the speed of passing certain objects from PyPy to C +code, most notably None, True, False, types, all instances of C-defined types. +Before, a dict lookup was needed every time such an object crossed over, now it +is just a field read. + + +.. branch: 2634_datetime_timedelta_performance + +Improve datetime + timedelta performance. + +.. branch: memory-accounting + +Improve way to describe memory + +.. branch: msvc14 + +Allow compilaiton with Visual Studio 2017 compiler suite on windows + +.. branch: refactor-slots + +Refactor cpyext slots. + + +.. branch: call-loopinvariant-into-bridges + +Speed up branchy code that does a lot of function inlining by saving one call +to read the TLS in most bridges. + +.. branch: rpython-sprint + +Refactor in rpython signatures + +.. branch: cpyext-tls-operror2 + +Store error state thread-locally in executioncontext, fixes issue #2764 + +.. branch: cpyext-fast-typecheck + +Optimize `Py*_Check` for `Bool`, `Float`, `Set`. Also refactor and simplify +`W_PyCWrapperObject` which is used to call slots from the C-API, greatly +improving microbenchmarks in https://github.com/antocuni/cpyext-benchmarks + + +.. branch: fix-sre-problems + +Fix two (unrelated) JIT bugs manifesting in the re module: + +- green fields are broken and were thus disabled, plus their usage removed from + the _sre implementation + +- in rare "trace is too long" situations, the JIT could break behaviour + arbitrarily. + +.. branch: jit-hooks-can-be-disabled + +Be more efficient about JIT hooks. Make it possible for the frontend to declare +that jit hooks are currently not enabled at all. in that case, the list of ops +does not have to be created in the case of the on_abort hook (which is +expensive). + + +.. branch: pyparser-improvements + +Improve speed of Python parser, improve ParseError messages slightly. + +.. branch: ioctl-arg-size + +Work around possible bugs in upstream ioctl users, like CPython allocate at +least 1024 bytes for the arg in calls to ``ioctl(fd, request, arg)``. Fixes +issue #2776 + +.. branch: cpyext-subclass-setattr + +Fix for python-level classes that inherit from C-API types, previously the +`w_obj` was not necessarily preserved throughout the lifetime of the `pyobj` +which led to cases where instance attributes were lost. Fixes issue #2793 + + +.. branch: pyparser-improvements-2 + +Improve line offsets that are reported by SyntaxError. Improve error messages +for a few situations, including mismatched parenthesis. diff --git a/pypy/interpreter/pyparser/error.py b/pypy/interpreter/pyparser/error.py --- a/pypy/interpreter/pyparser/error.py +++ b/pypy/interpreter/pyparser/error.py @@ -6,6 +6,7 @@ lastlineno=0): self.msg = msg self.lineno = lineno + # NB: offset is a 1-based index! self.offset = offset self.text = text self.filename = filename diff --git a/pypy/interpreter/pyparser/metaparser.py b/pypy/interpreter/pyparser/metaparser.py --- a/pypy/interpreter/pyparser/metaparser.py +++ b/pypy/interpreter/pyparser/metaparser.py @@ -147,8 +147,10 @@ for label, next in state.arcs.iteritems(): arcs.append((self.make_label(gram, label), dfa.index(next))) states.append((arcs, state.is_final)) - gram.dfas.append((states, self.make_first(gram, name))) - assert len(gram.dfas) - 1 == gram.symbol_ids[name] - 256 + symbol_id = gram.symbol_ids[name] + dfa = parser.DFA(symbol_id, states, self.make_first(gram, name)) + gram.dfas.append(dfa) + assert len(gram.dfas) - 1 == symbol_id - 256 gram.start = gram.symbol_ids[self.start_symbol] return gram @@ -162,6 +164,13 @@ else: gram.labels.append(gram.symbol_ids[label]) gram.symbol_to_label[label] = label_index + first = self.first[label] + if len(first) == 1: + first, = first + if not first[0].isupper(): + first = first.strip("\"'") + assert label_index not in gram.token_to_error_string + gram.token_to_error_string[label_index] = first return label_index elif label.isupper(): token_index = gram.TOKENS[label] @@ -183,7 +192,7 @@ else: gram.labels.append(gram.KEYWORD_TOKEN) gram.keyword_ids[value] = label_index - return label_index + result = label_index else: try: token_index = gram.OPERATOR_MAP[value] @@ -194,7 +203,10 @@ else: gram.labels.append(token_index) gram.token_ids[token_index] = label_index - return label_index + result = label_index + assert result not in gram.token_to_error_string + gram.token_to_error_string[result] = value + return result def make_first(self, gram, name): original_firsts = self.first[name] diff --git a/pypy/interpreter/pyparser/parser.py b/pypy/interpreter/pyparser/parser.py --- a/pypy/interpreter/pyparser/parser.py +++ b/pypy/interpreter/pyparser/parser.py @@ -1,6 +1,7 @@ """ A CPython inspired RPython parser. """ +from rpython.rlib.objectmodel import not_rpython class Grammar(object): @@ -16,6 +17,7 @@ self.symbol_names = {} self.symbol_to_label = {} self.keyword_ids = {} + self.token_to_error_string = {} self.dfas = [] self.labels = [0] self.token_ids = {} @@ -41,6 +43,27 @@ pass return True +class DFA(object): + def __init__(self, symbol_id, states, first): + self.symbol_id = symbol_id + self.states = states + self.first = self._first_to_string(first) + + def could_match_token(self, label_index): + pos = label_index >> 3 + bit = 1 << (label_index & 0b111) + return bool(ord(self.first[label_index >> 3]) & bit) + + @staticmethod + @not_rpython + def _first_to_string(first): + l = sorted(first.keys()) + b = bytearray(32) + for label_index in l: + pos = label_index >> 3 + bit = 1 << (label_index & 0b111) + b[pos] |= bit + return str(b) class Node(object): @@ -127,14 +150,17 @@ class Nonterminal(AbstractNonterminal): __slots__ = ("_children", ) - def __init__(self, type, children): + def __init__(self, type, children=None): Node.__init__(self, type) + if children is None: + children = [] self._children = children def __repr__(self): return "Nonterminal(type=%s, children=%r)" % (self.type, self._children) def get_child(self, i): + assert self._children is not None return self._children[i] def num_children(self): @@ -168,25 +194,50 @@ class ParseError(Exception): def __init__(self, msg, token_type, value, lineno, column, line, - expected=-1): + expected=-1, expected_str=None): self.msg = msg self.token_type = token_type self.value = value self.lineno = lineno + # this is a 0-based index self.column = column self.line = line self.expected = expected + self.expected_str = expected_str def __str__(self): return "ParserError(%s, %r)" % (self.token_type, self.value) +class StackEntry(object): + def __init__(self, next, dfa, state): + self.next = next + self.dfa = dfa + self.state = state + self.node = None + + def push(self, dfa, state): + return StackEntry(self, dfa, state) + + def pop(self): + return self.next + + def node_append_child(self, child): + node = self.node + if node is None: + self.node = Nonterminal1(self.dfa.symbol_id, child) + elif isinstance(node, Nonterminal1): + newnode = self.node = Nonterminal( + self.dfa.symbol_id, [node._child, child]) + else: + self.node.append_child(child) + + class Parser(object): def __init__(self, grammar): self.grammar = grammar self.root = None - self.stack = None def prepare(self, start=-1): """Setup the parser for parsing. @@ -196,16 +247,15 @@ if start == -1: start = self.grammar.start self.root = None - current_node = Nonterminal(start, []) - self.stack = [] - self.stack.append((self.grammar.dfas[start - 256], 0, current_node)) + self.stack = StackEntry(None, self.grammar.dfas[start - 256], 0) def add_token(self, token_type, value, lineno, column, line): label_index = self.classify(token_type, value, lineno, column, line) sym_id = 0 # for the annotator while True: - dfa, state_index, node = self.stack[-1] - states, first = dfa + dfa = self.stack.dfa + state_index = self.stack.state + states = dfa.states arcs, is_accepting = states[state_index] for i, next_state in arcs: sym_id = self.grammar.labels[i] @@ -217,16 +267,17 @@ # the stack. while state[1] and not state[0]: self.pop() - if not self.stack: + if self.stack is None: # Parsing is done. return True - dfa, state_index, node = self.stack[-1] - state = dfa[0][state_index] + dfa = self.stack.dfa + state_index = self.stack.state + state = dfa.states[state_index] return False elif sym_id >= 256: sub_node_dfa = self.grammar.dfas[sym_id - 256] # Check if this token can start a child node. - if label_index in sub_node_dfa[1]: + if sub_node_dfa.could_match_token(label_index): self.push(sub_node_dfa, next_state, sym_id, lineno, column) break @@ -235,7 +286,7 @@ # state is accepting, it's invalid input. if is_accepting: self.pop() - if not self.stack: + if self.stack is None: raise ParseError("too much input", token_type, value, lineno, column, line) else: @@ -243,10 +294,13 @@ # error. if len(arcs) == 1: expected = sym_id + expected_str = self.grammar.token_to_error_string.get( + arcs[0][0], None) else: expected = -1 + expected_str = None raise ParseError("bad input", token_type, value, lineno, - column, line, expected) + column, line, expected, expected_str) def classify(self, token_type, value, lineno, column, line): """Find the label for a token.""" @@ -262,26 +316,22 @@ def shift(self, next_state, token_type, value, lineno, column): """Shift a non-terminal and prepare for the next state.""" - dfa, state, node = self.stack[-1] new_node = Terminal(token_type, value, lineno, column) - node.append_child(new_node) - self.stack[-1] = (dfa, next_state, node) + self.stack.node_append_child(new_node) + self.stack.state = next_state def push(self, next_dfa, next_state, node_type, lineno, column): """Push a terminal and adjust the current state.""" - dfa, state, node = self.stack[-1] - new_node = Nonterminal(node_type, []) - self.stack[-1] = (dfa, next_state, node) - self.stack.append((next_dfa, 0, new_node)) + self.stack.state = next_state + self.stack = self.stack.push(next_dfa, 0) def pop(self): """Pop an entry off the stack and make its node a child of the last.""" - dfa, state, node = self.stack.pop() + top = self.stack + self.stack = top.pop() + node = top.node + assert node is not None if self.stack: - # we are now done with node, so we can store it more efficiently if - # it has just one child - if node.num_children() == 1: - node = Nonterminal1(node.type, node.get_child(0)) - self.stack[-1][2].append_child(node) + self.stack.node_append_child(node) else: self.root = node diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py --- a/pypy/interpreter/pyparser/pyparse.py +++ b/pypy/interpreter/pyparser/pyparse.py @@ -132,7 +132,11 @@ w_message = space.str(e.get_w_value(space)) raise error.SyntaxError(space.text_w(w_message)) raise + if enc is not None: + compile_info.encoding = enc + return self._parse(textsrc, compile_info) + def _parse(self, textsrc, compile_info): flags = compile_info.flags # The tokenizer is very picky about how it wants its input. @@ -181,13 +185,16 @@ else: new_err = error.SyntaxError msg = "invalid syntax" - raise new_err(msg, e.lineno, e.column, e.line, + if e.expected_str is not None: + msg += " (expected '%s')" % e.expected_str + + # parser.ParseError(...).column is 0-based, but the offsets in the + # exceptions in the error module are 1-based, hence the '+ 1' + raise new_err(msg, e.lineno, e.column + 1, e.line, compile_info.filename) else: tree = self.root finally: # Avoid hanging onto the tree. self.root = None - if enc is not None: - compile_info.encoding = enc return tree diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py --- a/pypy/interpreter/pyparser/pytokenizer.py +++ b/pypy/interpreter/pyparser/pytokenizer.py @@ -73,14 +73,14 @@ logical line; continuation lines are included. """ token_list = [] - lnum = parenlev = continued = 0 + lnum = continued = 0 namechars = NAMECHARS numchars = NUMCHARS contstr, needcont = '', 0 contline = None indents = [0] last_comment = '' - parenlevstart = (0, 0, "") + parenstack = [] # make the annotator happy endDFA = DUMMY_DFA @@ -97,7 +97,7 @@ if contstr: if not line: raise TokenError( - "EOF while scanning triple-quoted string literal", + "end of file (EOF) while scanning triple-quoted string literal", strstart[2], strstart[0], strstart[1]+1, token_list, lnum-1) endmatch = endDFA.recognize(line) @@ -123,7 +123,7 @@ contline = contline + line continue - elif parenlev == 0 and not continued: # new statement + elif not parenstack and not continued: # new statement if not line: break column = 0 while pos < max: # measure leading whitespace @@ -143,21 +143,21 @@ token_list.append((tokens.INDENT, line[:pos], lnum, 0, line)) last_comment = '' while column < indents[-1]: - indents = indents[:-1] + indents.pop() token_list.append((tokens.DEDENT, '', lnum, pos, line)) last_comment = '' if column != indents[-1]: err = "unindent does not match any outer indentation level" - raise TokenIndentationError(err, line, lnum, 0, token_list) + raise TokenIndentationError(err, line, lnum, column+1, token_list) else: # continued statement if not line: - if parenlev > 0: - lnum1, start1, line1 = parenlevstart + if parenstack: + _, lnum1, start1, line1 = parenstack[0] raise TokenError("parenthesis is never closed", line1, lnum1, start1 + 1, token_list, lnum) - raise TokenError("EOF in multi-line statement", line, - lnum, 0, token_list) + raise TokenError("end of file (EOF) in multi-line statement", line, + lnum, 0, token_list) # XXX why is the offset 0 here? continued = 0 while pos < max: @@ -180,7 +180,7 @@ token_list.append((tokens.NUMBER, token, lnum, start, line)) last_comment = '' elif initial in '\r\n': - if parenlev <= 0: + if not parenstack: tok = (tokens.NEWLINE, last_comment, lnum, start, line) token_list.append(tok) last_comment = '' @@ -222,14 +222,22 @@ continued = 1 else: if initial in '([{': - if parenlev == 0: - parenlevstart = (lnum, start, line) - parenlev = parenlev + 1 + parenstack.append((initial, lnum, start, line)) elif initial in ')]}': - parenlev = parenlev - 1 - if parenlev < 0: + if not parenstack: raise TokenError("unmatched '%s'" % initial, line, lnum, start + 1, token_list) + opening, lnum1, start1, line1 = parenstack.pop() + if not ((opening == "(" and initial == ")") or + (opening == "[" and initial == "]") or + (opening == "{" and initial == "}")): + msg = "closing parenthesis '%s' does not match opening parenthesis '%s'" % ( + initial, opening) + + if lnum1 != lnum: + msg += " on line " + str(lnum1) + raise TokenError( + msg, line, lnum, start + 1, token_list) if token in python_opmap: punct = python_opmap[token] else: @@ -241,7 +249,7 @@ if start < 0: start = pos if start", "exec") + parser = pyparse.PythonParser(fakespace) + tree = parser._parse(s, info) + b = time.clock() + print fn, (b-a) + + +def entry_point(argv): + if len(argv) == 2: + fn = argv[1] + else: + fn = "../../../../rpython/rlib/unicodedata/unicodedb_5_2_0.py" + fd = os.open(fn, os.O_RDONLY, 0777) + res = [] + while True: + s = os.read(fd, 4096) + if not s: + break + res.append(s) + os.close(fd) + s = "".join(res) + print len(s) + bench(fn, s) + + return 0 + +# _____ Define and setup target ___ + +def target(*args): + return entry_point, None + +if __name__ == '__main__': + entry_point(sys.argv) diff --git a/pypy/interpreter/pyparser/test/test_metaparser.py b/pypy/interpreter/pyparser/test/test_metaparser.py --- a/pypy/interpreter/pyparser/test/test_metaparser.py +++ b/pypy/interpreter/pyparser/test/test_metaparser.py @@ -34,8 +34,8 @@ assert len(g.dfas) == 1 eval_sym = g.symbol_ids["eval"] assert g.start == eval_sym - states, first = g.dfas[eval_sym - 256] - assert states == [([(1, 1)], False), ([], True)] + dfa = g.dfas[eval_sym - 256] + assert dfa.states == [([(1, 1)], False), ([], True)] assert g.labels[0] == 0 def test_load_python_grammars(self): @@ -51,7 +51,7 @@ def test_items(self): g = self.gram_for("foo: NAME STRING OP '+'") assert len(g.dfas) == 1 - states = g.dfas[g.symbol_ids["foo"] - 256][0] + states = g.dfas[g.symbol_ids["foo"] - 256].states last = states[0][0][0][1] for state in states[1:-1]: assert last < state[0][0][1] diff --git a/pypy/interpreter/pyparser/test/test_parser.py b/pypy/interpreter/pyparser/test/test_parser.py --- a/pypy/interpreter/pyparser/test/test_parser.py +++ b/pypy/interpreter/pyparser/test/test_parser.py @@ -7,6 +7,12 @@ from pypy.interpreter.pyparser.test.test_metaparser import MyGrammar +def test_char_set(): + first = {5: None, 9: None, 100: None, 255:None} + p = parser.DFA(None, None, first) + for i in range(256): + assert p.could_match_token(i) == (i in first) + class SimpleParser(parser.Parser): def parse(self, input): @@ -55,8 +61,7 @@ n = parser.Terminal(tp, value, 0, 0) else: tp = gram.symbol_ids[data[0]] - children = [] - n = parser.Nonterminal(tp, children) + n = parser.Nonterminal(tp) new_indent = count_indent(line) if new_indent >= last_indent: if new_indent == last_indent and node_stack: @@ -291,3 +296,37 @@ NEWLINE ENDMARKER""" assert tree_from_string(expected, gram) == p.parse("hi 42 end") + + + def test_optimized_terminal(self): + gram = """foo: bar baz 'end' NEWLINE ENDMARKER +bar: NAME +baz: NUMBER +""" + p, gram = self.parser_for(gram, False) + expected = """ + foo + bar + NAME "a_name" + baz + NUMBER "42" + NAME "end" + NEWLINE + ENDMARKER""" + input = "a_name 42 end" + tree = p.parse(input) + assert tree_from_string(expected, gram) == tree + assert isinstance(tree, parser.Nonterminal) + assert isinstance(tree.get_child(0), parser.Nonterminal1) + assert isinstance(tree.get_child(1), parser.Nonterminal1) + + + def test_error_string(self): + p, gram = self.parser_for( + "foo: 'if' NUMBER '+' NUMBER" + ) + info = py.test.raises(parser.ParseError, p.parse, "if 42") + info.value.expected_str is None + info = py.test.raises(parser.ParseError, p.parse, "if 42 42") + info.value.expected_str == '+' + diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py --- a/pypy/interpreter/pyparser/test/test_pyparse.py +++ b/pypy/interpreter/pyparser/test/test_pyparse.py @@ -76,14 +76,14 @@ exc = py.test.raises(SyntaxError, parse, "name another for").value assert exc.msg == "invalid syntax" assert exc.lineno == 1 - assert exc.offset == 5 + assert exc.offset == 6 assert exc.text.startswith("name another for") exc = py.test.raises(SyntaxError, parse, "x = \"blah\n\n\n").value - assert exc.msg == "EOL while scanning string literal" + assert exc.msg == "end of line (EOL) while scanning string literal" assert exc.lineno == 1 assert exc.offset == 5 exc = py.test.raises(SyntaxError, parse, "x = '''\n\n\n").value - assert exc.msg == "EOF while scanning triple-quoted string literal" + assert exc.msg == "end of file (EOF) while scanning triple-quoted string literal" assert exc.lineno == 1 assert exc.offset == 5 assert exc.lastlineno == 3 @@ -112,7 +112,7 @@ assert exc.msg == "expected an indented block" assert exc.lineno == 3 assert exc.text.startswith("pass") - assert exc.offset == 0 + assert exc.offset == 1 input = "hi\n indented" exc = py.test.raises(IndentationError, parse, input).value assert exc.msg == "unexpected indent" @@ -120,6 +120,7 @@ exc = py.test.raises(IndentationError, parse, input).value assert exc.msg == "unindent does not match any outer indentation level" assert exc.lineno == 3 + assert exc.offset == 3 def test_mac_newline(self): self.parse("this_is\ra_mac\rfile") @@ -165,3 +166,11 @@ for linefeed in ["\r\n","\r"]: tree = self.parse(fmt % linefeed) assert expected_tree == tree + + def test_error_forgotten_chars(self): + info = py.test.raises(SyntaxError, self.parse, "if 1\n print 4") + assert "(expected ':')" in info.value.msg + info = py.test.raises(SyntaxError, self.parse, "for i in range(10)\n print i") + assert "(expected ':')" in info.value.msg + info = py.test.raises(SyntaxError, self.parse, "def f:\n print 1") + assert "(expected '(')" in info.value.msg diff --git a/pypy/interpreter/pyparser/test/test_pytokenizer.py b/pypy/interpreter/pyparser/test/test_pytokenizer.py new file mode 100644 --- /dev/null +++ b/pypy/interpreter/pyparser/test/test_pytokenizer.py @@ -0,0 +1,66 @@ +import pytest +from pypy.interpreter.pyparser import pytokenizer +from pypy.interpreter.pyparser.pygram import tokens +from pypy.interpreter.pyparser.error import TokenError + +def tokenize(s): + return pytokenizer.generate_tokens(s.splitlines(True) + ["\n"], 0) + +def check_token_error(s, msg=None, pos=-1, line=-1): + error = pytest.raises(TokenError, tokenize, s) + if msg is not None: + assert error.value.msg == msg + if pos != -1: + assert error.value.offset == pos + if line != -1: + assert error.value.lineno == line + + +class TestTokenizer(object): + + def test_simple(self): + line = "a+1" + tks = tokenize(line) + assert tks == [ + (tokens.NAME, 'a', 1, 0, line), + (tokens.PLUS, '+', 1, 1, line), + (tokens.NUMBER, '1', 1, 2, line), + (tokens.NEWLINE, '', 2, 0, '\n'), + (tokens.NEWLINE, '', 2, 0, '\n'), + (tokens.ENDMARKER, '', 2, 0, ''), + ] + + def test_error_parenthesis(self): + for paren in "([{": + check_token_error(paren + "1 + 2", + "parenthesis is never closed", + 1) + + for paren in ")]}": + check_token_error("1 + 2" + paren, + "unmatched '%s'" % (paren, ), + 6) + + for i, opening in enumerate("([{"): + for j, closing in enumerate(")]}"): + if i == j: + continue + check_token_error(opening + "1\n" + closing, + "closing parenthesis '%s' does not match opening parenthesis '%s' on line 1" % (closing, opening), + pos=1, line=2) + check_token_error(opening + "1" + closing, + "closing parenthesis '%s' does not match opening parenthesis '%s'" % (closing, opening), + pos=3, line=1) + check_token_error(opening + closing, + "closing parenthesis '%s' does not match opening parenthesis '%s'" % (closing, opening), + pos=2, line=1) + + + def test_unknown_char(self): + check_token_error("?", "Unknown character", 1) + + def test_eol_string(self): + check_token_error("x = 'a", pos=5, line=1) + + def test_eof_triple_quoted(self): + check_token_error("'''", pos=1, line=1) diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py --- a/pypy/interpreter/test/test_compiler.py +++ b/pypy/interpreter/test/test_compiler.py @@ -77,7 +77,7 @@ """) assert self.space.unwrap(w_args) == ( 'unindent does not match any outer indentation level', - ('', 3, 0, ' y\n')) + ('', 3, 2, ' y\n')) def test_getcodeflags(self): code = self.compiler.compile('from __future__ import division\n', diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py --- a/pypy/interpreter/test/test_syntax.py +++ b/pypy/interpreter/test/test_syntax.py @@ -750,7 +750,7 @@ except SyntaxError as e: assert e.lineno == 4 assert e.text.endswith('a b c d e\n') - assert e.offset == e.text.index('b') + assert e.offset == e.text.index('b') + 1 # offset is 1-based else: raise Exception("no SyntaxError??") diff --git a/pypy/interpreter/test/test_typedef.py b/pypy/interpreter/test/test_typedef.py --- a/pypy/interpreter/test/test_typedef.py +++ b/pypy/interpreter/test/test_typedef.py @@ -423,3 +423,10 @@ def test_get_with_none_arg(self): raises(TypeError, type.__dict__['__mro__'].__get__, None) raises(TypeError, type.__dict__['__mro__'].__get__, None, None) + + def test_builtin_readonly_property(self): + import sys + x = lambda: 5 + e = raises(TypeError, 'x.func_globals = {}') + if '__pypy__' in sys.builtin_module_names: + assert str(e.value) == "readonly attribute 'func_globals'" diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py --- a/pypy/interpreter/typedef.py +++ b/pypy/interpreter/typedef.py @@ -309,12 +309,18 @@ self.reqcls, Arguments(space, [w_obj, space.newtext(self.name)])) + def readonly_attribute(self, space): # overwritten in cpyext + if self.name == '': + raise oefmt(space.w_TypeError, "readonly attribute") + else: + raise oefmt(space.w_TypeError, "readonly attribute '%s'", self.name) + def descr_property_set(self, space, w_obj, w_value): """property.__set__(obj, value) Change the value of the property of the given obj.""" fset = self.fset if fset is None: - raise oefmt(space.w_TypeError, "readonly attribute") + raise self.readonly_attribute(space) try: fset(self, space, w_obj, w_value) except DescrMismatch: diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py --- a/pypy/module/_cffi_backend/ccallback.py +++ b/pypy/module/_cffi_backend/ccallback.py @@ -232,7 +232,9 @@ "different from the 'ffi.h' file seen at compile-time)") def py_invoke(self, ll_res, ll_args): + key_pycode = self.key_pycode jitdriver1.jit_merge_point(callback=self, + key_pycode=key_pycode, ll_res=ll_res, ll_args=ll_args) self.do_invoke(ll_res, ll_args) @@ -294,7 +296,7 @@ return 'cffi_callback ' + key_pycode.get_repr() jitdriver1 = jit.JitDriver(name='cffi_callback', - greens=['callback.key_pycode'], + greens=['key_pycode'], reds=['ll_res', 'll_args', 'callback'], get_printable_location=get_printable_location1) diff --git a/pypy/module/_io/test/test_interp_textio.py b/pypy/module/_io/test/test_interp_textio.py --- a/pypy/module/_io/test/test_interp_textio.py +++ b/pypy/module/_io/test/test_interp_textio.py @@ -7,6 +7,11 @@ from pypy.module._io.interp_bytesio import W_BytesIO from pypy.module._io.interp_textio import W_TextIOWrapper, DecodeBuffer +# workaround suggestion for slowness by David McIver: +# force hypothesis to initialize some lazy stuff +# (which takes a lot of time, which trips the timer otherwise) +st.text().example() + def translate_newlines(text): text = text.replace(u'\r\n', u'\n') text = text.replace(u'\r', u'\n') @@ -29,7 +34,7 @@ @given(data=st_readline(), mode=st.sampled_from(['\r', '\n', '\r\n', ''])) - at settings(deadline=None) + at settings(deadline=None, database=None) def test_readline(space, data, mode): txt, limits = data w_stream = W_BytesIO(space) diff --git a/pypy/module/_rawffi/alt/test/test_struct.py b/pypy/module/_rawffi/alt/test/test_struct.py --- a/pypy/module/_rawffi/alt/test/test_struct.py +++ b/pypy/module/_rawffi/alt/test/test_struct.py @@ -43,7 +43,11 @@ def setup_class(cls): BaseAppTestFFI.setup_class.im_func(cls) - @unwrap_spec(addr=int, typename='text', length=int) + from rpython.rlib import clibffi + from rpython.rlib.rarithmetic import r_uint + from rpython.rtyper.lltypesystem import lltype, rffi + + @unwrap_spec(addr=r_uint, typename='text', length=int) def read_raw_mem(space, addr, typename, length): import ctypes addr = ctypes.cast(addr, ctypes.c_void_p) @@ -58,9 +62,6 @@ else: cls.w_read_raw_mem = cls.space.wrap(interp2app(read_raw_mem)) # - from rpython.rlib import clibffi - from rpython.rlib.rarithmetic import r_uint - from rpython.rtyper.lltypesystem import lltype, rffi dummy_type = lltype.malloc(clibffi.FFI_TYPE_P.TO, flavor='raw') dummy_type.c_size = r_uint(123) dummy_type.c_alignment = rffi.cast(rffi.USHORT, 0) diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py --- a/pypy/module/_sre/interp_sre.py +++ b/pypy/module/_sre/interp_sre.py @@ -77,15 +77,15 @@ w_import = space.getattr(w_builtin, space.newtext("__import__")) return space.call_function(w_import, space.newtext("re")) -def matchcontext(space, ctx): +def matchcontext(space, ctx, pattern): try: - return rsre_core.match_context(ctx) + return rsre_core.match_context(ctx, pattern) except rsre_core.Error as e: raise OperationError(space.w_RuntimeError, space.newtext(e.msg)) -def searchcontext(space, ctx): +def searchcontext(space, ctx, pattern): try: - return rsre_core.search_context(ctx) + return rsre_core.search_context(ctx, pattern) except rsre_core.Error as e: raise OperationError(space.w_RuntimeError, space.newtext(e.msg)) @@ -114,7 +114,7 @@ pos = len(unicodestr) if endpos > len(unicodestr): endpos = len(unicodestr) - return rsre_core.UnicodeMatchContext(self.code, unicodestr, + return rsre_core.UnicodeMatchContext(unicodestr, pos, endpos, self.flags) elif space.isinstance_w(w_string, space.w_bytes): str = space.bytes_w(w_string) @@ -122,7 +122,7 @@ pos = len(str) if endpos > len(str): endpos = len(str) - return rsre_core.StrMatchContext(self.code, str, + return rsre_core.StrMatchContext(str, pos, endpos, self.flags) else: buf = space.readbuf_w(w_string) @@ -132,7 +132,7 @@ pos = size if endpos > size: endpos = size - return rsre_core.BufMatchContext(self.code, buf, + return rsre_core.BufMatchContext(buf, pos, endpos, self.flags) def getmatch(self, ctx, found): @@ -144,12 +144,12 @@ @unwrap_spec(pos=int, endpos=int) def match_w(self, w_string, pos=0, endpos=sys.maxint): ctx = self.make_ctx(w_string, pos, endpos) - return self.getmatch(ctx, matchcontext(self.space, ctx)) + return self.getmatch(ctx, matchcontext(self.space, ctx, self.code)) @unwrap_spec(pos=int, endpos=int) def search_w(self, w_string, pos=0, endpos=sys.maxint): ctx = self.make_ctx(w_string, pos, endpos) - return self.getmatch(ctx, searchcontext(self.space, ctx)) + return self.getmatch(ctx, searchcontext(self.space, ctx, self.code)) @unwrap_spec(pos=int, endpos=int) def findall_w(self, w_string, pos=0, endpos=sys.maxint): @@ -157,7 +157,7 @@ matchlist_w = [] ctx = self.make_ctx(w_string, pos, endpos) while ctx.match_start <= ctx.end: - if not searchcontext(space, ctx): + if not searchcontext(space, ctx, self.code): break num_groups = self.num_groups w_emptystr = space.newtext("") @@ -182,7 +182,7 @@ # this also works as the implementation of the undocumented # scanner() method. ctx = self.make_ctx(w_string, pos, endpos) - scanner = W_SRE_Scanner(self, ctx) + scanner = W_SRE_Scanner(self, ctx, self.code) return scanner @unwrap_spec(maxsplit=int) @@ -193,7 +193,7 @@ last = 0 ctx = self.make_ctx(w_string) while not maxsplit or n < maxsplit: - if not searchcontext(space, ctx): From pypy.commits at gmail.com Sun Apr 15 13:01:50 2018 From: pypy.commits at gmail.com (rlamy) Date: Sun, 15 Apr 2018 10:01:50 -0700 (PDT) Subject: [pypy-commit] pypy py3tests: hg merge apptest-file Message-ID: <5ad3857e.568bdf0a.98be6.8f23@mx.google.com> Author: Ronan Lamy Branch: py3tests Changeset: r94343:77838bcabf15 Date: 2018-04-15 18:01 +0100 http://bitbucket.org/pypy/pypy/changeset/77838bcabf15/ Log: hg merge apptest-file diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -252,6 +252,10 @@ BoolOption("newshortcut", "cache and shortcut calling __new__ from builtin types", default=False), + BoolOption("reinterpretasserts", + "Perform reinterpretation when an assert fails " + "(only relevant for tests)", + default=False), ]), ]) diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -132,10 +132,9 @@ """ def accept_regular_test(self): if self.config.option.runappdirect: - # only collect regular tests if we are in an 'app_test' directory, - # or in test_lib_pypy + # only collect regular tests if we are in test_lib_pypy for name in self.listnames(): - if "app_test" in name or "test_lib_pypy" in name: + if "test_lib_pypy" in name: return True return False return True diff --git a/pypy/tool/pytest/apptest.py b/pypy/tool/pytest/apptest.py --- a/pypy/tool/pytest/apptest.py +++ b/pypy/tool/pytest/apptest.py @@ -237,7 +237,7 @@ src = extract_docstring_if_empty_function(target) if self.config.option.runappdirect: return run_with_python(self.config.option.python, src, None) - space = gettestobjspace() + space = gettestobjspace(**{'objspace.std.reinterpretasserts': True}) filename = self._getdynfilename(target) func = app2interp_temp(src, filename=filename) # print "executing", func diff --git a/pypy/tool/pytest/objspace.py b/pypy/tool/pytest/objspace.py --- a/pypy/tool/pytest/objspace.py +++ b/pypy/tool/pytest/objspace.py @@ -7,7 +7,7 @@ _SPACECACHE={} def gettestobjspace(**kwds): - """ helper for instantiating and caching space's for testing. + """ helper for instantiating and caching spaces for testing. """ try: config = make_config(option, **kwds) @@ -33,8 +33,9 @@ config.objspace.extmodules = 'pypy.tool.pytest.fake_pytest' space = make_objspace(config) space.startup() # Initialize all builtin modules - space.setitem(space.builtin.w_dict, space.wrap('AssertionError'), - appsupport.build_pytest_assertion(space)) + if config.objspace.std.reinterpretasserts: + space.setitem(space.builtin.w_dict, space.wrap('AssertionError'), + appsupport.build_pytest_assertion(space)) space.setitem(space.builtin.w_dict, space.wrap('raises'), space.wrap(appsupport.app_raises)) space.setitem(space.builtin.w_dict, space.wrap('skip'), diff --git a/pypy/tool/pytest/test/test_appsupport.py b/pypy/tool/pytest/test/test_appsupport.py --- a/pypy/tool/pytest/test/test_appsupport.py +++ b/pypy/tool/pytest/test/test_appsupport.py @@ -24,10 +24,16 @@ def test_method(self): pass """) + testdir.makepyfile(apptest_collection=""" + def test_app(): + pass + """) setpypyconftest(testdir) result = testdir.runpytest("--collectonly") assert result.ret == 0 result.stdout.fnmatch_lines([ + "*AppTestModule*apptest_collection*", + "*AppTestFunction*test_app*", "*Function*test_func*", "*Class*TestClassInt*", "*Function*test_method*", @@ -128,6 +134,47 @@ "*E*application-level*KeyError*42*", ]) +def test_apptest_raise(testdir): + setpypyconftest(testdir) + p = testdir.makepyfile(apptest_raise=""" + def test_raise(): + raise KeyError(42) + """) + result = testdir.runpytest(p) + assert result.ret == 1 + result.stdout.fnmatch_lines([ + "*E*application-level*KeyError*42*", + ]) + +def test_apptest_fail_plain(testdir): + setpypyconftest(testdir) + p = testdir.makepyfile(apptest_fail=""" + def test_fail(): + x = 'foo' + assert x == 'bar' + """) + result = testdir.runpytest(p) + assert result.ret == 1 + result.stdout.fnmatch_lines([ + "*E*(application-level) AssertionError", + ]) + +def test_apptest_fail_rewrite(testdir): + setpypyconftest(testdir) + p = testdir.makepyfile(apptest_fail_rewrite=""" + def test_fail(): + x = 'foo' + assert x == 'bar' + """) + result = testdir.runpytest(p, "--applevel-rewrite") + assert result.ret == 1 + result.stdout.fnmatch_lines([ + "*E*application-level*AssertionError: assert 'foo' == 'bar'", + "*E*- foo*", + "*E*+ bar*", + ]) + + def app_test_raises(): info = raises(TypeError, id) assert info.type is TypeError From pypy.commits at gmail.com Mon Apr 16 05:38:44 2018 From: pypy.commits at gmail.com (antocuni) Date: Mon, 16 Apr 2018 02:38:44 -0700 (PDT) Subject: [pypy-commit] pypy gc-hooks: add JIT support for rtimer.get_timestamp_unit Message-ID: <5ad46f24.af90df0a.9ac51.7ea4@mx.google.com> Author: Antonio Cuni Branch: gc-hooks Changeset: r94345:43e5624d7947 Date: 2018-04-16 11:27 +0200 http://bitbucket.org/pypy/pypy/changeset/43e5624d7947/ Log: add JIT support for rtimer.get_timestamp_unit diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -2164,6 +2164,11 @@ oopspecindex=EffectInfo.OS_MATH_READ_TIMESTAMP, extraeffect=EffectInfo.EF_CANNOT_RAISE) + def rewrite_op_ll_get_timestamp_unit(self, op): + op1 = self.prepare_builtin_call(op, "ll_get_timestamp_unit", []) + return self.handle_residual_call(op1, + extraeffect=EffectInfo.EF_CANNOT_RAISE) + def rewrite_op_jit_force_quasi_immutable(self, op): v_inst, c_fieldname = op.args descr1 = self.cpu.fielddescrof(v_inst.concretetype.TO, diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py --- a/rpython/jit/codewriter/support.py +++ b/rpython/jit/codewriter/support.py @@ -285,6 +285,9 @@ from rpython.rlib import rtimer return rtimer.read_timestamp() +def _ll_0_ll_get_timestamp_unit(): + from rpython.rlib import rtimer + return rtimer.get_timestamp_unit() # math support # ------------ diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py --- a/rpython/jit/metainterp/test/test_ajit.py +++ b/rpython/jit/metainterp/test/test_ajit.py @@ -2577,6 +2577,14 @@ res = self.interp_operations(f, []) assert res + def test_get_timestamp_unit(self): + import time + from rpython.rlib import rtimer + def f(): + return rtimer.get_timestamp_unit() + unit = self.interp_operations(f, []) + assert unit == rtimer.UNIT_NS + def test_bug688_multiple_immutable_fields(self): myjitdriver = JitDriver(greens=[], reds=['counter','context']) From pypy.commits at gmail.com Mon Apr 16 05:38:42 2018 From: pypy.commits at gmail.com (antocuni) Date: Mon, 16 Apr 2018 02:38:42 -0700 (PDT) Subject: [pypy-commit] pypy gc-hooks: add rlib.rtimer.get_timestamp_unit, which tells some info about the value returned by read_timestamp; will expose an app-level interface later Message-ID: <5ad46f22.07c5df0a.910b6.44c5@mx.google.com> Author: Antonio Cuni Branch: gc-hooks Changeset: r94344:76a781fe5853 Date: 2018-04-16 11:20 +0200 http://bitbucket.org/pypy/pypy/changeset/76a781fe5853/ Log: add rlib.rtimer.get_timestamp_unit, which tells some info about the value returned by read_timestamp; will expose an app-level interface later diff --git a/rpython/rlib/rtimer.py b/rpython/rlib/rtimer.py --- a/rpython/rlib/rtimer.py +++ b/rpython/rlib/rtimer.py @@ -7,6 +7,11 @@ _is_64_bit = r_uint.BITS > 32 +# unit of values returned by read_timestamp. Should be in sync with the ones +# defined in translator/c/debug_print.h +UNIT_TSC = 0 +UNIT_NS = 1 +UNIT_QUERY_PERFORMANCE_COUNTER = 2 def read_timestamp(): # Returns a longlong on 32-bit, and a regular int on 64-bit. @@ -17,6 +22,11 @@ else: return longlongmask(x) +def get_timestamp_unit(): + # an unit which is as arbitrary as the way we build the result of + # read_timestamp :) + return UNIT_NS + class ReadTimestampEntry(ExtRegistryEntry): _about_ = read_timestamp @@ -35,3 +45,15 @@ else: resulttype = rffi.LONGLONG return hop.genop("ll_read_timestamp", [], resulttype=resulttype) + + +class ReadTimestampEntry(ExtRegistryEntry): + _about_ = get_timestamp_unit + + def compute_result_annotation(self): + from rpython.annotator.model import SomeInteger + return SomeInteger(nonneg=True) + + def specialize_call(self, hop): + hop.exception_cannot_occur() + return hop.genop("ll_get_timestamp_unit", [], resulttype=lltype.Signed) diff --git a/rpython/rlib/test/test_rtimer.py b/rpython/rlib/test/test_rtimer.py --- a/rpython/rlib/test/test_rtimer.py +++ b/rpython/rlib/test/test_rtimer.py @@ -1,28 +1,56 @@ import time - -from rpython.rlib.rtimer import read_timestamp +import platform +from rpython.rlib import rtimer from rpython.rtyper.test.test_llinterp import interpret from rpython.translator.c.test.test_genc import compile -def timer(): - t1 = read_timestamp() - start = time.time() - while time.time() - start < 0.1: - # busy wait - pass - t2 = read_timestamp() - return t2 - t1 +class TestTimer(object): -def test_timer(): - diff = timer() - # We're counting ticks, verify they look correct - assert diff > 1000 + @staticmethod + def timer(): + t1 = rtimer.read_timestamp() + start = time.time() + while time.time() - start < 0.1: + # busy wait + pass + t2 = rtimer.read_timestamp() + return t2 - t1 -def test_annotation(): - diff = interpret(timer, []) - assert diff > 1000 + def test_direct(self): + diff = self.timer() + # We're counting ticks, verify they look correct + assert diff > 1000 -def test_compile_c(): - function = compile(timer, []) - diff = function() - assert diff > 1000 \ No newline at end of file + def test_annotation(self): + diff = interpret(self.timer, []) + assert diff > 1000 + + def test_compile_c(self): + function = compile(self.timer, []) + diff = function() + assert diff > 1000 + + +class TestGetUnit(object): + + @staticmethod + def get_unit(): + return rtimer.get_timestamp_unit() + + def test_direct(self): + unit = self.get_unit() + assert unit == rtimer.UNIT_NS + + def test_annotation(self): + unit = interpret(self.get_unit, []) + assert unit == rtimer.UNIT_NS + + def test_compile_c(self): + function = compile(self.get_unit, []) + unit = function() + if platform.processor() in ('x86', 'x86_64'): + assert unit == rtimer.UNIT_TSC + else: + assert unit in (rtimer.UNIT_TSC, + rtimer.UNIT_NS, + rtimer.UNIT_QUERY_PERFORMANCE_COUNTER) diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -445,6 +445,7 @@ 'get_write_barrier_from_array_failing_case': LLOp(sideeffects=False), 'gc_get_type_info_group': LLOp(sideeffects=False), 'll_read_timestamp': LLOp(canrun=True), + 'll_get_timestamp_unit': LLOp(canrun=True), # __________ GC operations __________ diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py --- a/rpython/rtyper/lltypesystem/opimpl.py +++ b/rpython/rtyper/lltypesystem/opimpl.py @@ -696,6 +696,10 @@ from rpython.rlib.rtimer import read_timestamp return read_timestamp() +def op_ll_get_timestamp_unit(): + from rpython.rlib.rtimer import get_timestamp_unit + return get_timestamp_unit() + def op_debug_fatalerror(ll_msg): from rpython.rtyper.lltypesystem import lltype, rstr from rpython.rtyper.llinterp import LLFatalError diff --git a/rpython/translator/c/src/asm_gcc_x86.h b/rpython/translator/c/src/asm_gcc_x86.h --- a/rpython/translator/c/src/asm_gcc_x86.h +++ b/rpython/translator/c/src/asm_gcc_x86.h @@ -70,6 +70,7 @@ // lfence // I don't know how important it is, comment talks about time warps +#define READ_TIMESTAMP_UNIT TIMESTAMP_UNIT_TSC #ifndef PYPY_CPU_HAS_STANDARD_PRECISION /* On x86-32, we have to use the following hacks to set and restore diff --git a/rpython/translator/c/src/asm_gcc_x86_64.h b/rpython/translator/c/src/asm_gcc_x86_64.h --- a/rpython/translator/c/src/asm_gcc_x86_64.h +++ b/rpython/translator/c/src/asm_gcc_x86_64.h @@ -7,5 +7,6 @@ val = (_rdx << 32) | _rax; \ } while (0) +#define READ_TIMESTAMP_UNIT TIMESTAMP_UNIT_TSC #define RPy_YieldProcessor() asm("pause") diff --git a/rpython/translator/c/src/asm_msvc.h b/rpython/translator/c/src/asm_msvc.h --- a/rpython/translator/c/src/asm_msvc.h +++ b/rpython/translator/c/src/asm_msvc.h @@ -13,3 +13,4 @@ #include #pragma intrinsic(__rdtsc) #define READ_TIMESTAMP(val) do { val = (long long)__rdtsc(); } while (0) +#define READ_TIMESTAMP_UNIT TIMESTAMP_UNIT_TSC diff --git a/rpython/translator/c/src/debug_print.h b/rpython/translator/c/src/debug_print.h --- a/rpython/translator/c/src/debug_print.h +++ b/rpython/translator/c/src/debug_print.h @@ -51,6 +51,11 @@ RPY_EXTERN long pypy_have_debug_prints; RPY_EXPORTED FILE *pypy_debug_file; +/* these should be in sync with the values defined in rlib/rtimer.py */ +#define TIMESTAMP_UNIT_TSC 0 +#define TIMESTAMP_UNIT_NS 1 +#define TIMESTAMP_UNIT_QUERY_PERFORMANCE_COUNTER 2 + #define OP_LL_READ_TIMESTAMP(val) READ_TIMESTAMP(val) #include "src/asm.h" @@ -62,11 +67,15 @@ # ifdef _WIN32 # define READ_TIMESTAMP(val) QueryPerformanceCounter((LARGE_INTEGER*)&(val)) +# define READ_TIMESTAMP_UNIT TIMESTAMP_UNIT_QUERY_PERFORMANCE_COUNTER # else RPY_EXTERN long long pypy_read_timestamp(void); # define READ_TIMESTAMP(val) (val) = pypy_read_timestamp() +# define READ_TIMESTAMP_UNIT TIMESTAMP_UNIT_NS # endif #endif + +#define OP_LL_GET_TIMESTAMP_UNIT(res) res = READ_TIMESTAMP_UNIT From pypy.commits at gmail.com Mon Apr 16 05:38:48 2018 From: pypy.commits at gmail.com (antocuni) Date: Mon, 16 Apr 2018 02:38:48 -0700 (PDT) Subject: [pypy-commit] pypy gc-hooks: expose rtimer.get_timestamp_unit to applevel, for the same motivations as the previous commit Message-ID: <5ad46f28.117c1c0a.7609e.a077@mx.google.com> Author: Antonio Cuni Branch: gc-hooks Changeset: r94347:0803e12ba2c3 Date: 2018-04-16 11:37 +0200 http://bitbucket.org/pypy/pypy/changeset/0803e12ba2c3/ Log: expose rtimer.get_timestamp_unit to applevel, for the same motivations as the previous commit diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -83,6 +83,7 @@ 'debug_print_once' : 'interp_debug.debug_print_once', 'debug_flush' : 'interp_debug.debug_flush', 'debug_read_timestamp' : 'interp_debug.debug_read_timestamp', + 'debug_get_timestamp_unit' : 'interp_debug.debug_get_timestamp_unit', 'builtinify' : 'interp_magic.builtinify', 'hidden_applevel' : 'interp_magic.hidden_applevel', 'get_hidden_tb' : 'interp_magic.get_hidden_tb', diff --git a/pypy/module/__pypy__/interp_debug.py b/pypy/module/__pypy__/interp_debug.py --- a/pypy/module/__pypy__/interp_debug.py +++ b/pypy/module/__pypy__/interp_debug.py @@ -2,7 +2,6 @@ from rpython.rlib import debug, jit from rpython.rlib import rtimer - @jit.dont_look_inside @unwrap_spec(category='text') def debug_start(space, category): @@ -32,3 +31,11 @@ def debug_read_timestamp(space): return space.newint(rtimer.read_timestamp()) + +def debug_get_timestamp_unit(space): + unit = rtimer.get_timestamp_unit() + try: + unit_str = rtimer.UNITS[unit] + except IndexError: + unit_str = 'UNKNOWN(%d)' % unit + return space.newtext(unit_str) diff --git a/pypy/module/__pypy__/test/test_debug.py b/pypy/module/__pypy__/test/test_debug.py --- a/pypy/module/__pypy__/test/test_debug.py +++ b/pypy/module/__pypy__/test/test_debug.py @@ -54,3 +54,8 @@ a = debug_read_timestamp() b = debug_read_timestamp() assert b > a + + def test_debug_get_timestamp_unit(self): + from __pypy__ import debug_get_timestamp_unit + unit = debug_get_timestamp_unit() + assert unit in ('tsc', 'ns', 'QueryPerformanceCounter') diff --git a/rpython/rlib/rtimer.py b/rpython/rlib/rtimer.py --- a/rpython/rlib/rtimer.py +++ b/rpython/rlib/rtimer.py @@ -10,8 +10,9 @@ # unit of values returned by read_timestamp. Should be in sync with the ones # defined in translator/c/debug_print.h UNIT_TSC = 0 -UNIT_NS = 1 +UNIT_NS = 1 # nanoseconds UNIT_QUERY_PERFORMANCE_COUNTER = 2 +UNITS = ('tsc', 'ns', 'QueryPerformanceCounter') def read_timestamp(): # Returns a longlong on 32-bit, and a regular int on 64-bit. From pypy.commits at gmail.com Mon Apr 16 05:38:46 2018 From: pypy.commits at gmail.com (antocuni) Date: Mon, 16 Apr 2018 02:38:46 -0700 (PDT) Subject: [pypy-commit] pypy gc-hooks: expose rlib.rtimer.read_timestamp as __pypy__.debug_read_timestamp. The underlying idea is that these values are now exposed to applevel by gchooks (in particular, stats.duration), so some 3rd-party library might want to use this to do calibration or to convert the raw values to (milli)seconds Message-ID: <5ad46f26.14a1df0a.ef44c.5b28@mx.google.com> Author: Antonio Cuni Branch: gc-hooks Changeset: r94346:d68302cf28a3 Date: 2018-04-16 11:33 +0200 http://bitbucket.org/pypy/pypy/changeset/d68302cf28a3/ Log: expose rlib.rtimer.read_timestamp as __pypy__.debug_read_timestamp. The underlying idea is that these values are now exposed to applevel by gchooks (in particular, stats.duration), so some 3rd-party library might want to use this to do calibration or to convert the raw values to (milli)seconds diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -82,6 +82,7 @@ 'debug_stop' : 'interp_debug.debug_stop', 'debug_print_once' : 'interp_debug.debug_print_once', 'debug_flush' : 'interp_debug.debug_flush', + 'debug_read_timestamp' : 'interp_debug.debug_read_timestamp', 'builtinify' : 'interp_magic.builtinify', 'hidden_applevel' : 'interp_magic.hidden_applevel', 'get_hidden_tb' : 'interp_magic.get_hidden_tb', diff --git a/pypy/module/__pypy__/interp_debug.py b/pypy/module/__pypy__/interp_debug.py --- a/pypy/module/__pypy__/interp_debug.py +++ b/pypy/module/__pypy__/interp_debug.py @@ -1,5 +1,6 @@ from pypy.interpreter.gateway import unwrap_spec from rpython.rlib import debug, jit +from rpython.rlib import rtimer @jit.dont_look_inside @@ -28,3 +29,6 @@ @jit.dont_look_inside def debug_flush(space): debug.debug_flush() + +def debug_read_timestamp(space): + return space.newint(rtimer.read_timestamp()) diff --git a/pypy/module/__pypy__/test/test_debug.py b/pypy/module/__pypy__/test/test_debug.py --- a/pypy/module/__pypy__/test/test_debug.py +++ b/pypy/module/__pypy__/test/test_debug.py @@ -48,3 +48,9 @@ from __pypy__ import debug_flush debug_flush() # assert did not crash + + def test_debug_read_timestamp(self): + from __pypy__ import debug_read_timestamp + a = debug_read_timestamp() + b = debug_read_timestamp() + assert b > a From pypy.commits at gmail.com Mon Apr 16 05:57:45 2018 From: pypy.commits at gmail.com (antocuni) Date: Mon, 16 Apr 2018 02:57:45 -0700 (PDT) Subject: [pypy-commit] pypy gc-hooks: try to fix translation Message-ID: <5ad47399.13151c0a.11901.ee40@mx.google.com> Author: Antonio Cuni Branch: gc-hooks Changeset: r94348:2ca9948f1f76 Date: 2018-04-16 11:57 +0200 http://bitbucket.org/pypy/pypy/changeset/2ca9948f1f76/ Log: try to fix translation diff --git a/pypy/module/__pypy__/interp_debug.py b/pypy/module/__pypy__/interp_debug.py --- a/pypy/module/__pypy__/interp_debug.py +++ b/pypy/module/__pypy__/interp_debug.py @@ -34,8 +34,12 @@ def debug_get_timestamp_unit(space): unit = rtimer.get_timestamp_unit() - try: - unit_str = rtimer.UNITS[unit] - except IndexError: + if unit == rtimer.UNIT_TSC: + unit_str = 'tsc' + elif unit == rtimer.UNIT_NS: + unit_str = 'ns' + elif unit == rtimer.UNIT_QUERY_PERFORMANCE_COUNTER: + unit_str = 'QueryPerformanceCounter' + else: unit_str = 'UNKNOWN(%d)' % unit return space.newtext(unit_str) diff --git a/rpython/rlib/rtimer.py b/rpython/rlib/rtimer.py --- a/rpython/rlib/rtimer.py +++ b/rpython/rlib/rtimer.py @@ -12,7 +12,6 @@ UNIT_TSC = 0 UNIT_NS = 1 # nanoseconds UNIT_QUERY_PERFORMANCE_COUNTER = 2 -UNITS = ('tsc', 'ns', 'QueryPerformanceCounter') def read_timestamp(): # Returns a longlong on 32-bit, and a regular int on 64-bit. From pypy.commits at gmail.com Mon Apr 16 08:33:59 2018 From: pypy.commits at gmail.com (antocuni) Date: Mon, 16 Apr 2018 05:33:59 -0700 (PDT) Subject: [pypy-commit] pypy gc-hooks: fix translation on 32bit Message-ID: <5ad49837.c6b7df0a.47188.2565@mx.google.com> Author: Antonio Cuni Branch: gc-hooks Changeset: r94349:41878f255d82 Date: 2018-04-16 12:33 +0000 http://bitbucket.org/pypy/pypy/changeset/41878f255d82/ Log: fix translation on 32bit diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py --- a/pypy/module/gc/hook.py +++ b/pypy/module/gc/hook.py @@ -1,7 +1,7 @@ from rpython.memory.gc.hook import GcHooks from rpython.memory.gc import incminimark from rpython.rlib.nonconst import NonConstant -from rpython.rlib.rarithmetic import r_uint +from rpython.rlib.rarithmetic import r_uint, r_longlong from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.typedef import TypeDef, interp_attrproperty, GetSetProperty @@ -113,7 +113,7 @@ class GcMinorHookAction(AsyncAction): count = 0 - duration = 0 + duration = r_longlong(0) total_memory_used = 0 pinned_objects = 0 @@ -123,7 +123,7 @@ def reset(self): self.count = 0 - self.duration = 0 + self.duration = r_longlong(0) def fix_annotation(self): # the annotation of the class and its attributes must be completed @@ -131,7 +131,7 @@ # annotated with the correct types if NonConstant(False): self.count = NonConstant(-42) - self.duration = NonConstant(-42) + self.duration = NonConstant(r_longlong(-42)) self.total_memory_used = NonConstant(r_uint(42)) self.pinned_objects = NonConstant(-42) self.fire() @@ -148,7 +148,7 @@ class GcCollectStepHookAction(AsyncAction): count = 0 - duration = 0 + duration = r_longlong(0) oldstate = 0 newstate = 0 @@ -158,7 +158,7 @@ def reset(self): self.count = 0 - self.duration = 0 + self.duration = r_longlong(0) def fix_annotation(self): # the annotation of the class and its attributes must be completed @@ -166,7 +166,7 @@ # annotated with the correct types if NonConstant(False): self.count = NonConstant(-42) - self.duration = NonConstant(-42) + self.duration = NonConstant(r_longlong(-42)) self.oldstate = NonConstant(-42) self.newstate = NonConstant(-42) self.fire() @@ -196,7 +196,6 @@ def reset(self): self.count = 0 - self.duration = 0 def fix_annotation(self): # the annotation of the class and its attributes must be completed From pypy.commits at gmail.com Mon Apr 16 08:37:13 2018 From: pypy.commits at gmail.com (antocuni) Date: Mon, 16 Apr 2018 05:37:13 -0700 (PDT) Subject: [pypy-commit] pypy gc-hooks: hg merge default Message-ID: <5ad498f9.3cb8df0a.1661c.04a3@mx.google.com> Author: Antonio Cuni Branch: gc-hooks Changeset: r94350:8711cbc8d281 Date: 2018-04-16 12:36 +0000 http://bitbucket.org/pypy/pypy/changeset/8711cbc8d281/ Log: hg merge default diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst --- a/pypy/doc/how-to-release.rst +++ b/pypy/doc/how-to-release.rst @@ -40,6 +40,8 @@ sure things are ported back to the trunk and to the branch as necessary. +* Make sure the RPython builds on the buildbot pass with no failures + * Maybe bump the SOABI number in module/imp/importing. This has many implications, so make sure the PyPy community agrees to the change. diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -3,16 +3,10 @@ ========================== .. this is a revision shortly after release-pypy-6.0.0 -.. startrev: 2e04adf1b89f +.. startrev: f22145c34985 -.. branch: cpyext-subclass-setattr -Fix for python-level classes that inherit from C-API types, previously the -`w_obj` was not necessarily preserved throughout the lifetime of the `pyobj` -which led to cases where instance attributes were lost. Fixes issue #2793 +.. branch: issue2752 - -.. branch: pyparser-improvements-2 - -Improve line offsets that are reported by SyntaxError. Improve error messages -for a few situations, including mismatched parenthesis. +Fix a rare GC bug that was introduced more than one year ago, but was +not diagnosed before issue #2752. diff --git a/pypy/doc/whatsnew-pypy2-6.0.0.rst b/pypy/doc/whatsnew-pypy2-6.0.0.rst --- a/pypy/doc/whatsnew-pypy2-6.0.0.rst +++ b/pypy/doc/whatsnew-pypy2-6.0.0.rst @@ -97,3 +97,15 @@ Work around possible bugs in upstream ioctl users, like CPython allocate at least 1024 bytes for the arg in calls to ``ioctl(fd, request, arg)``. Fixes issue #2776 + +.. branch: cpyext-subclass-setattr + +Fix for python-level classes that inherit from C-API types, previously the +`w_obj` was not necessarily preserved throughout the lifetime of the `pyobj` +which led to cases where instance attributes were lost. Fixes issue #2793 + + +.. branch: pyparser-improvements-2 + +Improve line offsets that are reported by SyntaxError. Improve error messages +for a few situations, including mismatched parenthesis. diff --git a/pypy/module/cpyext/test/test_bufferobject.py b/pypy/module/cpyext/test/test_bufferobject.py --- a/pypy/module/cpyext/test/test_bufferobject.py +++ b/pypy/module/cpyext/test/test_bufferobject.py @@ -4,6 +4,7 @@ from pypy.module.cpyext.api import PyObject class AppTestBufferObject(AppTestCpythonExtensionBase): + def test_FromMemory(self): module = self.import_extension('foo', [ ("get_FromMemory", "METH_NOARGS", @@ -62,3 +63,61 @@ a = array.array('c', 'text') b = buffer(a) assert module.roundtrip(b) == 'text' + + + def test_issue2752(self): + iterations = 10 + if self.runappdirect: + iterations = 2000 + module = self.import_extension('foo', [ + ("test_mod", 'METH_VARARGS', + """ + PyObject *obj; + Py_buffer bp; + if (!PyArg_ParseTuple(args, "O", &obj)) + return NULL; + + if (PyObject_GetBuffer(obj, &bp, PyBUF_SIMPLE) == -1) + return NULL; + + if (((unsigned char*)bp.buf)[0] != '0') { + void * buf = (void*)bp.buf; + unsigned char val[4]; + char * s = PyString_AsString(obj); + memcpy(val, bp.buf, 4); + PyBuffer_Release(&bp); + if (PyObject_GetBuffer(obj, &bp, PyBUF_SIMPLE) == -1) + return NULL; + PyErr_Format(PyExc_ValueError, + "mismatch: %p [%x %x %x %x...] now %p [%x %x %x %x...] as str '%s'", + buf, val[0], val[1], val[2], val[3], + (void *)bp.buf, + ((unsigned char*)bp.buf)[0], + ((unsigned char*)bp.buf)[1], + ((unsigned char*)bp.buf)[2], + ((unsigned char*)bp.buf)[3], + s); + PyBuffer_Release(&bp); + return NULL; + } + + PyBuffer_Release(&bp); + Py_RETURN_NONE; + """), + ]) + bufsize = 4096 + def getdata(bufsize): + data = b'01234567' + for x in range(18): + data += data + if len(data) >= bufsize: + break + return data + for j in range(iterations): + block = getdata(bufsize) + assert block[:8] == '01234567' + try: + module.test_mod(block) + except ValueError as e: + print("%s at it=%d" % (e, j)) + assert False diff --git a/pypy/tool/pytest/genreportdata.py b/pypy/tool/pytest/genreportdata.py deleted file mode 100755 --- a/pypy/tool/pytest/genreportdata.py +++ /dev/null @@ -1,30 +0,0 @@ -#! /usr/bin/env python -import py -import sys - -mydir = py.path.local(__file__).dirpath().realpath() -from pypy.tool.pytest import htmlreport -from pypy.tool.pytest import confpath - -if __name__ == '__main__': - if len(sys.argv) > 1: - testresultdir = py.path.local(sys.argv[1]) - assert testresultdir.check(dir=1) - else: - testresultdir = confpath.testresultdir - assert testresultdir.check(dir=1) - try: - resultwc = py.path.svnwc(testresultdir) - print "updating", resultwc - resultwc.update() - except (KeyboardInterrupt, RuntimeError): - raise - except Exception as e: #py.process.ExecutionFailed,e: - print >> sys.stderr, "Warning: ",e #Subversion update failed" - - print "traversing", mydir - rep = htmlreport.HtmlReport(testresultdir) - rep.parselatest() - - print "making html files" - rep.makeindex(testresultdir.join('index.html')) diff --git a/pypy/tool/pytest/htmlreport.py b/pypy/tool/pytest/htmlreport.py deleted file mode 100644 --- a/pypy/tool/pytest/htmlreport.py +++ /dev/null @@ -1,239 +0,0 @@ -#! /usr/bin/env python - -""" -the html test reporter - -""" -import sys, os, re -import pprint -import py -from pypy.tool.pytest import result -from pypy.tool.pytest.overview import ResultCache - -# -# various interesting path objects -# - -html = py.xml.html -NBSP = py.xml.raw(" ") - -class HtmlReport(object): - def __init__(self, resultdir): - self.resultcache = ResultCache(resultdir) - - def parselatest(self): - self.resultcache.parselatest() - - # - # rendering - # - - def render_latest_table(self, results): - table = html.table( - [html.th(x, align='left') - for x in ("failure", "filename", "revision", - "user", "platform", "elapsed", - "options", "last error line" - )], - ) - r = results[:] - def f(x, y): - xnum = x.isok() and 1 or (x.istimeout() and 2 or 3) - ynum = y.isok() and 1 or (y.istimeout() and 2 or 3) - res = -cmp(xnum, ynum) - if res == 0: - return cmp(x['execution-time'], y['execution-time']) - return res - r.sort(f) - for result in r: - table.append(self.render_result_row(result)) - return table - - def render_result_row(self, result): - dp = py.path.local(result['fspath']) - - options = " ".join([x for x in result.get('options', []) if x!= 'core']) - if not options: - options = NBSP - - failureratio = 100 * (1.0 - result.ratio_of_passed()) - self.data[result.testname] = failureratio - return html.tr( - html.td("%.2f%%" % failureratio, - style = "background-color: %s" % (getresultcolor(result),)), - html.td(self.render_test_references(result)), - html.td(result['pypy-revision']), - html.td(result['userhost'][:15]), - html.td(result['platform']), - html.td("%.2fs" % result['execution-time']), - html.td(options), - html.td(result.repr_short_error() or NBSP) - ) - - def getrelpath(self, p): - return p.relto(self.indexpath.dirpath()) - - def render_test_references(self, result): - dest = self.make_single_test_result(result) - #XXX: ask hg for differences between test and vendor branch - modified = result.ismodifiedtest() and " [mod]" or "" - return html.div(html.a(result.path.purebasename + modified, - href=self.getrelpath(dest)), - style="background-color: transparent") - - def make_single_test_result(self, result): - cache = self.indexpath.dirpath('.cache', result['userhost'][:15]) - cache.ensure(dir=1) - dest = cache.join(result.path.basename).new(ext='.html') - doc = ViewResult(result) - doc.writetopath(dest) - return dest - - def getcorelists(self): - def iscore(result): - return 'core' in result.get('options', []) - coretests = [] - noncoretests = [] - for name in self.resultcache.getnames(): - result = self.resultcache.getlatestrelevant(name) - if iscore(result): - coretests.append(result) - else: - noncoretests.append(result) - return coretests, noncoretests - - # generate html files - # - def makeindex(self, indexpath, detail="PyPy - latest"): - self.indexpath = indexpath - self.data = {} - doc = Document(title='pypy test results') - body = doc.body - coretests, noncoretests = self.getcorelists() - body.append(html.h2("%s compliance test results - " - "core tests" % detail)) - - body.append(self.render_test_summary('core', coretests)) - body.append(self.render_latest_table(coretests)) - body.append( - html.h2("%s compliance test results - non-core tests" % detail)) - body.append(self.render_test_summary('noncore', noncoretests)) - body.append(self.render_latest_table(noncoretests)) - doc.writetopath(indexpath) - datapath = indexpath.dirpath().join('data') - d = datapath.open('w') - print >>d, "data = ", - pprint.pprint(self.data, stream=d) - d.close() - self.data = None - - def render_test_summary(self, tag, tests): - ok = len([x for x in tests if x.isok()]) - err = len([x for x in tests if x.iserror()]) - to = len([x for x in tests if x.istimeout()]) - numtests = ok + err + to - assert numtests == len(tests) - assert numtests - - t = html.table() - sum100 = numtests / 100.0 - def row(*args): - return html.tr(*[html.td(arg) for arg in args]) - - sum_passed = sum([x.ratio_of_passed() for x in tests]) - compliancy = sum_passed/sum100 - self.data['%s-compliancy' % tag] = compliancy - t.append(row(html.b("tests compliancy"), - html.b("%.2f%%" % (compliancy,)))) - - passed = ok/sum100 - self.data['%s-passed' % tag] = passed - t.append(row("testmodules passed completely", "%.2f%%" % passed)) - failed = err/sum100 - self.data['%s-failed' % tag] = failed - t.append(row("testmodules (partially) failed", "%.2f%%" % failed)) - timedout = to/sum100 - self.data['%s-timedout' % tag] = timedout - t.append(row("testmodules timeout", "%.2f%%" % timedout)) - return t - -class Document(object): - def __init__(self, title=None): - self.body = html.body() - self.head = html.head() - self.doc = html.html(self.head, self.body) - if title is not None: - self.head.append( - html.meta(name="title", content=title)) - self.head.append( - html.link(rel="Stylesheet", type="text/css", href="/pypy/default.css")) - - def writetopath(self, p): - assert p.ext == '.html' - self.head.append( - html.meta(name="Content-Type", content="text/html;charset=UTF-8") - ) - s = self.doc.unicode().encode('utf-8') - p.write(s) - -def getresultcolor(result): - if result.isok(): - color = "#00ee00" - elif result.iserror(): - color = "#ee0000" - elif result.istimeout: - color = "#0000ee" - else: - color = "#444444" - return color - -class ViewResult(Document): - def __init__(self, result): - title = "%s testresult" % (result.path.purebasename,) - super(ViewResult, self).__init__(title=title) - color = getresultcolor(result) - self.body.append(html.h2(title, - style="background-color: %s" % color)) - self.body.append(self.render_meta_info(result)) - - for name in ('reportdiff', 'stdout', 'stderr'): - try: - text = result.getnamedtext(name) - except KeyError: - continue - self.body.append(html.h3(name)) - self.body.append(html.pre(text)) - - def render_meta_info(self, result): - t = html.table() - items = result.items() - items.sort() - for name, value in items: - if name.lower() == name: - t.append(html.tr( - html.td(name), html.td(value))) - return t - -class TestOfHtmlReportClass: - def setup_class(cls): - py.test.skip('needs move to own test file') - cls.testresultdir = confpath.testresultdir - cls.rep = rep = HtmlReport() - rep.parse_all(cls.testresultdir) - - def test_pickling(self): - # test pickling of report - tempdir = py.test.ensuretemp('reportpickle') - picklepath = tempdir.join('report.pickle') - picklepath.dump(self.rep) - x = picklepath.load() - assert len(x.results) == len(self.rep.results) - - def test_render_latest(self): - t = self.rep.render_latest_table(self.rep.results) - assert unicode(t) - -mydir = py.path.local(__file__).dirpath() - -def getpicklepath(): - return mydir.join('.htmlreport.pickle') diff --git a/pypy/tool/pytest/overview.py b/pypy/tool/pytest/overview.py deleted file mode 100644 --- a/pypy/tool/pytest/overview.py +++ /dev/null @@ -1,56 +0,0 @@ -from pypy.tool.pytest import result -import sys - -class ResultCache: - def __init__(self, resultdir): - self.resultdir = resultdir - self.name2result = {} - - def parselatest(self): - def filefilter(p): - return p.check(fnmatch='test_*.txt', file=1) - def rec(p): - return p.check(dotfile=0) - for x in self.resultdir.visit(filefilter, rec): - self.parse_one(x) - - def parse_one(self, resultpath): - try: - res = result.ResultFromMime(resultpath) - ver = res['testreport-version'] - if ver != "1.1" and ver != "1.1.1": - raise TypeError - except TypeError: # xxx - print >>sys.stderr, "could not parse %s" % resultpath - return - name = res.testname - print name - self.name2result.setdefault(name, []).append(res) - return res - - def getnames(self): - return self.name2result.keys() - - def getlatest(self, name, timeout=0, error=0, ok=0): - l = [] - resultlist = self.name2result[name] - maxrev = 0 - maxresult = None - for res in resultlist: - resrev = res['pypy-revision'] - if resrev == 'unknown': - continue - if resrev <= maxrev: - continue - if timeout or error or ok: - if not (timeout and res.istimeout() or - error and res.iserror() or - ok and res.isok()): - continue - maxrev = resrev - maxresult = res - return maxresult - - def getlatestrelevant(self, name): - # get the latest revision that did not time out. - return self.getlatest(name, error=1, ok=1) or self.getlatest(name) diff --git a/pypy/tool/pytest/result.py b/pypy/tool/pytest/result.py deleted file mode 100644 --- a/pypy/tool/pytest/result.py +++ /dev/null @@ -1,215 +0,0 @@ -import sys -import py -import re - -class Result(object): - def __init__(self, init=True): - self._headers = {} - self._blocks = {} - self._blocknames = [] - if init: - stdinit(self) - - def __setitem__(self, name, value): - self._headers[name.lower()] = value - - def __getitem__(self, name): - return self._headers[name.lower()] - - def get(self, name, default): - return self._headers.get(name, default) - - def __delitem__(self, name): - del self._headers[name.lower()] - - def items(self): - return self._headers.items() - - def addnamedtext(self, name, text): - assert isinstance(text, basestring) - assert isinstance(name, str) - self._blocknames.append(name) - self._blocks[name] = text - - def getnamedtext(self, name): - return self._blocks[name] - - def repr_short_error(self): - if not self.isok(): - if 'reportdiff' in self._blocks: - return "output comparison failed, see reportdiff" - else: - text = self.getnamedtext('stderr') - lines = text.strip().split('\n') - if lines: - return lines[-1] - - def repr_mimemessage(self): - from email.MIMEMultipart import MIMEMultipart - from email.MIMEText import MIMEText - - outer = MIMEMultipart() - items = self._headers.items() - items.sort() - reprs = {} - for name, value in items: - assert ':' not in name - chars = map(ord, name) - assert min(chars) >= 33 and max(chars) <= 126 - outer[name] = str(value) - if not isinstance(value, str): - typename = type(value).__name__ - assert typename in vars(py.std.__builtin__) - reprs[name] = typename - - outer['_reprs'] = repr(reprs) - - for name in self._blocknames: - text = self._blocks[name] - m = MIMEText(text) - m.add_header('Content-Disposition', 'attachment', filename=name) - outer.attach(m) - return outer - - def grep_nr(self,text,section='stdout'): - stdout = self._blocks[section] - find = re.search('%s(?P\d+)'%text,stdout) - if find: - return float(find.group('nr')) - return 0. - - def ratio_of_passed(self): - if self.isok(): - return 1. - elif self.istimeout(): - return 0. - else: - nr = self.grep_nr('Ran ') - if nr > 0: - return (nr - (self.grep_nr('errors=') + self.grep_nr('failures=')))/nr - else: - passed = self.grep_nr('TestFailed: ',section='stderr') - run = self.grep_nr('TestFailed: \d+/',section='stderr') - if run > 0: - return passed/run - else: - run = self.grep_nr('TestFailed: \d+ of ',section='stderr') - if run > 0 : - return (run-passed)/run - else: - return 0.0 - - def isok(self): - return self['outcome'].lower() == 'ok' - - def iserror(self): - return self['outcome'].lower()[:3] == 'err' or self['outcome'].lower() == 'fail' - - def istimeout(self): - return self['outcome'].lower() == 't/o' - -# XXX backward compatibility -def sanitize(msg, path): - if 'exit-status' in msg.keys(): - return msg - f = open(str(path), 'r') - msg = f.read() - f.close() - for broken in ('exit status', 'cpu model', 'cpu mhz'): - valid = broken.replace(' ','-') - invalid = msg.find(broken+':') - msg = (msg[:invalid] + valid + - msg[invalid+len(valid):]) - from email import message_from_string - msg = message_from_string(msg) - return msg - -def sanitize_reprs(reprs): - if 'exit status' in reprs: - reprs['exit-status'] = reprs.pop('exit status') - -class ResultFromMime(Result): - def __init__(self, path): - super(ResultFromMime, self).__init__(init=False) - f = open(str(path), 'r') - from email import message_from_file - msg = message_from_file(f) - f.close() - msg = sanitize(msg, path) - # XXX security wise evil (keep in mind once we accept reporsts - # from anonymous - #print msg['_reprs'] - self._reprs = eval(msg['_reprs']) - del msg['_reprs'] - sanitize_reprs(self._reprs) - for name, value in msg.items(): - if name in self._reprs: - value = eval(value) # XXX security - self._headers[name] = value - self.fspath = self['fspath'] - if self['platform'] == 'win32' and '\\' in self.fspath: - self.testname = self.fspath.split('\\')[-1] - else: - self.testname = self.fspath.split('/')[-1] - #if sys.platform != 'win32' and '\\' in self.fspath: - # self.fspath = py.path.local(self['fspath'].replace('\\' - self.path = path - - payload = msg.get_payload() - if payload: - for submsg in payload: - assert submsg.get_content_type() == 'text/plain' - fn = submsg.get_filename() - assert fn - # XXX we need to deal better with encodings to - # begin with - content = submsg.get_payload() - for candidate in 'utf8', 'latin1': - try: - text = unicode(content, candidate) - except UnicodeDecodeError: - continue - else: - unicode(content, candidate) - self.addnamedtext(fn, text) - - def ismodifiedtest(self): - # XXX we need proper cross-platform paths! - return 'modified' in self.fspath - - def __repr__(self): - return '<%s (%s) %r rev=%s>' %(self.__class__.__name__, - self['outcome'], - self.fspath, - self['pypy-revision']) - -def stdinit(result): - import getpass - import socket - try: - username = getpass.getuser() - except: - username = 'unknown' - userhost = '%s@%s' % (username, socket.gethostname()) - result['testreport-version'] = "1.1.1" - result['userhost'] = userhost - result['platform'] = sys.platform - result['python-version-info'] = sys.version_info - info = try_getcpuinfo() - if info is not None: - result['cpu-model'] = info.get('model name', "unknown") - result['cpu-mhz'] = info.get('cpu mhz', 'unknown') -# -# -# -def try_getcpuinfo(): - if sys.platform.startswith('linux'): - cpuinfopath = py.path.local('/proc/cpuinfo') - if cpuinfopath.check(file=1): - d = {} - for line in cpuinfopath.readlines(): - if line.strip(): - name, value = line.split(':', 1) - name = name.strip().lower() - d[name] = value.strip() - return d diff --git a/pypy/tool/pytest/test/data/test___all__.txt b/pypy/tool/pytest/test/data/test___all__.txt deleted file mode 100644 --- a/pypy/tool/pytest/test/data/test___all__.txt +++ /dev/null @@ -1,94 +0,0 @@ -Content-Type: multipart/mixed; boundary="===============0790678169==" -MIME-Version: 1.0 -execution-time: 1445.14346004 -exit status: 1 -fspath: /Users/anderslehmann/pypy/lib-python/2.4.1/test/test___all__.py -options: ['core', '_sre'] -outcome: T/O -platform: darwin -pypy-revision: 16114 -python-version-info: (2, 4, 1, 'final', 0) -startdate: Wed Aug 17 23:51:59 2005 -testreport-version: 1.1 -timeout: 1369.0 -userhost: anderslehmann at anders-lehmanns-15-powerbook-g4.local -_reprs: {'execution-time': 'float', 'python-version-info': 'tuple', - 'options': 'list', 'timeout': 'float', 'pypy-revision': 'int', - 'exit status': 'int'} - ---===============0790678169== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stdout" - -test_all (__main__.AllTest) ... ERROR - -====================================================================== -ERROR: test_all (__main__.AllTest) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test___all__.py", line 163, in test_all - self.check_all("tty") - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test___all__.py", line 26, in check_all - "%s has no __all__ attribute" % modname) - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_support.py", line 208, in verify - raise TestFailed(reason) -TestFailed: tty has no __all__ attribute - ----------------------------------------------------------------------- - ---===============0790678169== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stderr" - -faking -Loading grammar /Users/anderslehmann/pypy/pypy/interpreter/pyparser/data/Grammar2.4 -faking -faking -faking -fake-wrapping interp file ', mode 'w' at 0x12068> -fake-wrapping interp file ', mode 'w' at 0x120b0> -fake-wrapping interp file ', mode 'r' at 0x12020> -faking -faking -faking -faking -==========================timedout========================== -Traceback (application-level): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test___all__.py", line 192 in - test_main() - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test___all__.py", line 189 in test_main - test_support.run_unittest(AllTest) - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_support.py", line 290 in run_unittest - run_suite(suite, testclass) - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_support.py", line 262 in run_suite - result = runner.run(suite) -Traceback (application-level): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/atexit.py", line 29 in _run_exitfuncs - print >> sys.stderr, "Error in atexit._run_exitfuncs:" -KeyboardInterrupt -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/pypy/tool/alarm.py", line 43, in ? - execfile(_main_with_alarm(finished)) - File "/Users/anderslehmann/pypy/pypy/bin/py.py", line 206, in ? - sys.exit(main_(sys.argv)) - File "/Users/anderslehmann/pypy/pypy/bin/py.py", line 115, in main_ - if not main.run_toplevel(space, doit, verbose=Options.verbose): - File "/Users/anderslehmann/pypy/pypy/interpreter/main.py", line 150, in run_toplevel - operationerr.print_application_traceback(space) - File "/Users/anderslehmann/pypy/pypy/interpreter/error.py", line 83, in print_application_traceback - self.print_app_tb_only(file) - File "/Users/anderslehmann/pypy/pypy/interpreter/error.py", line 104, in print_app_tb_only - l = linecache.getline(fname, lineno) - File "/Library/Frameworks/Python.framework/Versions/2.4/lib/python2.4/linecache.py", line 14, in getline - lines = getlines(filename) - File "/Library/Frameworks/Python.framework/Versions/2.4/lib/python2.4/linecache.py", line 40, in getlines - return updatecache(filename) - File "/Library/Frameworks/Python.framework/Versions/2.4/lib/python2.4/linecache.py", line 101, in updatecache - lines = fp.readlines() -KeyboardInterrupt - ---===============0790678169==-- \ No newline at end of file diff --git a/pypy/tool/pytest/test/data/test_compile.txt b/pypy/tool/pytest/test/data/test_compile.txt deleted file mode 100644 --- a/pypy/tool/pytest/test/data/test_compile.txt +++ /dev/null @@ -1,111 +0,0 @@ -Content-Type: multipart/mixed; boundary="===============2137793924==" -MIME-Version: 1.0 -execution-time: 34.8464071751 -exit status: 1 -fspath: /Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py -options: ['core', '_sre'] -outcome: ERR -platform: darwin -pypy-revision: 16114 -python-version-info: (2, 4, 1, 'final', 0) -startdate: Thu Aug 18 03:08:18 2005 -testreport-version: 1.1 -timeout: 1521.0 -userhost: anderslehmann at anders-lehmanns-15-powerbook-g4.local -_reprs: {'execution-time': 'float', 'python-version-info': 'tuple', - 'options': 'list', 'timeout': 'float', 'pypy-revision': 'int', - 'exit status': 'int'} - ---===============2137793924== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stdout" - -test_argument_handling (__main__.TestSpecifics) ... FAIL -test_argument_order (__main__.TestSpecifics) ... FAIL -test_complex_args (__main__.TestSpecifics) ... ok -test_debug_assignment (__main__.TestSpecifics) ... FAIL -test_duplicate_global_local (__main__.TestSpecifics) ... ok -test_exec_with_general_mapping_for_locals (__main__.TestSpecifics) ... ok -test_float_literals (__main__.TestSpecifics) ... ok -test_for_distinct_code_objects (__main__.TestSpecifics) ... ok -test_import (__main__.TestSpecifics) ... FAIL -test_indentation (__main__.TestSpecifics) ... ok -test_literals_with_leading_zeroes (__main__.TestSpecifics) ... ok -test_none_assignment (__main__.TestSpecifics) ... FAIL -test_sequence_unpacking_error (__main__.TestSpecifics) ... ok -test_syntax_error (__main__.TestSpecifics) ... ok -test_unary_minus (__main__.TestSpecifics) ... ok - -====================================================================== -FAIL: test_argument_handling (__main__.TestSpecifics) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 18, in test_argument_handling - self.assertRaises(SyntaxError, eval, 'lambda a,a:0') -AssertionError: SyntaxError not raised - -====================================================================== -FAIL: test_argument_order (__main__.TestSpecifics) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 127, in test_argument_order - self.fail("non-default args after default") -AssertionError: non-default args after default - -====================================================================== -FAIL: test_debug_assignment (__main__.TestSpecifics) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 10, in test_debug_assignment - self.assertRaises(SyntaxError, compile, '__debug__ = 1', '?', 'single') -AssertionError: SyntaxError not raised - -====================================================================== -FAIL: test_import (__main__.TestSpecifics) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 253, in test_import - self.assertRaises(SyntaxError, compile, stmt, 'tmp', 'exec') -AssertionError: SyntaxError not raised - -====================================================================== -FAIL: test_none_assignment (__main__.TestSpecifics) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 211, in test_none_assignment - self.assertRaises(SyntaxError, compile, stmt, 'tmp', 'single') -AssertionError: SyntaxError not raised - ----------------------------------------------------------------------- -Ran 15 tests in 14.363s - -FAILED (failures=5) - ---===============2137793924== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stderr" - -faking -Loading grammar /Users/anderslehmann/pypy/pypy/interpreter/pyparser/data/Grammar2.4 -faking -faking -faking -fake-wrapping interp file ', mode 'w' at 0x12068> -fake-wrapping interp file ', mode 'w' at 0x120b0> -fake-wrapping interp file ', mode 'r' at 0x12020> -Traceback (application-level): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 268 in - test_main() - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 265 in test_main - test_support.run_unittest(TestSpecifics) - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_support.py", line 290 in run_unittest - run_suite(suite, testclass) - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_support.py", line 274 in run_suite - raise TestFailed(msg) -TestFailed: errors occurred in __main__.TestSpecifics - ---===============2137793924==-- \ No newline at end of file diff --git a/pypy/tool/pytest/test/data/test_descr.txt b/pypy/tool/pytest/test/data/test_descr.txt deleted file mode 100644 --- a/pypy/tool/pytest/test/data/test_descr.txt +++ /dev/null @@ -1,233 +0,0 @@ -Content-Type: multipart/mixed; boundary="===============1265023865==" -MIME-Version: 1.0 -execution-time: 4098.8407588 -exit status: 1 -fspath: /Users/anderslehmann/pypy/lib-python/modified-2.4.1/test/test_descr.py -options: ['oldstyle', 'core'] -outcome: ERR -platform: darwin -pypy-revision: 16388 -python-version-info: (2, 4, 1, 'final', 0) -startdate: Wed Aug 24 16:54:12 2005 -testreport-version: 1.1 -timeout: 10000.0 -userhost: anderslehmann at anders-lehmanns-15-powerbook-g4.local -_reprs: {'execution-time': 'float', 'python-version-info': 'tuple', - 'options': 'list', 'timeout': 'float', 'pypy-revision': 'int', - 'exit status': 'int'} - ---===============1265023865== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stdout" - -**************************************** ---> weakref_segfault FAILURE(0/92) _weakref -**************************************** ---> do_this_first OK(1/92) -**************************************** ---> class_docstrings OK(2/92) -**************************************** ---> lists FAILURE(2/92) type object 'list' has no attribute '__getslice__' -**************************************** ---> dicts FAILURE(2/92) type object 'dict' has no attribute '__cmp__' -**************************************** ---> dict_constructor OK(3/92) -**************************************** ---> test_dir OK(4/92) -**************************************** ---> ints OK(5/92) -**************************************** ---> longs OK(6/92) -**************************************** ---> floats OK(7/92) -**************************************** ---> complexes OK(8/92) -**************************************** ---> spamlists FAILURE(8/92) xxsubtype -**************************************** ---> spamdicts FAILURE(8/92) xxsubtype -**************************************** ---> pydicts OK(9/92) -**************************************** ---> pylists OK(10/92) -**************************************** ---> metaclass OK(11/92) -**************************************** ---> pymods OK(12/92) -**************************************** ---> multi OK(13/92) -**************************************** ---> mro_disagreement FAILURE(13/92) Message 'cycle among base classes: A < B < A', expected 'Cannot create a consistent method resolution\norder (MRO) for bases ' -**************************************** ---> diamond OK(14/92) -**************************************** ---> ex5 OK(15/92) -**************************************** ---> monotonicity OK(16/92) -**************************************** ---> consistency_with_epg OK(17/92) -**************************************** ---> objects OK(18/92) -**************************************** ---> slots FAILURE(18/92) ['foo bar'] slots not caught -**************************************** ---> slotspecials FAILURE(18/92) test failed -**************************************** ---> dynamics OK(19/92) -**************************************** ---> errors FAILURE(19/92) inheritance from CFunction should be illegal -**************************************** ---> classmethods OK(20/92) -**************************************** ---> classmethods_in_c FAILURE(20/92) xxsubtype -**************************************** ---> staticmethods OK(21/92) -**************************************** ---> staticmethods_in_c FAILURE(21/92) xxsubtype -**************************************** ---> classic OK(22/92) -**************************************** ---> compattr OK(23/92) -**************************************** ---> newslot OK(24/92) -**************************************** ---> altmro OK(25/92) -**************************************** ---> overloading OK(26/92) -**************************************** ---> methods FAILURE(26/92) test failed -**************************************** ---> specials FAILURE(26/92) shouldn't allow .__cmp__(u'123', '123') -**************************************** ---> weakrefs FAILURE(26/92) 'module' object has no attribute 'ref' -**************************************** ---> properties FAILURE(26/92) expected TypeError from trying to set readonly '__doc__' attr on a property -**************************************** ---> supers OK(27/92) -**************************************** ---> inherits FAILURE(27/92) test failed -**************************************** ---> keywords FAILURE(27/92) descr__new__() got an unexpected keyword argument 'string' -**************************************** ---> restricted OK(28/92) -**************************************** ---> str_subclass_as_dict_key OK(29/92) -**************************************** ---> classic_comparisons OK(30/92) -**************************************** ---> rich_comparisons OK(31/92) -**************************************** ---> coercions OK(32/92) -**************************************** ---> descrdoc FAILURE(32/92) 'The most base type' == 'True if the file is closed' -**************************************** ---> setclass OK(33/92) -**************************************** ---> setdict OK(34/92) -**************************************** ---> pickles OK(35/92) -**************************************** ---> copies OK(36/92) -**************************************** ---> binopoverride OK(37/92) -**************************************** ---> subclasspropagation OK(38/92) -**************************************** ---> buffer_inherit OK(39/92) -**************************************** ---> str_of_str_subclass OK(40/92) -**************************************** ---> kwdargs OK(41/92) -**************************************** ---> delhook OK(42/92) -**************************************** ---> hashinherit OK(43/92) -**************************************** ---> strops OK(44/92) -**************************************** ---> deepcopyrecursive OK(45/92) -**************************************** ---> modules OK(46/92) -**************************************** ---> dictproxyiterkeys FAILURE(46/92) ['__dict__', '__module__', 'meth'] == ['__dict__', '__doc__', '__module__', '__weakref__', 'meth'] -**************************************** ---> dictproxyitervalues FAILURE(46/92) 3 == 5 -**************************************** ---> dictproxyiteritems FAILURE(46/92) ['__dict__', '__module__', 'meth'] == ['__dict__', '__doc__', '__module__', '__weakref__', 'meth'] -**************************************** ---> pickleslots OK(47/92) -**************************************** ---> funnynew OK(48/92) -**************************************** ---> imulbug OK(49/92) -**************************************** ---> docdescriptor OK(50/92) -**************************************** ---> string_exceptions FAILURE(50/92) string subclass allowed as exception -**************************************** ---> copy_setstate OK(51/92) -**************************************** ---> slices OK(52/92) -**************************************** ---> subtype_resurrection OK(53/92) -**************************************** ---> slottrash OK(54/92) -**************************************** ---> slotmultipleinheritance FAILURE(54/92) type object 'C' has no attribute '__basicsize__' -**************************************** ---> testrmul OK(55/92) -**************************************** ---> testipow OK(56/92) -**************************************** ---> test_mutable_bases FAILURE(56/92) readonly attribute -**************************************** ---> test_mutable_bases_with_failing_mro FAILURE(56/92) readonly attribute -**************************************** ---> test_mutable_bases_catch_mro_conflict OK(57/92) -**************************************** ---> mutable_names OK(58/92) -**************************************** ---> subclass_right_op OK(59/92) -**************************************** ---> dict_type_with_metaclass OK(60/92) -**************************************** ---> meth_class_get FAILURE(60/92) shouldn't have allowed descr.__get__(None, None) -**************************************** ---> isinst_isclass OK(61/92) -**************************************** ---> proxysuper OK(62/92) -**************************************** ---> carloverre OK(63/92) -**************************************** ---> filefault OK(64/92) -**************************************** ---> vicious_descriptor_nonsense OK(65/92) -**************************************** ---> test_init FAILURE(65/92) did not test __init__() for None return - ---===============1265023865== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stderr" - -faking -Loading grammar /Users/anderslehmann/pypy/pypy/interpreter/pyparser/data/Grammar2.4 -faking -faking -faking -fake-wrapping interp file ', mode 'w' at 0x12068> -fake-wrapping interp file ', mode 'w' at 0x120b0> -fake-wrapping interp file ', mode 'r' at 0x12020> -faking -faking -Traceback (application-level): - File "/Users/anderslehmann/pypy/pypy/tool/pytest/regrverbose.py", line 12 in - indirect_test() - File "/Users/anderslehmann/pypy/lib-python/modified-2.4.1/test/test_descr.py", line 4102 in test_main - raise TestFailed, "%d/%d" % (success, n) -TestFailed: 65/92 - ---===============1265023865==-- \ No newline at end of file diff --git a/pypy/tool/pytest/test/data/test_generators.txt b/pypy/tool/pytest/test/data/test_generators.txt deleted file mode 100644 --- a/pypy/tool/pytest/test/data/test_generators.txt +++ /dev/null @@ -1,317 +0,0 @@ -Content-Type: multipart/mixed; boundary="===============1565511160==" -MIME-Version: 1.0 -cpu mhz: unknown -cpu model: unknown -execution-time: 671.678878069 -exit status: 1 -fspath: /home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py -options: ['core'] -outcome: ERR -platform: linux2 -pypy-revision: 16123 -python-version-info: (2, 4, 1, 'final', 0) -startdate: Thu Aug 18 02:08:13 2005 -testreport-version: 1.1 -timeout: 3136.0 -userhost: xoraxax at tick -_reprs: {'execution-time': 'float', 'python-version-info': 'tuple', - 'options': 'list', 'timeout': 'float', 'pypy-revision': 'int', - 'exit status': 'int'} - ---===============1565511160== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stdout" - -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.weakref -Failed example: - import weakref -Exception raised: - Traceback (most recent call last): - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/doctest.py", line 1243, in __run - compileflags, 1) in test.globs - File "", line 1, in - import weakref - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/weakref.py", line 14, in - from _weakref import ( - ImportError: _weakref -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.weakref -Failed example: - wr = weakref.ref(gen) -Exception raised: - Traceback (most recent call last): - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/doctest.py", line 1243, in __run - compileflags, 1) in test.globs - File "", line 1, in - wr = weakref.ref(gen) - NameError: global name 'weakref' is not defined -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.weakref -Failed example: - wr() is gen -Exception raised: - Traceback (most recent call last): - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/doctest.py", line 1243, in __run - compileflags, 1) in test.globs - File "", line 1, in - wr() is gen - NameError: global name 'wr' is not defined -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.weakref -Failed example: - p = weakref.proxy(gen) -Exception raised: - Traceback (most recent call last): - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/doctest.py", line 1243, in __run - compileflags, 1) in test.globs - File "", line 1, in - p = weakref.proxy(gen) - NameError: global name 'weakref' is not defined -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.weakref -Failed example: - wr = weakref.ref(gi) -Exception raised: - Traceback (most recent call last): - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/doctest.py", line 1243, in __run - compileflags, 1) in test.globs - File "", line 1, in - wr = weakref.ref(gi) - NameError: global name 'weakref' is not defined -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.weakref -Failed example: - wr() is gi -Exception raised: - Traceback (most recent call last): - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/doctest.py", line 1243, in __run - compileflags, 1) in test.globs - File "", line 1, in - wr() is gi - NameError: global name 'wr' is not defined -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.weakref -Failed example: - p = weakref.proxy(gi) -Exception raised: - Traceback (most recent call last): - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/doctest.py", line 1243, in __run - compileflags, 1) in test.globs - File "", line 1, in - p = weakref.proxy(gi) - NameError: global name 'weakref' is not defined -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.weakref -Failed example: - list(p) -Exception raised: - Traceback (most recent call last): - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/doctest.py", line 1243, in __run - compileflags, 1) in test.globs - File "", line 1, in - list(p) - NameError: global name 'p' is not defined -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.pep -Failed example: - k.next() -Expected: - Traceback (most recent call last): - File "", line 1, in ? - File "", line 2, in g - File "", line 2, in f - ZeroDivisionError: integer division or modulo by zero -Got: - Traceback (most recent call last): - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/doctest.py", line 1243, in __run - compileflags, 1) in test.globs - File "", line 1, in - k.next() - File "", line 2, in g - yield f() # the zero division exception propagates - File "", line 2, in f - return 1//0 - ZeroDivisionError: integer division by zero -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.syntax -Failed example: - def f(): - return 22 - yield 1 -Expected: - Traceback (most recent call last): - .. - SyntaxError: 'return' with argument inside generator (, line 2) -Got nothing -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.syntax -Failed example: - def f(): - yield 1 - return 22 -Expected: - Traceback (most recent call last): - .. - SyntaxError: 'return' with argument inside generator (, line 3) -Got nothing -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.syntax -Failed example: - def f(): - yield 1 - return None -Expected: - Traceback (most recent call last): - .. - SyntaxError: 'return' with argument inside generator (, line 3) -Got nothing -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.syntax -Failed example: - def f(): - try: - yield 1 - finally: - pass -Expected: - Traceback (most recent call last): - .. - SyntaxError: 'yield' not allowed in a 'try' block with a 'finally' clause (, line 3) -Got nothing -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.syntax -Failed example: - def f(): - try: - try: - 1//0 - except ZeroDivisionError: - yield 666 # bad because *outer* try has finally - except: - pass - finally: - pass -Expected: - Traceback (most recent call last): - ... - SyntaxError: 'yield' not allowed in a 'try' block with a 'finally' clause (, line 6) -Got nothing -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.syntax -Failed example: - def f(): - yield -Expected: - Traceback (most recent call last): - SyntaxError: invalid syntax -Got: - Traceback (most recent call last): - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/doctest.py", line 1243, in __run - compileflags, 1) in test.globs - File "", line 2 - def f(): - ^ - SyntaxError: error -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.syntax -Failed example: - def f(): - if 0: - yield -Expected: - Traceback (most recent call last): - SyntaxError: invalid syntax -Got: - Traceback (most recent call last): - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/doctest.py", line 1243, in __run - compileflags, 1) in test.globs - File "", line 3 - def f(): - ^ - SyntaxError: error -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.syntax -Failed example: - type(f()) -Expected: - -Got: - -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.syntax -Failed example: - type(f()) -Expected: - -Got: - -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.syntax -Failed example: - type(f()) -Expected: - -Got: - -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.syntax -Failed example: - type(f()) -Expected: - -Got: - -********************************************************************** -File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line ?, in test.test_generators.__test__.syntax -Failed example: - def f(): - if 0: - lambda x: x # shouldn't trigger here - return # or here - def f(i): - return 2*i # or here - if 0: - return 3 # but *this* sucks (line 8) - if 0: - yield 2 # because it's a generator -Expected: - Traceback (most recent call last): - SyntaxError: 'return' with argument inside generator (, line 8) -Got nothing -********************************************************************** -3 items had failures: - 1 of 22 in test.test_generators.__test__.pep - 12 of 29 in test.test_generators.__test__.syntax - 8 of 10 in test.test_generators.__test__.weakref -***Test Failed*** 21 failures. - ---===============1565511160== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stderr" - -faking -Loading grammar /home/contest/xoraxax/pypy-dist/pypy/interpreter/pyparser/data/Grammar2.4 -faking -faking -faking -fake-wrapping interp file ', mode 'w' at 0xf7fa3068> -fake-wrapping interp file ', mode 'w' at 0xf7fa30b0> -fake-wrapping interp file ', mode 'r' at 0xf7fa3020> -faking -faking -faking -Traceback (application-level): - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line 1405 in - test_main(1) - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_generators.py", line 1401 in test_main - test_support.run_doctest(test_generators, verbose) - File "/home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_support.py", line 319 in run_doctest - finally: -TestFailed: 21 of 154 doctests failed - ---===============1565511160==-- \ No newline at end of file diff --git a/pypy/tool/pytest/test/data/test_global.txt b/pypy/tool/pytest/test/data/test_global.txt deleted file mode 100644 --- a/pypy/tool/pytest/test/data/test_global.txt +++ /dev/null @@ -1,94 +0,0 @@ -Content-Type: multipart/mixed; boundary="===============1988336873==" -MIME-Version: 1.0 -cpu mhz: unknown -cpu model: unknown -execution-time: 12.6530230045 -exit status: 2 -fspath: /home/contest/xoraxax/pypy-dist/lib-python/2.4.1/test/test_global.py -options: ['core'] -outcome: ERROUT -platform: linux2 -pypy-revision: 16123 -python-version-info: (2, 4, 1, 'final', 0) -startdate: Thu Aug 18 02:12:59 2005 -testreport-version: 1.1 -timeout: 3844.0 -userhost: xoraxax at tick -_reprs: {'execution-time': 'float', 'python-version-info': 'tuple', - 'options': 'list', 'timeout': 'float', 'pypy-revision': 'int', - 'exit status': 'int'} - ---===============1988336873== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stdout" - -should have raised SyntaxError: -def wrong1(): - a = 1 - b = 2 - global a - global b - -should have raised SyntaxError: -def wrong2(): - print x - global x - -should have raised SyntaxError: -def wrong3(): - print x - x = 2 - global x - -as expected, no SyntaxError - ---===============1988336873== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stderr" - -faking -Loading grammar /home/contest/xoraxax/pypy-dist/pypy/interpreter/pyparser/data/Grammar2.4 -faking -faking -faking -fake-wrapping interp file ', mode 'w' at 0xf7fa3068> -fake-wrapping interp file ', mode 'w' at 0xf7fa30b0> -fake-wrapping interp file ', mode 'r' at 0xf7fa3020> -faking - ---===============1988336873== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="reportdiff" - -********************************************************************** -*** mismatch between lines 2-4 of expected output and lines 2-19 of actual output: -- got SyntaxError as expected -- got SyntaxError as expected -- got SyntaxError as expected -+ should have raised SyntaxError: -+ def wrong1(): -+ a = 1 -+ b = 2 -+ global a -+ global b -+ -+ should have raised SyntaxError: -+ def wrong2(): -+ print x -+ global x -+ -+ should have raised SyntaxError: -+ def wrong3(): -+ print x -+ x = 2 -+ global x -+ -********************************************************************** - ---===============1988336873==-- \ No newline at end of file diff --git a/pypy/tool/pytest/test/data/test_sys.txt b/pypy/tool/pytest/test/data/test_sys.txt deleted file mode 100644 --- a/pypy/tool/pytest/test/data/test_sys.txt +++ /dev/null @@ -1,61 +0,0 @@ -Content-Type: multipart/mixed; boundary="===============1380540766==" -MIME-Version: 1.0 -cpu mhz: unknown -cpu model: unknown -execution-time: 22.5916008949 -exit status: 0 -fspath: /home/contest/xoraxax/pypy-dist/lib-python/modified-2.4.1/test/test_sys.py -options: ['core'] -outcome: OK -platform: linux2 -pypy-revision: 16123 -python-version-info: (2, 4, 1, 'final', 0) -startdate: Thu Aug 18 04:16:48 2005 -testreport-version: 1.1 -timeout: 3364.0 -userhost: xoraxax at tick -_reprs: {'execution-time': 'float', 'python-version-info': 'tuple', - 'options': 'list', 'timeout': 'float', 'pypy-revision': 'int', - 'exit status': 'int'} - ---===============1380540766== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stdout" - -test_attributes (__main__.SysModuleTest) ... ok -test_custom_displayhook (__main__.SysModuleTest) ... ok -test_dlopenflags (__main__.SysModuleTest) ... ok -test_exc_clear (__main__.SysModuleTest) ... ok -test_exit (__main__.SysModuleTest) ... ok -test_getdefaultencoding (__main__.SysModuleTest) ... ok -test_getframe (__main__.SysModuleTest) ... ok -test_getwindowsversion (__main__.SysModuleTest) ... ok -test_lost_displayhook (__main__.SysModuleTest) ... ok -test_original_displayhook (__main__.SysModuleTest) ... ok -test_original_excepthook (__main__.SysModuleTest) ... ok -test_recursionlimit (__main__.SysModuleTest) ... ok -test_setcheckinterval (__main__.SysModuleTest) ... ok - ----------------------------------------------------------------------- -Ran 13 tests in 7.241s - -OK - ---===============1380540766== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stderr" - -faking -Loading grammar /home/contest/xoraxax/pypy-dist/pypy/interpreter/pyparser/data/Grammar2.4 -faking -faking -faking -fake-wrapping interp file ', mode 'w' at 0xf7fa3068> -fake-wrapping interp file ', mode 'w' at 0xf7fa30b0> -fake-wrapping interp file ', mode 'r' at 0xf7fa3020> - ---===============1380540766==-- \ No newline at end of file diff --git a/pypy/tool/pytest/test/test_new_count.py b/pypy/tool/pytest/test/test_new_count.py deleted file mode 100644 --- a/pypy/tool/pytest/test/test_new_count.py +++ /dev/null @@ -1,32 +0,0 @@ - -import py -#from pypy.tool.pytest.confpath import testresultdir -from pypy.tool.pytest.result import ResultFromMime -testpath = py.path.local(__file__).dirpath('data') - -class TestResultCache: - - def test_timeout(self): - test = ResultFromMime(testpath.join('test___all__.txt')) - assert test.ratio_of_passed() == 0. - - def test_passed(self): - test = ResultFromMime(testpath.join('test_sys.txt')) - assert test.ratio_of_passed() == 1. - - def test_unittest_partial(self): - test = ResultFromMime(testpath.join('test_compile.txt')) - assert test.ratio_of_passed() == 10./15 - - def test_doctest_of(self): - test = ResultFromMime(testpath.join('test_generators.txt')) - assert test.ratio_of_passed() == 133./154 - - def test_doctest_slash(self): - test = ResultFromMime(testpath.join('test_descr.txt')) - assert test.ratio_of_passed() == 65./92 - - def test_fail(self): - test = ResultFromMime(testpath.join('test_global.txt')) - assert test.ratio_of_passed() == 0. - diff --git a/pypy/tool/pytest/test/test_overview.py b/pypy/tool/pytest/test/test_overview.py deleted file mode 100644 --- a/pypy/tool/pytest/test/test_overview.py +++ /dev/null @@ -1,23 +0,0 @@ - -import py -from pypy.tool.pytest.confpath import testresultdir -from pypy.tool.pytest.overview import ResultCache - -class TestResultCache: - def setup_class(cls): - if not testresultdir.check(dir=1): - py.test.skip("testresult directory not checked out") - cls.rc = ResultCache(testresultdir) - cls.rc.parselatest() - - def test_getlatest_all(self): - for type in 'error', 'timeout', 'ok': - for name in self.rc.getnames(): - result = self.rc.getlatest(name, **{type:1}) - if result: - meth = getattr(result, 'is'+type) - assert meth() - - #def test_getlatest_datetime(self): - # result = self.rc.getlatest('test_datetime', ok=1) - # assert result diff --git a/pypy/tool/release/force-builds.py b/pypy/tool/release/force-builds.py --- a/pypy/tool/release/force-builds.py +++ b/pypy/tool/release/force-builds.py @@ -31,6 +31,9 @@ 'pypy-c-jit-linux-s390x', 'build-pypy-c-jit-linux-armhf-raspbian', 'build-pypy-c-jit-linux-armel', + 'rpython-linux-x86-32', + 'rpython-linux-x86-64' + 'rpython-win-x86-32' ] def get_user(): diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -732,14 +732,16 @@ def move_out_of_nursery(self, obj): # called twice, it should return the same shadow object, - # and not creating another shadow object - if self.header(obj).tid & GCFLAG_HAS_SHADOW: - shadow = self.nursery_objects_shadows.get(obj) - ll_assert(shadow != llmemory.NULL, - "GCFLAG_HAS_SHADOW but no shadow found") - return shadow - - return self._allocate_shadow(obj, copy=True) + # and not creating another shadow object. As a safety feature, + # when called on a non-nursery object, do nothing. + if not self.is_in_nursery(obj): + return obj + shadow = self._find_shadow(obj) + if (self.header(obj).tid & GCFLAG_SHADOW_INITIALIZED) == 0: + self.header(obj).tid |= GCFLAG_SHADOW_INITIALIZED + totalsize = self.get_size(obj) + llmemory.raw_memcopy(obj, shadow, totalsize) + return shadow def collect(self, gen=2): """Do a minor (gen=0), start a major (gen=1), or do a full @@ -2081,13 +2083,12 @@ ll_assert(newobj != llmemory.NULL, "GCFLAG_HAS_SHADOW but no shadow found") newhdr = newobj - size_gc_header # - # Remove the flag GCFLAG_HAS_SHADOW, so that it doesn't get - # copied to the shadow itself. - self.header(obj).tid &= ~GCFLAG_HAS_SHADOW + # The flags GCFLAG_HAS_SHADOW and GCFLAG_SHADOW_INITIALIZED + # have no meaning in non-nursery objects. We don't need to + # remove them explicitly here before doing the copy. tid = self.header(obj).tid if (tid & GCFLAG_SHADOW_INITIALIZED) != 0: copy = False - self.header(obj).tid &= ~GCFLAG_SHADOW_INITIALIZED # totalsize = size_gc_header + self.get_size(obj) self.nursery_surviving_size += raw_malloc_usage(totalsize) @@ -2665,8 +2666,7 @@ # ---------- # id() and identityhash() support - @specialize.arg(2) - def _allocate_shadow(self, obj, copy=False): + def _allocate_shadow(self, obj): size_gc_header = self.gcheaderbuilder.size_gc_header size = self.get_size(obj) shadowhdr = self._malloc_out_of_nursery(size_gc_header + @@ -2688,12 +2688,6 @@ # self.header(obj).tid |= GCFLAG_HAS_SHADOW self.nursery_objects_shadows.setitem(obj, shadow) - - if copy: - self.header(obj).tid |= GCFLAG_SHADOW_INITIALIZED - totalsize = size_gc_header + self.get_size(obj) - llmemory.raw_memcopy(obj - size_gc_header, shadow, totalsize) - return shadow def _find_shadow(self, obj): diff --git a/rpython/memory/gc/test/test_direct.py b/rpython/memory/gc/test/test_direct.py --- a/rpython/memory/gc/test/test_direct.py +++ b/rpython/memory/gc/test/test_direct.py @@ -675,6 +675,25 @@ self.gc.debug_gc_step_until(incminimark.STATE_SCANNING) assert self.stackroots[1].x == 13 + def test_move_out_of_nursery(self): + obj0 = self.malloc(S) + obj0.x = 123 + adr1 = self.gc.move_out_of_nursery(llmemory.cast_ptr_to_adr(obj0)) + obj1 = llmemory.cast_adr_to_ptr(adr1, lltype.Ptr(S)) + assert obj1.x == 123 + # + import pytest + obj2 = self.malloc(S) + obj2.x = 456 + adr3 = self.gc._find_shadow(llmemory.cast_ptr_to_adr(obj2)) + obj3 = llmemory.cast_adr_to_ptr(adr3, lltype.Ptr(S)) + with pytest.raises(lltype.UninitializedMemoryAccess): + obj3.x # the shadow is not populated yet + adr4 = self.gc.move_out_of_nursery(llmemory.cast_ptr_to_adr(obj2)) + assert adr4 == adr3 + assert obj3.x == 456 # it is populated now + + class TestIncrementalMiniMarkGCFull(DirectGCTest): from rpython.memory.gc.incminimark import IncrementalMiniMarkGC as GCClass def test_malloc_fixedsize_no_cleanup(self): diff --git a/rpython/rlib/rvmprof/src/shared/_vmprof.c b/rpython/rlib/rvmprof/src/shared/_vmprof.c --- a/rpython/rlib/rvmprof/src/shared/_vmprof.c +++ b/rpython/rlib/rvmprof/src/shared/_vmprof.c @@ -36,6 +36,8 @@ register PY_STACK_FRAME_T * callee_saved asm("rbx"); #elif defined(X86_32) register PY_STACK_FRAME_T * callee_saved asm("edi"); +#elif defined(__arm__) + register PY_STACK_FRAME_T * callee_saved asm("r4"); #else # error "platform not supported" #endif @@ -45,6 +47,8 @@ "movq %1, %0\t\n" #elif defined(X86_32) "mov %1, %0\t\n" +#elif defined(__arm__) + "mov %1, %0\t\n" #else # error "platform not supported" #endif diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.c b/rpython/rlib/rvmprof/src/shared/vmp_stack.c --- a/rpython/rlib/rvmprof/src/shared/vmp_stack.c +++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.c @@ -16,7 +16,7 @@ #ifdef VMP_SUPPORTS_NATIVE_PROFILING -#ifdef VMPROF_LINUX +#if defined(VMPROF_LINUX) || defined(VMPROF_BSD) #include "unwind/vmprof_unwind.h" typedef mcontext_t unw_context_t; @@ -510,13 +510,15 @@ static const char * vmprof_error = NULL; static void * libhandle = NULL; - #ifdef VMPROF_LINUX +#include #define LIBUNWIND "libunwind.so" #ifdef __i386__ #define PREFIX "x86" +#define LIBUNWIND_SUFFIX "" #elif __x86_64__ #define PREFIX "x86_64" +#define LIBUNWIND_SUFFIX "-x86_64" #endif #define U_PREFIX "_U" #define UL_PREFIX "_UL" @@ -524,10 +526,41 @@ int vmp_native_enable(void) { #ifdef VMPROF_LINUX + void * oldhandle = NULL; + struct link_map * map = NULL; if (libhandle == NULL) { + // on linux, the wheel includes the libunwind shared object. + libhandle = dlopen(NULL, RTLD_NOW); + if (libhandle != NULL) { + // load the link map, it will contain an entry to + // .libs_vmprof/libunwind-...so, this is the file that is + // distributed with the wheel. + if (dlinfo(libhandle, RTLD_DI_LINKMAP, &map) != 0) { + (void)dlclose(libhandle); + libhandle = NULL; + goto bail_out; + } + // grab the new handle + do { + if (strstr(map->l_name, ".libs_vmprof/libunwind" LIBUNWIND_SUFFIX) != NULL) { + oldhandle = libhandle; + libhandle = dlopen(map->l_name, RTLD_LAZY|RTLD_LOCAL); + (void)dlclose(oldhandle); + oldhandle = NULL; + goto loaded_libunwind; + } + map = map->l_next; + } while (map != NULL); + // did not find .libs_vmprof/libunwind... + (void)dlclose(libhandle); + libhandle = NULL; + } + + // fallback! try to load the system's libunwind.so if ((libhandle = dlopen(LIBUNWIND, RTLD_LAZY | RTLD_LOCAL)) == NULL) { goto bail_out; } +loaded_libunwind: if ((unw_get_reg = dlsym(libhandle, UL_PREFIX PREFIX "_get_reg")) == NULL) { goto bail_out; } diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_common.h b/rpython/rlib/rvmprof/src/shared/vmprof_common.h --- a/rpython/rlib/rvmprof/src/shared/vmprof_common.h +++ b/rpython/rlib/rvmprof/src/shared/vmprof_common.h @@ -23,6 +23,10 @@ #include #endif +#ifdef VMPROF_BSD +#include +#endif + #define MAX_FUNC_NAME 1024 #ifdef VMPROF_UNIX From pypy.commits at gmail.com Mon Apr 16 11:40:16 2018 From: pypy.commits at gmail.com (arigo) Date: Mon, 16 Apr 2018 08:40:16 -0700 (PDT) Subject: [pypy-commit] cffi default: Issue #363 Message-ID: <5ad4c3e0.09e61c0a.dd7ca.9fe7@mx.google.com> Author: Armin Rigo Branch: Changeset: r3117:17c94f9cb463 Date: 2018-04-16 17:40 +0200 http://bitbucket.org/cffi/cffi/changeset/17c94f9cb463/ Log: Issue #363 Improve the error message for ffi.new(_, XX) diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -1288,7 +1288,13 @@ Py_ssize_t explicitlength; explicitlength = PyNumber_AsSsize_t(value, PyExc_OverflowError); if (explicitlength < 0) { - if (!PyErr_Occurred()) + if (PyErr_Occurred()) { + if (PyErr_ExceptionMatches(PyExc_TypeError)) + PyErr_Format(PyExc_TypeError, + "expected new array length or list/tuple/str, " + "not %.200s", Py_TYPE(value)->tp_name); + } + else PyErr_SetString(PyExc_ValueError, "negative array length"); return -1; } diff --git a/c/test_c.py b/c/test_c.py --- a/c/test_c.py +++ b/c/test_c.py @@ -289,6 +289,9 @@ assert repr(q).startswith(" Author: Armin Rigo Branch: Changeset: r94351:3f46c49c65fa Date: 2018-04-16 17:45 +0200 http://bitbucket.org/pypy/pypy/changeset/3f46c49c65fa/ Log: update to cffi/17c94f9cb463 diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py --- a/pypy/module/_cffi_backend/ctypearray.py +++ b/pypy/module/_cffi_backend/ctypearray.py @@ -70,7 +70,15 @@ length = wchar_helper.unicode_size_as_char32(u) return (w_value, length + 1) else: - explicitlength = space.getindex_w(w_value, space.w_OverflowError) + try: + explicitlength = space.getindex_w(w_value, + space.w_OverflowError) + except OperationError as e: + if e.match(space, space.w_TypeError): + raise oefmt(space.w_TypeError, + "expected new array length or list/tuple/str, " + "not %T", w_value) + raise if explicitlength < 0: raise oefmt(space.w_ValueError, "negative array length") return (space.w_None, explicitlength) diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -278,6 +278,9 @@ assert repr(q).startswith(" Author: Ronan Lamy Branch: py3tests Changeset: r94352:faae29c1f3fd Date: 2018-04-16 17:36 +0100 http://bitbucket.org/pypy/pypy/changeset/faae29c1f3fd/ Log: Convert test_boolobject.py diff --git a/pypy/objspace/std/test/apptest_boolobject.py b/pypy/objspace/std/test/apptest_boolobject.py new file mode 100644 --- /dev/null +++ b/pypy/objspace/std/test/apptest_boolobject.py @@ -0,0 +1,59 @@ +from pytest import raises + +def test_bool_callable(): + assert True == bool(1) + assert False == bool(0) + assert False == bool() + +def test_bool_string(): + assert "True" == str(True) + assert "False" == str(False) + assert "True" == repr(True) + assert "False" == repr(False) + +def test_bool_int(): + assert int(True) is 1 + assert int(False) is 0 + assert True.__int__() is 1 + +def test_bool_ops(): + assert True + True == 2 + assert False | False is False + assert True | False is True + assert True & True is True + assert True ^ True is False + assert False ^ False is False + assert True ^ False is True + assert True & 1 == 1 + assert False & 0 == 0 & 0 + +def test_bool_int_ops(): + assert True == 1 + assert 1 == True + assert False == 0 + assert 0 == False + + assert True is not 1 + assert 1 is not True + assert False is not 0 + assert 0 is not False + +def test_new(): + assert bool.__new__(bool, "hi") is True + assert bool.__new__(bool, "") is False + raises(TypeError, bool.__new__, int) + raises(TypeError, bool.__new__, 42) + +def test_cant_subclass_bool(): + raises(TypeError, "class b(bool): pass") + +def test_convert_to_bool(): + check = lambda o: raises(TypeError, bool, o) + class Spam(int): + def __bool__(): + return 1 + raises(TypeError, bool, Spam()) + +def test_from_bytes(): + assert bool.from_bytes(b"", 'little') is False + assert bool.from_bytes(b"dasijldjs" * 157, 'little') is True diff --git a/pypy/objspace/std/test/test_boolobject.py b/pypy/objspace/std/test/test_boolobject.py --- a/pypy/objspace/std/test/test_boolobject.py +++ b/pypy/objspace/std/test/test_boolobject.py @@ -30,61 +30,3 @@ assert self.space.bigint_w(self.true)._digits == [1] -class AppTestAppBoolTest: - def test_bool_callable(self): - assert True == bool(1) - assert False == bool(0) - assert False == bool() - - def test_bool_string(self): - assert "True" == str(True) - assert "False" == str(False) - assert "True" == repr(True) - assert "False" == repr(False) - - def test_bool_int(self): - assert int(True) is 1 - assert int(False) is 0 - assert True.__int__() is 1 - - def test_bool_ops(self): - assert True + True == 2 - assert False | False is False - assert True | False is True - assert True & True is True - assert True ^ True is False - assert False ^ False is False - assert True ^ False is True - assert True & 1 == 1 - assert False & 0 == 0 & 0 - - def test_bool_int_ops(self): - assert True == 1 - assert 1 == True - assert False == 0 - assert 0 == False - - assert True is not 1 - assert 1 is not True - assert False is not 0 - assert 0 is not False - - def test_new(self): - assert bool.__new__(bool, "hi") is True - assert bool.__new__(bool, "") is False - raises(TypeError, bool.__new__, int) - raises(TypeError, bool.__new__, 42) - - def test_cant_subclass_bool(self): - raises(TypeError, "class b(bool): pass") - - def test_convert_to_bool(self): - check = lambda o: raises(TypeError, bool, o) - class Spam(int): - def __bool__(self): - return 1 - raises(TypeError, bool, Spam()) - - def test_from_bytes(self): - assert bool.from_bytes(b"", 'little') is False - assert bool.from_bytes(b"dasijldjs" * 157, 'little') is True From pypy.commits at gmail.com Mon Apr 16 19:43:44 2018 From: pypy.commits at gmail.com (rlamy) Date: Mon, 16 Apr 2018 16:43:44 -0700 (PDT) Subject: [pypy-commit] pypy py3tests: Move all pypy/module/*/__init__.py to pypy/module/*/moduledef.py (breaks the world a little) Message-ID: <5ad53530.d15d1c0a.84f1.76a5@mx.google.com> Author: Ronan Lamy Branch: py3tests Changeset: r94356:543e3bec6299 Date: 2018-04-17 00:39 +0100 http://bitbucket.org/pypy/pypy/changeset/543e3bec6299/ Log: Move all pypy/module/*/__init__.py to pypy/module/*/moduledef.py (breaks the world a little) This will allow py3 to import the tests. The moves were done through the rename_moduledef.py script, committed here. diff too long, truncating to 2000 out of 3849 lines diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -487,7 +487,7 @@ fullname = importname importname = fullname.rsplit('.', 1)[1] else: - fullname = "pypy.module.%s" % importname + fullname = "pypy.module.%s.moduledef" % importname Module = __import__(fullname, None, None, ["Module"]).Module diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py --- a/pypy/module/__builtin__/__init__.py +++ b/pypy/module/__builtin__/__init__.py @@ -1,114 +0,0 @@ -from pypy.interpreter.error import OperationError -from pypy.interpreter import module -from pypy.interpreter.mixedmodule import MixedModule -import pypy.module.imp.importing - -# put builtins here that should be optimized somehow - -class Module(MixedModule): - """Built-in functions, exceptions, and other objects.""" - applevel_name = 'builtins' - - appleveldefs = { - 'input' : 'app_io.input', - 'print' : 'app_io.print_', - - 'sorted' : 'app_functional.sorted', - 'any' : 'app_functional.any', - 'all' : 'app_functional.all', - 'sum' : 'app_functional.sum', - 'vars' : 'app_inspect.vars', - 'dir' : 'app_inspect.dir', - - 'bin' : 'app_operation.bin', - 'oct' : 'app_operation.oct', - 'hex' : 'app_operation.hex', - } - - interpleveldefs = { - # constants - '__debug__' : '(space.w_True)', - 'None' : '(space.w_None)', - 'False' : '(space.w_False)', - 'True' : '(space.w_True)', - 'open' : 'state.get(space).w_open', - - # interp-level function definitions - 'abs' : 'operation.abs', - 'ascii' : 'operation.ascii', - 'chr' : 'operation.chr', - 'len' : 'operation.len', - 'ord' : 'operation.ord', - 'pow' : 'operation.pow', - 'repr' : 'operation.repr', - 'hash' : 'operation.hash', - 'round' : 'operation.round', - 'divmod' : 'operation.divmod', - 'format' : 'operation.format', - 'issubclass' : 'abstractinst.app_issubclass', - 'isinstance' : 'abstractinst.app_isinstance', - 'getattr' : 'operation.getattr', - 'setattr' : 'operation.setattr', - 'delattr' : 'operation.delattr', - 'hasattr' : 'operation.hasattr', - 'iter' : 'operation.iter', - 'next' : 'operation.next', - 'id' : 'operation.id', - 'callable' : 'operation.callable', - - 'compile' : 'compiling.compile', - 'eval' : 'compiling.eval', - 'exec' : 'compiling.exec_', - '__build_class__': 'compiling.build_class', - - '__import__' : 'pypy.module.imp.importing.importhook', - - 'range' : 'functional.W_Range', - 'enumerate' : 'functional.W_Enumerate', - 'map' : 'functional.W_Map', - 'filter' : 'functional.W_Filter', - 'zip' : 'functional.W_Zip', - 'min' : 'functional.min', - 'max' : 'functional.max', - 'reversed' : 'functional.W_ReversedIterator', - 'super' : 'descriptor.W_Super', - 'staticmethod' : 'pypy.interpreter.function.StaticMethod', - 'classmethod' : 'pypy.interpreter.function.ClassMethod', - 'property' : 'descriptor.W_Property', - - 'globals' : 'interp_inspect.globals', - 'locals' : 'interp_inspect.locals', - - } - - def pick_builtin(self, w_globals): - "Look up the builtin module to use from the __builtins__ global" - # pick the __builtins__ roughly in the same way CPython does it - # this is obscure and slow - space = self.space - try: - w_builtin = space.getitem(w_globals, space.newtext('__builtins__')) - except OperationError as e: - if not e.match(space, space.w_KeyError): - raise - else: - if w_builtin is space.builtin: # common case - return space.builtin - if space.isinstance_w(w_builtin, space.w_dict): - return module.Module(space, None, w_builtin) - if isinstance(w_builtin, module.Module): - return w_builtin - # no builtin! make a default one. Give them None, at least. - builtin = module.Module(space, None) - space.setitem(builtin.w_dict, space.newtext('None'), space.w_None) - return builtin - - def setup_after_space_initialization(self): - """NOT_RPYTHON""" - space = self.space - # install the more general version of isinstance() & co. in the space - from pypy.module.__builtin__ import abstractinst as ab - space.abstract_isinstance_w = ab.abstract_isinstance_w.__get__(space) - space.abstract_issubclass_w = ab.abstract_issubclass_w.__get__(space) - space.abstract_isclass_w = ab.abstract_isclass_w.__get__(space) - space.abstract_getclass = ab.abstract_getclass.__get__(space) diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/moduledef.py copy from pypy/module/__builtin__/__init__.py copy to pypy/module/__builtin__/moduledef.py diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -1,144 +0,0 @@ -import sys - -from pypy.interpreter.mixedmodule import MixedModule -from pypy.module.imp.importing import get_pyc_magic -from rpython.rlib import rtime - - -class BuildersModule(MixedModule): - """ Module containing string and unicode builders """ - - appleveldefs = {} - - interpleveldefs = { - "StringBuilder": "interp_builders.W_StringBuilder", - "BytesBuilder": "interp_builders.W_BytesBuilder", - } - -class TimeModule(MixedModule): - appleveldefs = {} - interpleveldefs = {} - if rtime.HAS_CLOCK_GETTIME: - interpleveldefs["clock_gettime"] = "interp_time.clock_gettime" - interpleveldefs["clock_getres"] = "interp_time.clock_getres" - for name in rtime.ALL_DEFINED_CLOCKS: - interpleveldefs[name] = "space.wrap(%d)" % getattr(rtime, name) - - -class ThreadModule(MixedModule): - appleveldefs = { - 'signals_enabled': 'app_signal.signals_enabled', - } - interpleveldefs = { - '_signals_enter': 'interp_signal.signals_enter', - '_signals_exit': 'interp_signal.signals_exit', - } - - -class IntOpModule(MixedModule): - """ Module for integer operations that have two-complement overflow - behaviour instead of overflowing to longs """ - appleveldefs = {} - interpleveldefs = { - 'int_add': 'interp_intop.int_add', - 'int_sub': 'interp_intop.int_sub', - 'int_mul': 'interp_intop.int_mul', - 'int_floordiv': 'interp_intop.int_floordiv', - 'int_mod': 'interp_intop.int_mod', - 'int_lshift': 'interp_intop.int_lshift', - 'int_rshift': 'interp_intop.int_rshift', - 'uint_rshift': 'interp_intop.uint_rshift', - } - - -class OsModule(MixedModule): - appleveldefs = {} - interpleveldefs = { - 'real_getenv': 'interp_os.real_getenv' - } - - -class PyPyDateTime(MixedModule): - appleveldefs = {} - interpleveldefs = { - 'dateinterop': 'interp_pypydatetime.W_DateTime_Date', - 'timeinterop' : 'interp_pypydatetime.W_DateTime_Time', - 'deltainterop' : 'interp_pypydatetime.W_DateTime_Delta', - } - -class Module(MixedModule): - """ PyPy specific "magic" functions. A lot of them are experimental and - subject to change, many are internal. """ - appleveldefs = { - } - - interpleveldefs = { - 'attach_gdb' : 'interp_magic.attach_gdb', - 'internal_repr' : 'interp_magic.internal_repr', - 'objects_in_repr' : 'interp_magic.objects_in_repr', - 'bytebuffer' : 'bytebuffer.bytebuffer', - 'identity_dict' : 'interp_identitydict.W_IdentityDict', - 'debug_start' : 'interp_debug.debug_start', - 'debug_print' : 'interp_debug.debug_print', - 'debug_stop' : 'interp_debug.debug_stop', - 'debug_print_once' : 'interp_debug.debug_print_once', - 'debug_flush' : 'interp_debug.debug_flush', - 'builtinify' : 'interp_magic.builtinify', - 'hidden_applevel' : 'interp_magic.hidden_applevel', - 'lookup_special' : 'interp_magic.lookup_special', - 'do_what_I_mean' : 'interp_magic.do_what_I_mean', - 'resizelist_hint' : 'interp_magic.resizelist_hint', - 'newlist_hint' : 'interp_magic.newlist_hint', - 'add_memory_pressure' : 'interp_magic.add_memory_pressure', - 'newdict' : 'interp_dict.newdict', - 'reversed_dict' : 'interp_dict.reversed_dict', - 'dict_popitem_first' : 'interp_dict.dict_popitem_first', - 'delitem_if_value_is' : 'interp_dict.delitem_if_value_is', - 'move_to_end' : 'interp_dict.move_to_end', - 'strategy' : 'interp_magic.strategy', # dict,set,list - 'set_debug' : 'interp_magic.set_debug', - 'locals_to_fast' : 'interp_magic.locals_to_fast', - 'set_code_callback' : 'interp_magic.set_code_callback', - 'decode_long' : 'interp_magic.decode_long', - '_promote' : 'interp_magic._promote', - 'normalize_exc' : 'interp_magic.normalize_exc', - 'StdErrPrinter' : 'interp_stderrprinter.W_StdErrPrinter', - 'stack_almost_full' : 'interp_magic.stack_almost_full', - 'fsencode' : 'interp_magic.fsencode', - 'fsdecode' : 'interp_magic.fsdecode', - } - - submodules = { - "builders": BuildersModule, - "time": TimeModule, - "thread": ThreadModule, - "intop": IntOpModule, - "os": OsModule, - '_pypydatetime': PyPyDateTime, - } - - def setup_after_space_initialization(self): - """NOT_RPYTHON""" - if self.space.config.objspace.std.withmethodcachecounter: - self.extra_interpdef('method_cache_counter', - 'interp_magic.method_cache_counter') - self.extra_interpdef('reset_method_cache_counter', - 'interp_magic.reset_method_cache_counter') - self.extra_interpdef('mapdict_cache_counter', - 'interp_magic.mapdict_cache_counter') - PYC_MAGIC = get_pyc_magic(self.space) - self.extra_interpdef('PYC_MAGIC', 'space.wrap(%d)' % PYC_MAGIC) - try: - from rpython.jit.backend import detect_cpu - model = detect_cpu.autodetect() - self.extra_interpdef('cpumodel', 'space.wrap(%r)' % model) - except Exception: - if self.space.config.translation.jit: - raise - else: - pass # ok fine to ignore in this case - - if self.space.config.translation.jit: - features = detect_cpu.getcpufeatures(model) - self.extra_interpdef('jit_backend_features', - 'space.wrap(%r)' % features) diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/moduledef.py copy from pypy/module/__pypy__/__init__.py copy to pypy/module/__pypy__/moduledef.py diff --git a/pypy/module/_ast/__init__.py b/pypy/module/_ast/__init__.py --- a/pypy/module/_ast/__init__.py +++ b/pypy/module/_ast/__init__.py @@ -1,21 +0,0 @@ -from pypy.interpreter.mixedmodule import MixedModule -from pypy.interpreter.astcompiler import ast, consts - - -class Module(MixedModule): - - interpleveldefs = { - "PyCF_ONLY_AST" : "space.wrap(%s)" % consts.PyCF_ONLY_AST, - "PyCF_ACCEPT_NULL_BYTES": - "space.wrap(%s)" % consts.PyCF_ACCEPT_NULL_BYTES, - "__version__" : "space.wrap('82160')", # from CPython's svn. - } - appleveldefs = {} - - -def _setup(): - defs = Module.interpleveldefs - defs['AST'] = "pypy.interpreter.astcompiler.ast.get(space).w_AST" - for (name, base, fields, attributes) in ast.State.AST_TYPES: - defs[name] = "pypy.interpreter.astcompiler.ast.get(space).w_" + name -_setup() diff --git a/pypy/module/_ast/__init__.py b/pypy/module/_ast/moduledef.py copy from pypy/module/_ast/__init__.py copy to pypy/module/_ast/moduledef.py diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -1,92 +0,0 @@ -import sys -from pypy.interpreter.mixedmodule import MixedModule -from rpython.rlib import rdynload, clibffi -from rpython.rtyper.lltypesystem import rffi - -VERSION = "1.11.5" - -FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI -try: - FFI_STDCALL = clibffi.FFI_STDCALL - has_stdcall = True -except AttributeError: - has_stdcall = False - - -class Module(MixedModule): - - appleveldefs = { - } - interpleveldefs = { - '__version__': 'space.wrap("%s")' % VERSION, - - 'load_library': 'libraryobj.load_library', - - 'new_primitive_type': 'newtype.new_primitive_type', - 'new_pointer_type': 'newtype.new_pointer_type', - 'new_array_type': 'newtype.new_array_type', - 'new_struct_type': 'newtype.new_struct_type', - 'new_union_type': 'newtype.new_union_type', - 'complete_struct_or_union': 'newtype.complete_struct_or_union', - 'new_void_type': 'newtype.new_void_type', - 'new_enum_type': 'newtype.new_enum_type', - 'new_function_type': 'newtype.new_function_type', - - 'newp': 'func.newp', - 'cast': 'func.cast', - 'callback': 'func.callback', - 'alignof': 'func.alignof', - 'sizeof': 'func.sizeof', - 'typeof': 'func.typeof', - 'typeoffsetof': 'func.typeoffsetof', - 'rawaddressof': 'func.rawaddressof', - 'getcname': 'func.getcname', - 'newp_handle': 'handle.newp_handle', - 'from_handle': 'handle.from_handle', - '_get_types': 'func._get_types', - '_get_common_types': 'func._get_common_types', - 'from_buffer': 'func.from_buffer', - 'gcp': 'func.gcp', - - 'string': 'func.string', - 'unpack': 'func.unpack', - 'buffer': 'cbuffer.MiniBuffer', - 'memmove': 'func.memmove', - - 'get_errno': 'cerrno.get_errno', - 'set_errno': 'cerrno.set_errno', - - 'FFI_DEFAULT_ABI': 'space.wrap(%d)' % FFI_DEFAULT_ABI, - 'FFI_CDECL': 'space.wrap(%d)' % FFI_DEFAULT_ABI, # win32 name - - # CFFI 1.0 - 'FFI': 'ffi_obj.W_FFIObject', - } - if sys.platform == 'win32': - interpleveldefs['getwinerror'] = 'cerrno.getwinerror' - - if has_stdcall: - interpleveldefs['FFI_STDCALL'] = 'space.wrap(%d)' % FFI_STDCALL - - def __init__(self, space, *args): - MixedModule.__init__(self, space, *args) - # - if not space.config.objspace.disable_entrypoints: - # import 'embedding', which has the side-effect of registering - # the 'pypy_init_embedded_cffi_module' entry point - from pypy.module._cffi_backend import embedding - embedding.glob.space = space - - -def get_dict_rtld_constants(): - found = {} - for name in ["RTLD_LAZY", "RTLD_NOW", "RTLD_GLOBAL", "RTLD_LOCAL", - "RTLD_NODELETE", "RTLD_NOLOAD", "RTLD_DEEPBIND"]: - if getattr(rdynload.cConfig, name) is not None: - found[name] = getattr(rdynload.cConfig, name) - for name in ["RTLD_LAZY", "RTLD_NOW", "RTLD_GLOBAL", "RTLD_LOCAL"]: - found.setdefault(name, 0) - return found - -for _name, _value in get_dict_rtld_constants().items(): - Module.interpleveldefs[_name] = 'space.wrap(%d)' % _value diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/moduledef.py copy from pypy/module/_cffi_backend/__init__.py copy to pypy/module/_cffi_backend/moduledef.py diff --git a/pypy/module/_codecs/__init__.py b/pypy/module/_codecs/__init__.py --- a/pypy/module/_codecs/__init__.py +++ b/pypy/module/_codecs/__init__.py @@ -1,98 +0,0 @@ -from pypy.interpreter.mixedmodule import MixedModule -from rpython.rlib import runicode -from rpython.rlib.objectmodel import not_rpython -from pypy.module._codecs import interp_codecs - -class Module(MixedModule): - """ - _codecs -- Provides access to the codec registry and the builtin - codecs. - - This module should never be imported directly. The standard library - module "codecs" wraps this builtin module for use within Python. - - The codec registry is accessible via: - - register(search_function) -> None - - lookup(encoding) -> (encoder, decoder, stream_reader, stream_writer) - - The builtin Unicode codecs use the following interface: - - _encode(Unicode_object[,errors='strict']) -> - (string object, bytes consumed) - - _decode(char_buffer_obj[,errors='strict']) -> - (Unicode object, bytes consumed) - - _encode() interfaces also accept non-Unicode object as - input. The objects are then converted to Unicode using - PyUnicode_FromObject() prior to applying the conversion. - - These s are available: utf_8, unicode_escape, - raw_unicode_escape, unicode_internal, latin_1, ascii (7-bit), - mbcs (on win32). - - -Written by Marc-Andre Lemburg (mal at lemburg.com). - -Copyright (c) Corporation for National Research Initiatives. -""" - - appleveldefs = {} - - interpleveldefs = { - 'encode': 'interp_codecs.encode', - 'decode': 'interp_codecs.decode', - 'lookup': 'interp_codecs.lookup_codec', - 'lookup_error': 'interp_codecs.lookup_error', - 'register': 'interp_codecs.register_codec', - 'register_error': 'interp_codecs.register_error', - 'charmap_build' : 'interp_codecs.charmap_build', - - # encoders and decoders - 'ascii_decode' : 'interp_codecs.ascii_decode', - 'ascii_encode' : 'interp_codecs.ascii_encode', - 'latin_1_decode' : 'interp_codecs.latin_1_decode', - 'latin_1_encode' : 'interp_codecs.latin_1_encode', - 'utf_7_decode' : 'interp_codecs.utf_7_decode', - 'utf_7_encode' : 'interp_codecs.utf_7_encode', - 'utf_8_decode' : 'interp_codecs.utf_8_decode', - 'utf_8_encode' : 'interp_codecs.utf_8_encode', - 'utf_16_be_decode' : 'interp_codecs.utf_16_be_decode', - 'utf_16_be_encode' : 'interp_codecs.utf_16_be_encode', - 'utf_16_decode' : 'interp_codecs.utf_16_decode', - 'utf_16_encode' : 'interp_codecs.utf_16_encode', - 'utf_16_le_decode' : 'interp_codecs.utf_16_le_decode', - 'utf_16_le_encode' : 'interp_codecs.utf_16_le_encode', - 'utf_16_ex_decode' : 'interp_codecs.utf_16_ex_decode', - 'utf_32_decode' : 'interp_codecs.utf_32_decode', - 'utf_32_encode' : 'interp_codecs.utf_32_encode', - 'utf_32_be_decode' : 'interp_codecs.utf_32_be_decode', - 'utf_32_be_encode' : 'interp_codecs.utf_32_be_encode', - 'utf_32_le_decode' : 'interp_codecs.utf_32_le_decode', - 'utf_32_le_encode' : 'interp_codecs.utf_32_le_encode', - 'utf_32_ex_decode' : 'interp_codecs.utf_32_ex_decode', - 'readbuffer_encode': 'interp_codecs.readbuffer_encode', - 'charmap_decode' : 'interp_codecs.charmap_decode', - 'charmap_encode' : 'interp_codecs.charmap_encode', - 'escape_encode' : 'interp_codecs.escape_encode', - 'escape_decode' : 'interp_codecs.escape_decode', - 'unicode_escape_decode' : 'interp_codecs.unicode_escape_decode', - 'unicode_escape_encode' : 'interp_codecs.unicode_escape_encode', - 'raw_unicode_escape_decode' : 'interp_codecs.raw_unicode_escape_decode', - 'raw_unicode_escape_encode' : 'interp_codecs.raw_unicode_escape_encode', - 'unicode_internal_decode' : 'interp_codecs.unicode_internal_decode', - 'unicode_internal_encode' : 'interp_codecs.unicode_internal_encode', - } - - @not_rpython - def __init__(self, space, *args): - # mbcs codec is Windows specific, and based on rffi. - if (hasattr(runicode, 'str_decode_mbcs')): - self.interpleveldefs['mbcs_encode'] = 'interp_codecs.mbcs_encode' - self.interpleveldefs['mbcs_decode'] = 'interp_codecs.mbcs_decode' - - MixedModule.__init__(self, space, *args) - - interp_codecs.register_builtin_error_handlers(space) diff --git a/pypy/module/_codecs/__init__.py b/pypy/module/_codecs/moduledef.py copy from pypy/module/_codecs/__init__.py copy to pypy/module/_codecs/moduledef.py diff --git a/pypy/module/_collections/__init__.py b/pypy/module/_collections/__init__.py --- a/pypy/module/_collections/__init__.py +++ b/pypy/module/_collections/__init__.py @@ -1,39 +0,0 @@ -from pypy.interpreter.mixedmodule import MixedModule - -class Module(MixedModule): - """High performance data structures. -- deque: ordered collection accessible from endpoints only -- defaultdict: dict subclass with a default value factory -""" - - appleveldefs = { - 'defaultdict': 'app_defaultdict.defaultdict', - } - - interpleveldefs = { - 'deque' : 'interp_deque.W_Deque', - 'deque_iterator' : 'interp_deque.W_DequeIter', - 'deque_reverse_iterator' : 'interp_deque.W_DequeRevIter', - '__missing__': 'interp_defaultdict.missing', - } - - def setup_after_space_initialization(self): - """NOT_RPYTHON""" - # must remove the interp-level name '__missing__' after it has - # been used... otherwise, some code is not happy about seeing - # this code object twice - space = self.space - space.getattr(self, space.newtext('defaultdict')) # force importing - space.delattr(self, space.newtext('__missing__')) - - def startup(self, space): - # OrderedDict is normally present, but in some cases the line - # "from __pypy__ import reversed_dict, move_to_end" from - # _pypy_collections.py raises - space.appexec([self], """(mod): - try: - from _pypy_collections import OrderedDict - mod.OrderedDict = OrderedDict - except ImportError: - pass - """) diff --git a/pypy/module/_collections/__init__.py b/pypy/module/_collections/moduledef.py copy from pypy/module/_collections/__init__.py copy to pypy/module/_collections/moduledef.py diff --git a/pypy/module/_continuation/__init__.py b/pypy/module/_continuation/__init__.py --- a/pypy/module/_continuation/__init__.py +++ b/pypy/module/_continuation/__init__.py @@ -1,41 +0,0 @@ -from pypy.interpreter.mixedmodule import MixedModule - - -class Module(MixedModule): - """This module exposes 'one-shot continuation containers'. - -A 'continulet' object from this module is a container that stores a -one-shot continuation. It is similar in purpose to the 'f_back' -attribute of frames, which points to where execution should continue -after this frame finishes. The difference is that it will be changed -(often repeatedly) before the frame actually returns. - -To make a continulet object, call 'continulet' with a callable and -optional extra arguments. Later, the first time you switch() to the -continulet, the callable is invoked with the same continulet object as -the extra first argument. - -At this point, the one-shot continuation stored in the continulet points -to the caller of switch(). When switch() is called again, this one-shot -continuation is exchanged with the current one; it means that the caller -of switch() is suspended, its continuation stored in the container, and -the old continuation from the continulet object is resumed. - -Continulets are internally implemented using stacklets. Stacklets -are a bit more primitive (they are really one-shot continuations), but -that idea only works in C, not in Python, notably because of exceptions. - -The most primitive API is actually 'permute()', which just permutes the -one-shot continuation stored in two (or more) continulets. -""" - - appleveldefs = { - 'error': 'app_continuation.error', - 'generator': 'app_continuation.generator', - } - - interpleveldefs = { - 'continulet': 'interp_continuation.W_Continulet', - 'permute': 'interp_continuation.permute', - '_p': 'interp_continuation.unpickle', # pickle support - } diff --git a/pypy/module/_continuation/__init__.py b/pypy/module/_continuation/moduledef.py copy from pypy/module/_continuation/__init__.py copy to pypy/module/_continuation/moduledef.py diff --git a/pypy/module/_cppyy/__init__.py b/pypy/module/_cppyy/__init__.py --- a/pypy/module/_cppyy/__init__.py +++ b/pypy/module/_cppyy/__init__.py @@ -1,42 +0,0 @@ -from pypy.interpreter.mixedmodule import MixedModule - -class Module(MixedModule): - "This module brigdes the cppyy frontend with its backend, through PyPy.\n\ - See http://cppyy.readthedocs.io/en/latest for full details." - - interpleveldefs = { - '_resolve_name' : 'interp_cppyy.resolve_name', - '_scope_byname' : 'interp_cppyy.scope_byname', - '_is_template' : 'interp_cppyy.is_template', - '_std_string_name' : 'interp_cppyy.std_string_name', - '_set_class_generator' : 'interp_cppyy.set_class_generator', - '_set_function_generator': 'interp_cppyy.set_function_generator', - '_register_class' : 'interp_cppyy.register_class', - '_get_nullptr' : 'interp_cppyy.get_nullptr', - 'CPPClassBase' : 'interp_cppyy.W_CPPClass', - 'addressof' : 'interp_cppyy.addressof', - '_bind_object' : 'interp_cppyy._bind_object', - 'bind_object' : 'interp_cppyy.bind_object', - 'move' : 'interp_cppyy.move', - } - - appleveldefs = { - '_init_pythonify' : 'pythonify._init_pythonify', - 'add_pythonization' : 'pythonify.add_pythonization', - 'Template' : 'pythonify.CPPTemplate', - } - - def __init__(self, space, *args): - "NOT_RPYTHON" - MixedModule.__init__(self, space, *args) - - # pythonization functions may be written in RPython, but the interp2app - # code generation is not, so give it a chance to run now - from pypy.module._cppyy import capi - capi.register_pythonizations(space) - - def startup(self, space): - from pypy.module._cppyy import capi - capi.verify_backend(space) # may raise ImportError - - space.call_method(self, '_init_pythonify') diff --git a/pypy/module/_cppyy/__init__.py b/pypy/module/_cppyy/moduledef.py copy from pypy/module/_cppyy/__init__.py copy to pypy/module/_cppyy/moduledef.py diff --git a/pypy/module/_csv/__init__.py b/pypy/module/_csv/__init__.py --- a/pypy/module/_csv/__init__.py +++ b/pypy/module/_csv/__init__.py @@ -1,87 +0,0 @@ -from pypy.interpreter.mixedmodule import MixedModule - - -class Module(MixedModule): - """CSV parsing and writing. - -This module provides classes that assist in the reading and writing -of Comma Separated Value (CSV) files, and implements the interface -described by PEP 305. Although many CSV files are simple to parse, -the format is not formally defined by a stable specification and -is subtle enough that parsing lines of a CSV file with something -like line.split(\",\") is bound to fail. The module supports three -basic APIs: reading, writing, and registration of dialects. - - -DIALECT REGISTRATION: - -Readers and writers support a dialect argument, which is a convenient -handle on a group of settings. When the dialect argument is a string, -it identifies one of the dialects previously registered with the module. -If it is a class or instance, the attributes of the argument are used as -the settings for the reader or writer: - - class excel: - delimiter = ',' - quotechar = '\"' - escapechar = None - doublequote = True - skipinitialspace = False - lineterminator = '\\r\\n' - quoting = QUOTE_MINIMAL - -SETTINGS: - - * quotechar - specifies a one-character string to use as the - quoting character. It defaults to '\"'. - * delimiter - specifies a one-character string to use as the - field separator. It defaults to ','. - * skipinitialspace - specifies how to interpret whitespace which - immediately follows a delimiter. It defaults to False, which - means that whitespace immediately following a delimiter is part - of the following field. - * lineterminator - specifies the character sequence which should - terminate rows. - * quoting - controls when quotes should be generated by the writer. - It can take on any of the following module constants: - - csv.QUOTE_MINIMAL means only when required, for example, when a - field contains either the quotechar or the delimiter - csv.QUOTE_ALL means that quotes are always placed around fields. - csv.QUOTE_NONNUMERIC means that quotes are always placed around - fields which do not parse as integers or floating point - numbers. - csv.QUOTE_NONE means that quotes are never placed around fields. - * escapechar - specifies a one-character string used to escape - the delimiter when quoting is set to QUOTE_NONE. - * doublequote - controls the handling of quotes inside fields. When - True, two consecutive quotes are interpreted as one during read, - and when writing, each quote character embedded in the data is - written as two quotes. -""" - - appleveldefs = { - 'register_dialect': 'app_csv.register_dialect', - 'unregister_dialect': 'app_csv.unregister_dialect', - 'get_dialect': 'app_csv.get_dialect', - 'list_dialects': 'app_csv.list_dialects', - '_dialects': 'app_csv._dialects', - - 'Error': 'app_csv.Error', - } - - interpleveldefs = { - '__version__': 'space.wrap("1.0")', - - 'QUOTE_MINIMAL': 'space.wrap(interp_csv.QUOTE_MINIMAL)', - 'QUOTE_ALL': 'space.wrap(interp_csv.QUOTE_ALL)', - 'QUOTE_NONNUMERIC': 'space.wrap(interp_csv.QUOTE_NONNUMERIC)', - 'QUOTE_NONE': 'space.wrap(interp_csv.QUOTE_NONE)', - - 'Dialect': 'interp_csv.W_Dialect', - - 'reader': 'interp_reader.csv_reader', - 'field_size_limit': 'interp_reader.csv_field_size_limit', - - 'writer': 'interp_writer.csv_writer', - } diff --git a/pypy/module/_csv/__init__.py b/pypy/module/_csv/moduledef.py copy from pypy/module/_csv/__init__.py copy to pypy/module/_csv/moduledef.py diff --git a/pypy/module/_demo/__init__.py b/pypy/module/_demo/__init__.py --- a/pypy/module/_demo/__init__.py +++ b/pypy/module/_demo/__init__.py @@ -1,24 +0,0 @@ -from pypy.interpreter.mixedmodule import MixedModule - -class Module(MixedModule): - """A demo built-in module based on ctypes.""" - - interpleveldefs = { - 'measuretime' : 'demo.measuretime', - 'sieve' : 'demo.sieve', - 'MyType' : 'demo.W_MyType', - } - - appleveldefs = { - 'DemoError' : 'app_demo.DemoError', - } - - # Used in tests - demo_events = [] - def setup_after_space_initialization(self): - Module.demo_events.append('setup') - def startup(self, space): - Module.demo_events.append('startup') - def shutdown(self, space): - Module.demo_events.append('shutdown') - diff --git a/pypy/module/_demo/__init__.py b/pypy/module/_demo/moduledef.py copy from pypy/module/_demo/__init__.py copy to pypy/module/_demo/moduledef.py diff --git a/pypy/module/_frozen_importlib/__init__.py b/pypy/module/_frozen_importlib/__init__.py --- a/pypy/module/_frozen_importlib/__init__.py +++ b/pypy/module/_frozen_importlib/__init__.py @@ -1,84 +0,0 @@ -import os -from pypy.interpreter.mixedmodule import MixedModule -from pypy.module.sys import initpath -from pypy.module._frozen_importlib import interp_import - -lib_python = os.path.join(os.path.dirname(__file__), - '..', '..', '..', 'lib-python', '3') - -class Module(MixedModule): - interpleveldefs = { - } - - appleveldefs = { - } - - @staticmethod - def _compile_bootstrap_module(space, name, w_name, w_dict): - """NOT_RPYTHON""" - with open(os.path.join(lib_python, 'importlib', name + '.py')) as fp: - source = fp.read() - pathname = "" % name - code_w = Module._cached_compile(space, name, source, - pathname, 'exec', 0) - space.setitem(w_dict, space.wrap('__name__'), w_name) - space.setitem(w_dict, space.wrap('__builtins__'), - space.wrap(space.builtin)) - code_w.exec_code(space, w_dict, w_dict) - - def install(self): - """NOT_RPYTHON""" - from pypy.module.imp import interp_imp - - super(Module, self).install() - space = self.space - # "import importlib/_boostrap_external.py" - w_mod = Module(space, space.wrap("_frozen_importlib_external")) - # hack: inject MAGIC_NUMBER into this module's dict - space.setattr(w_mod, space.wrap('MAGIC_NUMBER'), - interp_imp.get_magic(space)) - self._compile_bootstrap_module( - space, '_bootstrap_external', w_mod.w_name, w_mod.w_dict) - space.sys.setmodule(w_mod) - # "from importlib/_boostrap.py import *" - # It's not a plain "import importlib._boostrap", because we - # don't want to freeze importlib.__init__. - self._compile_bootstrap_module( - space, '_bootstrap', self.w_name, self.w_dict) - - self.w_import = space.wrap(interp_import.import_with_frames_removed) - - @staticmethod - def _cached_compile(space, name, source, *args): - from rpython.config.translationoption import CACHE_DIR - from pypy.module.marshal import interp_marshal - from pypy.interpreter.pycode import default_magic - - cachename = os.path.join(CACHE_DIR, 'frozen_importlib_%d%s' % ( - default_magic, name)) - try: - if space.config.translating: - raise IOError("don't use the cache when translating pypy") - with open(cachename, 'rb') as f: - previous = f.read(len(source) + 1) - if previous != source + '\x00': - raise IOError("source changed") - w_bin = space.newbytes(f.read()) - code_w = interp_marshal.loads(space, w_bin) - except IOError: - # must (re)compile the source - ec = space.getexecutioncontext() - code_w = ec.compiler.compile(source, *args) - w_bin = interp_marshal.dumps(space, code_w, space.wrap(2)) - content = source + '\x00' + space.bytes_w(w_bin) - with open(cachename, 'wb') as f: - f.write(content) - return code_w - - def startup(self, space): - """Copy our __import__ to builtins.""" - w_install = self.getdictvalue(space, '_install') - space.call_function(w_install, - space.getbuiltinmodule('sys'), - space.getbuiltinmodule('_imp')) - self.space.builtin.setdictvalue(space, '__import__', self.w_import) diff --git a/pypy/module/_frozen_importlib/__init__.py b/pypy/module/_frozen_importlib/moduledef.py copy from pypy/module/_frozen_importlib/__init__.py copy to pypy/module/_frozen_importlib/moduledef.py diff --git a/pypy/module/_io/__init__.py b/pypy/module/_io/__init__.py --- a/pypy/module/_io/__init__.py +++ b/pypy/module/_io/__init__.py @@ -1,34 +0,0 @@ -from pypy.interpreter.mixedmodule import MixedModule - -class Module(MixedModule): - - appleveldefs = { - } - - interpleveldefs = { - 'DEFAULT_BUFFER_SIZE': 'space.wrap(interp_iobase.DEFAULT_BUFFER_SIZE)', - 'BlockingIOError': 'space.w_BlockingIOError', - 'UnsupportedOperation': - 'space.fromcache(interp_io.Cache).w_unsupportedoperation', - '_IOBase': 'interp_iobase.W_IOBase', - '_RawIOBase': 'interp_iobase.W_RawIOBase', - '_BufferedIOBase': 'interp_bufferedio.W_BufferedIOBase', - '_TextIOBase': 'interp_textio.W_TextIOBase', - - 'FileIO': 'interp_fileio.W_FileIO', - 'BytesIO': 'interp_bytesio.W_BytesIO', - 'StringIO': 'interp_stringio.W_StringIO', - 'BufferedReader': 'interp_bufferedio.W_BufferedReader', - 'BufferedWriter': 'interp_bufferedio.W_BufferedWriter', - 'BufferedRWPair': 'interp_bufferedio.W_BufferedRWPair', - 'BufferedRandom': 'interp_bufferedio.W_BufferedRandom', - 'TextIOWrapper': 'interp_textio.W_TextIOWrapper', - - 'open': 'interp_io.open', - 'IncrementalNewlineDecoder': 'interp_textio.W_IncrementalNewlineDecoder', - } - - def shutdown(self, space): - # at shutdown, flush all open streams. Ignore I/O errors. - from pypy.module._io.interp_iobase import get_autoflusher - get_autoflusher(space).flush_all(space) diff --git a/pypy/module/_io/__init__.py b/pypy/module/_io/moduledef.py copy from pypy/module/_io/__init__.py copy to pypy/module/_io/moduledef.py diff --git a/pypy/module/_jitlog/__init__.py b/pypy/module/_jitlog/__init__.py --- a/pypy/module/_jitlog/__init__.py +++ b/pypy/module/_jitlog/__init__.py @@ -1,13 +0,0 @@ -from pypy.interpreter.mixedmodule import MixedModule -from rpython.rlib.rvmprof import VMProfPlatformUnsupported - -class Module(MixedModule): - """ JitLog the new logging facility """ - appleveldefs = { - } - - interpleveldefs = { - 'enable': 'interp_jitlog.enable', - 'disable': 'interp_jitlog.disable', - 'JitlogError': 'space.fromcache(interp_jitlog.Cache).w_JitlogError', - } diff --git a/pypy/module/_jitlog/__init__.py b/pypy/module/_jitlog/moduledef.py copy from pypy/module/_jitlog/__init__.py copy to pypy/module/_jitlog/moduledef.py diff --git a/pypy/module/_locale/__init__.py b/pypy/module/_locale/__init__.py --- a/pypy/module/_locale/__init__.py +++ b/pypy/module/_locale/__init__.py @@ -1,45 +0,0 @@ -from pypy.interpreter.mixedmodule import MixedModule -from rpython.rlib import rlocale -import sys - -class Module(MixedModule): - """Support for POSIX locales.""" - - interpleveldefs = { - 'setlocale': 'interp_locale.setlocale', - 'localeconv': 'interp_locale.localeconv', - 'strcoll': 'interp_locale.strcoll', - 'strxfrm': 'interp_locale.strxfrm', - 'Error': 'interp_locale.W_Error', - } - - if sys.platform == 'win32': - interpleveldefs.update({ - '_getdefaultlocale': 'interp_locale.getdefaultlocale', - }) - - if rlocale.HAVE_LANGINFO: - interpleveldefs.update({ - 'nl_langinfo': 'interp_locale.nl_langinfo', - }) - if rlocale.HAVE_LIBINTL: - interpleveldefs.update({ - 'gettext': 'interp_locale.gettext', - 'dgettext': 'interp_locale.dgettext', - 'dcgettext': 'interp_locale.dcgettext', - 'textdomain': 'interp_locale.textdomain', - 'bindtextdomain': 'interp_locale.bindtextdomain', - }) - if rlocale.HAVE_BIND_TEXTDOMAIN_CODESET: - interpleveldefs.update({ - 'bind_textdomain_codeset':'interp_locale.bind_textdomain_codeset', - }) - - appleveldefs = { - } - - def buildloaders(cls): - for constant, value in rlocale.constants.iteritems(): - Module.interpleveldefs[constant] = "space.wrap(%r)" % value - super(Module, cls).buildloaders() - buildloaders = classmethod(buildloaders) diff --git a/pypy/module/_locale/__init__.py b/pypy/module/_locale/moduledef.py copy from pypy/module/_locale/__init__.py copy to pypy/module/_locale/moduledef.py diff --git a/pypy/module/_lsprof/__init__.py b/pypy/module/_lsprof/__init__.py --- a/pypy/module/_lsprof/__init__.py +++ b/pypy/module/_lsprof/__init__.py @@ -1,10 +0,0 @@ - -""" _lsprof module -""" - -from pypy.interpreter.mixedmodule import MixedModule - -class Module(MixedModule): - interpleveldefs = {'Profiler':'interp_lsprof.W_Profiler'} - - appleveldefs = {} diff --git a/pypy/module/_lsprof/__init__.py b/pypy/module/_lsprof/moduledef.py copy from pypy/module/_lsprof/__init__.py copy to pypy/module/_lsprof/moduledef.py diff --git a/pypy/module/_md5/__init__.py b/pypy/module/_md5/__init__.py --- a/pypy/module/_md5/__init__.py +++ b/pypy/module/_md5/__init__.py @@ -1,26 +0,0 @@ - -""" -Mixed-module definition for the md5 module. -Note that there is also a pure Python implementation in pypy/lib/md5.py; -the present mixed-module version of md5 takes precedence if it is enabled. -""" - -from pypy.interpreter.mixedmodule import MixedModule - - -class Module(MixedModule): - """\ -This module implements the interface to RSA's MD5 message digest -algorithm (see also Internet RFC 1321). Its use is quite -straightforward: use new() to create an md5 object. You can now feed -this object with arbitrary strings using the update() method, and at any -point you can ask it for the digest (a strong kind of 128-bit checksum, -a.k.a. ``fingerprint'') of the concatenation of the strings fed to it so -far using the digest() method.""" - - interpleveldefs = { - 'md5': 'interp_md5.W_MD5', - } - - appleveldefs = { - } diff --git a/pypy/module/_md5/__init__.py b/pypy/module/_md5/moduledef.py copy from pypy/module/_md5/__init__.py copy to pypy/module/_md5/moduledef.py diff --git a/pypy/module/_minimal_curses/__init__.py b/pypy/module/_minimal_curses/__init__.py --- a/pypy/module/_minimal_curses/__init__.py +++ b/pypy/module/_minimal_curses/__init__.py @@ -1,34 +0,0 @@ -try: - import _curses -except Exception: # probably ImportError or cffi's VerificationError - try: - # when running on top of pypy before it had _curses, settle for minimal - # we prefer _curses so any constants added make it into _minimal_curses - import _minimal_curses as _curses - except ImportError: - import py - py.test.skip("no _curses or _minimal_curses module") # no _curses at all - -from pypy.interpreter.mixedmodule import MixedModule - - -class Module(MixedModule): - """ Low-level interface for curses module, - not meant to be used directly - """ - - appleveldefs = { - 'error' : 'app_curses.error', - } - - interpleveldefs = { - 'setupterm' : 'interp_curses.setupterm', - 'tigetstr' : 'interp_curses.tigetstr', - 'tparm' : 'interp_curses.tparm', - } - -for i in dir(_curses): - i = str(i) # workaround for pypy 2.0-beta2 - val = getattr(_curses, i) - if i.isupper() and type(val) is int: - Module.interpleveldefs[i] = "space.wrap(%s)" % val diff --git a/pypy/module/_minimal_curses/__init__.py b/pypy/module/_minimal_curses/moduledef.py copy from pypy/module/_minimal_curses/__init__.py copy to pypy/module/_minimal_curses/moduledef.py diff --git a/pypy/module/_multibytecodec/__init__.py b/pypy/module/_multibytecodec/__init__.py --- a/pypy/module/_multibytecodec/__init__.py +++ b/pypy/module/_multibytecodec/__init__.py @@ -1,22 +0,0 @@ -from pypy.interpreter.mixedmodule import MixedModule - - -class Module(MixedModule): - - interpleveldefs = { - # for compatibility this name is obscured, and should be called - # via the _codecs_*.py modules written in lib_pypy. - '__getcodec': 'interp_multibytecodec.getcodec', - - 'MultibyteIncrementalDecoder': - 'interp_incremental.MultibyteIncrementalDecoder', - 'MultibyteIncrementalEncoder': - 'interp_incremental.MultibyteIncrementalEncoder', - } - - appleveldefs = { - 'MultibyteStreamReader': - 'app_multibytecodec.MultibyteStreamReader', - 'MultibyteStreamWriter': - 'app_multibytecodec.MultibyteStreamWriter', - } diff --git a/pypy/module/_multibytecodec/__init__.py b/pypy/module/_multibytecodec/moduledef.py copy from pypy/module/_multibytecodec/__init__.py copy to pypy/module/_multibytecodec/moduledef.py diff --git a/pypy/module/_multiprocessing/__init__.py b/pypy/module/_multiprocessing/__init__.py --- a/pypy/module/_multiprocessing/__init__.py +++ b/pypy/module/_multiprocessing/__init__.py @@ -1,19 +0,0 @@ -import sys - -from pypy.interpreter.mixedmodule import MixedModule - -class Module(MixedModule): - - interpleveldefs = { - 'SemLock' : 'interp_semaphore.W_SemLock', - } - - appleveldefs = { - } - - if sys.platform == 'win32': - interpleveldefs['closesocket'] = 'interp_win32_py3.multiprocessing_closesocket' - interpleveldefs['recv'] = 'interp_win32_py3.multiprocessing_recv' - interpleveldefs['send'] = 'interp_win32_py3.multiprocessing_send' - else: - interpleveldefs['sem_unlink'] = 'interp_semaphore.semaphore_unlink' diff --git a/pypy/module/_multiprocessing/__init__.py b/pypy/module/_multiprocessing/moduledef.py copy from pypy/module/_multiprocessing/__init__.py copy to pypy/module/_multiprocessing/moduledef.py diff --git a/pypy/module/_pickle_support/__init__.py b/pypy/module/_pickle_support/__init__.py --- a/pypy/module/_pickle_support/__init__.py +++ b/pypy/module/_pickle_support/__init__.py @@ -1,25 +0,0 @@ -from pypy.interpreter.mixedmodule import MixedModule - -class Module(MixedModule): - """Built-in functions, exceptions, and other objects.""" - - appleveldefs = { - } - - interpleveldefs = { - 'cell_new' : 'maker.cell_new', - 'code_new' : 'maker.code_new', - 'func_new' : 'maker.func_new', - 'module_new' : 'maker.module_new', - 'method_new' : 'maker.method_new', - 'builtin_method_new' : 'maker.builtin_method_new', - 'dictiter_surrogate_new' : 'maker.dictiter_surrogate_new', - 'frame_new' : 'maker.frame_new', - 'traceback_new' : 'maker.traceback_new', - 'generator_new' : 'maker.generator_new', - 'coroutine_new' : 'maker.coroutine_new', - 'longrangeiter_new': 'maker.longrangeiter_new', - 'intrangeiter_new': 'maker.intrangeiter_new', - 'builtin_code': 'maker.builtin_code', - 'builtin_function' : 'maker.builtin_function', - } diff --git a/pypy/module/_pickle_support/__init__.py b/pypy/module/_pickle_support/moduledef.py copy from pypy/module/_pickle_support/__init__.py copy to pypy/module/_pickle_support/moduledef.py diff --git a/pypy/module/_posixsubprocess/__init__.py b/pypy/module/_posixsubprocess/__init__.py --- a/pypy/module/_posixsubprocess/__init__.py +++ b/pypy/module/_posixsubprocess/__init__.py @@ -1,16 +0,0 @@ -from pypy.interpreter.mixedmodule import MixedModule - -class Module(MixedModule): - applevel_name = '_posixsubprocess' - - appleveldefs = { - } - - interpleveldefs = { - 'fork_exec': 'interp_subprocess.fork_exec', - 'cloexec_pipe': 'interp_subprocess.cloexec_pipe', - } - - def startup(self, space): - from interp_subprocess import c_init - c_init() diff --git a/pypy/module/_posixsubprocess/__init__.py b/pypy/module/_posixsubprocess/moduledef.py copy from pypy/module/_posixsubprocess/__init__.py copy to pypy/module/_posixsubprocess/moduledef.py diff --git a/pypy/module/_pypyjson/__init__.py b/pypy/module/_pypyjson/__init__.py --- a/pypy/module/_pypyjson/__init__.py +++ b/pypy/module/_pypyjson/__init__.py @@ -1,12 +0,0 @@ -from pypy.interpreter.mixedmodule import MixedModule - -class Module(MixedModule): - """fast json implementation""" - - appleveldefs = {} - - interpleveldefs = { - 'loads' : 'interp_decoder.loads', - 'raw_encode_basestring_ascii': - 'interp_encoder.raw_encode_basestring_ascii', - } diff --git a/pypy/module/_pypyjson/__init__.py b/pypy/module/_pypyjson/moduledef.py copy from pypy/module/_pypyjson/__init__.py copy to pypy/module/_pypyjson/moduledef.py diff --git a/pypy/module/_random/__init__.py b/pypy/module/_random/__init__.py --- a/pypy/module/_random/__init__.py +++ b/pypy/module/_random/__init__.py @@ -1,8 +0,0 @@ -from pypy.interpreter.mixedmodule import MixedModule - -class Module(MixedModule): - appleveldefs = {} - - interpleveldefs = { - 'Random' : 'interp_random.W_Random', - } diff --git a/pypy/module/_random/__init__.py b/pypy/module/_random/moduledef.py copy from pypy/module/_random/__init__.py copy to pypy/module/_random/moduledef.py diff --git a/pypy/module/_rawffi/__init__.py b/pypy/module/_rawffi/__init__.py --- a/pypy/module/_rawffi/__init__.py +++ b/pypy/module/_rawffi/__init__.py @@ -1,58 +0,0 @@ -""" Low-level interface to clibffi -""" - -from pypy.interpreter.mixedmodule import MixedModule -from pypy.module._rawffi import alt - -class Module(MixedModule): - interpleveldefs = { - 'CDLL' : 'interp_rawffi.W_CDLL', - 'FuncPtr' : 'interp_rawffi.W_FuncPtr', - 'Structure' : 'structure.W_Structure', - 'StructureInstance' : 'structure.W_StructureInstance', - 'StructureInstanceAutoFree' : 'structure.W_StructureInstanceAutoFree', - 'Array' : 'array.W_Array', - 'ArrayInstance' : 'array.W_ArrayInstance', - 'ArrayInstanceAutoFree' : 'array.W_ArrayInstanceAutoFree', - 'sizeof' : 'interp_rawffi.sizeof', - 'alignment' : 'interp_rawffi.alignment', - 'charp2string' : 'interp_rawffi.charp2string', - 'wcharp2unicode' : 'interp_rawffi.wcharp2unicode', - 'charp2rawstring' : 'interp_rawffi.charp2rawstring', - 'wcharp2rawunicode' : 'interp_rawffi.wcharp2rawunicode', - 'rawstring2charp' : 'interp_rawffi.rawstring2charp', - 'CallbackPtr' : 'callback.W_CallbackPtr', - '_num_of_allocated_objects' : 'tracker.num_of_allocated_objects', - 'get_libc' : 'interp_rawffi.get_libc', - 'get_errno' : 'interp_rawffi.get_errno', - 'set_errno' : 'interp_rawffi.set_errno', - 'get_last_error' : 'interp_rawffi.get_last_error', - 'set_last_error' : 'interp_rawffi.set_last_error', - 'SegfaultException' : 'space.new_exception_class("_rawffi.SegfaultException")', - 'exit' : 'interp_exit.exit', - } - - appleveldefs = { - } - - submodules = { - 'alt': alt.Module, - } - - def buildloaders(cls): - from pypy.module._rawffi import interp_rawffi - - if hasattr(interp_rawffi, 'FormatError'): - Module.interpleveldefs['FormatError'] = 'interp_rawffi.FormatError' - if hasattr(interp_rawffi, 'check_HRESULT'): - Module.interpleveldefs['check_HRESULT'] = 'interp_rawffi.check_HRESULT' - - from rpython.rlib import clibffi - for name in ['FUNCFLAG_STDCALL', 'FUNCFLAG_CDECL', 'FUNCFLAG_PYTHONAPI', - 'FUNCFLAG_USE_ERRNO', 'FUNCFLAG_USE_LASTERROR', - ]: - if hasattr(clibffi, name): - Module.interpleveldefs[name] = "space.wrap(%r)" % getattr(clibffi, name) - - super(Module, cls).buildloaders() - buildloaders = classmethod(buildloaders) diff --git a/pypy/module/_rawffi/__init__.py b/pypy/module/_rawffi/moduledef.py copy from pypy/module/_rawffi/__init__.py copy to pypy/module/_rawffi/moduledef.py diff --git a/pypy/module/_socket/__init__.py b/pypy/module/_socket/__init__.py --- a/pypy/module/_socket/__init__.py +++ b/pypy/module/_socket/__init__.py @@ -1,52 +0,0 @@ -from pypy.interpreter.mixedmodule import MixedModule -from rpython.rlib.rsocket import SOMAXCONN - - -class Module(MixedModule): - - appleveldefs = { - } - - interpleveldefs = { - 'SocketType': 'interp_socket.W_Socket', - 'socket' : 'interp_socket.W_Socket', - 'error' : 'interp_socket.get_error(space, "error")', - 'herror' : 'interp_socket.get_error(space, "herror")', - 'gaierror' : 'interp_socket.get_error(space, "gaierror")', - 'timeout' : 'interp_socket.get_error(space, "timeout")', - 'SOMAXCONN' : 'space.wrap(%d)' % SOMAXCONN, - } - - def startup(self, space): - from rpython.rlib.rsocket import rsocket_startup - rsocket_startup() - - def shutdown(self, space): - from pypy.module._socket.interp_socket import close_all_sockets - close_all_sockets(space) - - def buildloaders(cls): - from rpython.rlib import rsocket - for name in """ - gethostbyname gethostbyname_ex gethostbyaddr gethostname - getservbyname getservbyport getprotobyname - dup socketpair - ntohs ntohl htons htonl inet_aton inet_ntoa inet_pton inet_ntop - getaddrinfo getnameinfo - getdefaulttimeout setdefaulttimeout - CMSG_SPACE CMSG_LEN - """.split(): - - if (name in ('inet_pton', 'inet_ntop', 'socketpair', - 'CMSG_SPACE', 'CMSG_LEN') and - not hasattr(rsocket, name)): - continue - - Module.interpleveldefs[name] = 'interp_func.%s' % (name, ) - - for constant, value in rsocket.constants.iteritems(): - Module.interpleveldefs[constant] = "space.wrap(%r)" % value - super(Module, cls).buildloaders() - buildloaders = classmethod(buildloaders) - -#Module.interpleveldefs['has_ipv6'] = "space.wrap(%s)" % _socket.has_ipv6 diff --git a/pypy/module/_socket/__init__.py b/pypy/module/_socket/moduledef.py copy from pypy/module/_socket/__init__.py copy to pypy/module/_socket/moduledef.py diff --git a/pypy/module/_sre/__init__.py b/pypy/module/_sre/__init__.py --- a/pypy/module/_sre/__init__.py +++ b/pypy/module/_sre/__init__.py @@ -1,16 +0,0 @@ -from pypy.interpreter.mixedmodule import MixedModule - -class Module(MixedModule): - - appleveldefs = { - } - - interpleveldefs = { - 'CODESIZE': 'space.newint(interp_sre.CODESIZE)', - 'MAGIC': 'space.newint(20140917)', - 'MAXREPEAT': 'space.newint(interp_sre.MAXREPEAT)', - 'MAXGROUPS': 'space.newint(interp_sre.MAXGROUPS)', - 'compile': 'interp_sre.W_SRE_Pattern', - 'getlower': 'interp_sre.w_getlower', - 'getcodesize': 'interp_sre.w_getcodesize', - } diff --git a/pypy/module/_sre/__init__.py b/pypy/module/_sre/moduledef.py copy from pypy/module/_sre/__init__.py copy to pypy/module/_sre/moduledef.py diff --git a/pypy/module/_string/__init__.py b/pypy/module/_string/__init__.py --- a/pypy/module/_string/__init__.py +++ b/pypy/module/_string/__init__.py @@ -1,17 +0,0 @@ -"""A _string module, to export formatter_parser and - formatter_field_name_split to the string.Formatter class - implemented in Python.""" - - -from pypy.interpreter.mixedmodule import MixedModule - -class Module(MixedModule): - "string helper module" - - interpleveldefs = { - 'formatter_field_name_split': 'formatter.formatter_field_name_split', - 'formatter_parser': 'formatter.formatter_parser', - } - - appleveldefs = {} - diff --git a/pypy/module/_string/__init__.py b/pypy/module/_string/moduledef.py copy from pypy/module/_string/__init__.py copy to pypy/module/_string/moduledef.py diff --git a/pypy/module/_testing/__init__.py b/pypy/module/_testing/__init__.py --- a/pypy/module/_testing/__init__.py +++ b/pypy/module/_testing/__init__.py @@ -1,17 +0,0 @@ - -""" -Mixed-module definition for pypy own testing purposes -""" - -from pypy.interpreter.mixedmodule import MixedModule - - -class Module(MixedModule): - """PyPy own testing""" - - interpleveldefs = { - } - - appleveldefs = { - 'Hidden': 'app_notrpython.Hidden', - } diff --git a/pypy/module/_testing/__init__.py b/pypy/module/_testing/moduledef.py copy from pypy/module/_testing/__init__.py copy to pypy/module/_testing/moduledef.py diff --git a/pypy/module/_vmprof/__init__.py b/pypy/module/_vmprof/__init__.py --- a/pypy/module/_vmprof/__init__.py +++ b/pypy/module/_vmprof/__init__.py @@ -1,39 +0,0 @@ -from pypy.interpreter.mixedmodule import MixedModule -from rpython.rlib.rvmprof import VMProfPlatformUnsupported -from rpython.translator.platform import CompilationError - - -class Module(MixedModule): - """ - VMProf for PyPy: a statistical profiler - """ - appleveldefs = { - } - - interpleveldefs = { - 'enable': 'interp_vmprof.enable', - 'disable': 'interp_vmprof.disable', - 'is_enabled': 'interp_vmprof.is_enabled', - 'get_profile_path': 'interp_vmprof.get_profile_path', - 'stop_sampling': 'interp_vmprof.stop_sampling', - 'start_sampling': 'interp_vmprof.start_sampling', - - 'VMProfError': 'space.fromcache(interp_vmprof.Cache).w_VMProfError', - } - - -# Force the __extend__ hacks and method replacements to occur -# early. Without this, for example, 'PyCode._init_ready' was -# already found by the annotator to be the original empty -# method, and the annotator doesn't notice that interp_vmprof.py -# (loaded later) replaces this method. -try: - import pypy.module._vmprof.interp_vmprof -except VMProfPlatformUnsupported as e: - pass -except CompilationError as e: - import sys - if sys.platform == 'win32': - pass - else: - raise diff --git a/pypy/module/_vmprof/__init__.py b/pypy/module/_vmprof/moduledef.py copy from pypy/module/_vmprof/__init__.py copy to pypy/module/_vmprof/moduledef.py diff --git a/pypy/module/_warnings/__init__.py b/pypy/module/_warnings/__init__.py --- a/pypy/module/_warnings/__init__.py +++ b/pypy/module/_warnings/__init__.py @@ -1,22 +0,0 @@ -from pypy.interpreter.mixedmodule import MixedModule - -class Module(MixedModule): - """provides basic warning filtering support. - It is a helper module to speed up interpreter start-up.""" - - interpleveldefs = { - 'warn' : 'interp_warnings.warn', - 'warn_explicit': 'interp_warnings.warn_explicit', - '_filters_mutated': 'interp_warnings.filters_mutated', - } - - appleveldefs = { - } - - def setup_after_space_initialization(self): - from pypy.module._warnings.interp_warnings import State - state = self.space.fromcache(State) - self.setdictvalue(self.space, "filters", state.w_filters) - self.setdictvalue(self.space, "_onceregistry", state.w_once_registry) - self.setdictvalue(self.space, "_defaultaction", state.w_default_action) - diff --git a/pypy/module/_warnings/__init__.py b/pypy/module/_warnings/moduledef.py copy from pypy/module/_warnings/__init__.py copy to pypy/module/_warnings/moduledef.py diff --git a/pypy/module/_weakref/__init__.py b/pypy/module/_weakref/__init__.py --- a/pypy/module/_weakref/__init__.py +++ b/pypy/module/_weakref/__init__.py @@ -1,14 +0,0 @@ -from pypy.interpreter.mixedmodule import MixedModule - -class Module(MixedModule): - appleveldefs = { - } - interpleveldefs = { - 'ref': 'interp__weakref.W_Weakref', - 'getweakrefcount': 'interp__weakref.getweakrefcount', - 'getweakrefs': 'interp__weakref.getweakrefs', - 'ReferenceType': 'interp__weakref.W_Weakref', - 'ProxyType': 'interp__weakref.W_Proxy', - 'CallableProxyType': 'interp__weakref.W_CallableProxy', - 'proxy': 'interp__weakref.proxy' - } diff --git a/pypy/module/_weakref/__init__.py b/pypy/module/_weakref/moduledef.py copy from pypy/module/_weakref/__init__.py copy to pypy/module/_weakref/moduledef.py diff --git a/pypy/module/_winreg/__init__.py b/pypy/module/_winreg/__init__.py --- a/pypy/module/_winreg/__init__.py +++ b/pypy/module/_winreg/__init__.py @@ -1,76 +0,0 @@ -from pypy.interpreter.mixedmodule import MixedModule -from rpython.rlib.rwinreg import constants - -class Module(MixedModule): - """This module provides access to the Windows registry API. - -Functions: - -CloseKey() - Closes a registry key. -ConnectRegistry() - Establishes a connection to a predefined registry handle - on another computer. -CreateKey() - Creates the specified key, or opens it if it already exists. -DeleteKey() - Deletes the specified key. -DeleteValue() - Removes a named value from the specified registry key. -EnumKey() - Enumerates subkeys of the specified open registry key. -EnumValue() - Enumerates values of the specified open registry key. -ExpandEnvironmentStrings() - Expand the env strings in a REG_EXPAND_SZ string. -FlushKey() - Writes all the attributes of the specified key to the registry. -LoadKey() - Creates a subkey under HKEY_USER or HKEY_LOCAL_MACHINE and stores - registration information from a specified file into that subkey. -OpenKey() - Alias for -OpenKeyEx() - Opens the specified key. -QueryValue() - Retrieves the value associated with the unnamed value for a - specified key in the registry. -QueryValueEx() - Retrieves the type and data for a specified value name - associated with an open registry key. -QueryInfoKey() - Returns information about the specified key. -SaveKey() - Saves the specified key, and all its subkeys a file. -SetValue() - Associates a value with a specified key. -SetValueEx() - Stores data in the value field of an open registry key. - -Special objects: - -HKEYType -- type object for HKEY objects -error -- exception raised for Win32 errors - -Integer constants: -Many constants are defined - see the documentation for each function -to see what constants are used, and where.""" - - applevel_name = 'winreg' - - appleveldefs = { - } - interpleveldefs = { - 'error' : 'space.w_WindowsError', - 'HKEYType' : 'interp_winreg.W_HKEY', - 'SetValue' : 'interp_winreg.SetValue', - 'SetValueEx' : 'interp_winreg.SetValueEx', - 'QueryValue' : 'interp_winreg.QueryValue', - 'QueryValueEx' : 'interp_winreg.QueryValueEx', - 'CreateKey' : 'interp_winreg.CreateKey', - 'CreateKeyEx' : 'interp_winreg.CreateKeyEx', - 'DeleteKey' : 'interp_winreg.DeleteKey', - 'DeleteValue' : 'interp_winreg.DeleteValue', - 'OpenKey' : 'interp_winreg.OpenKey', - 'OpenKeyEx' : 'interp_winreg.OpenKey', - 'EnumValue' : 'interp_winreg.EnumValue', - 'EnumKey' : 'interp_winreg.EnumKey', - 'FlushKey' : 'interp_winreg.FlushKey', - 'CloseKey' : 'interp_winreg.CloseKey', - 'QueryInfoKey' : 'interp_winreg.QueryInfoKey', - 'LoadKey' : 'interp_winreg.LoadKey', - 'SaveKey' : 'interp_winreg.SaveKey', - 'ConnectRegistry': 'interp_winreg.ConnectRegistry', - - 'ExpandEnvironmentStrings': 'interp_winreg.ExpandEnvironmentStrings', - - 'DisableReflectionKey': 'interp_winreg.DisableReflectionKey', - 'EnableReflectionKey': 'interp_winreg.EnableReflectionKey', - 'QueryReflectionKey': 'interp_winreg.QueryReflectionKey', - 'DeleteKeyEx': 'interp_winreg.DeleteKeyEx', - } - - for name, value in constants.iteritems(): - interpleveldefs[name] = "space.wrap(%s)" % (value,) diff --git a/pypy/module/_winreg/__init__.py b/pypy/module/_winreg/moduledef.py copy from pypy/module/_winreg/__init__.py copy to pypy/module/_winreg/moduledef.py diff --git a/pypy/module/array/__init__.py b/pypy/module/array/__init__.py --- a/pypy/module/array/__init__.py +++ b/pypy/module/array/__init__.py @@ -1,11 +0,0 @@ -from pypy.interpreter.mixedmodule import MixedModule - -class Module(MixedModule): - interpleveldefs = { - 'array': 'interp_array.W_ArrayBase', - 'ArrayType': 'interp_array.W_ArrayBase', - '_array_reconstructor': 'reconstructor.array_reconstructor', - } - - appleveldefs = { - } diff --git a/pypy/module/array/__init__.py b/pypy/module/array/moduledef.py copy from pypy/module/array/__init__.py copy to pypy/module/array/moduledef.py diff --git a/pypy/module/atexit/__init__.py b/pypy/module/atexit/__init__.py --- a/pypy/module/atexit/__init__.py +++ b/pypy/module/atexit/__init__.py @@ -1,20 +0,0 @@ -from pypy.interpreter.mixedmodule import MixedModule - -class Module(MixedModule): - """Allow programmer to define multiple exit functions to be - executed upon normal program termination. - - Two public functions, register and unregister, are defined. - """ - - interpleveldefs = { - } - - appleveldefs = { - 'register': 'app_atexit.register', - 'unregister': 'app_atexit.unregister', - '_clear': 'app_atexit.clear', - '_run_exitfuncs': 'app_atexit.run_exitfuncs', - '_ncallbacks': 'app_atexit.ncallbacks', - } - diff --git a/pypy/module/atexit/__init__.py b/pypy/module/atexit/moduledef.py copy from pypy/module/atexit/__init__.py copy to pypy/module/atexit/moduledef.py diff --git a/pypy/module/binascii/__init__.py b/pypy/module/binascii/__init__.py --- a/pypy/module/binascii/__init__.py +++ b/pypy/module/binascii/__init__.py @@ -1,36 +0,0 @@ - -""" -Mixed-module definition for the binascii module. -Note that there is also a pure Python implementation in lib_pypy/binascii.py; -the pypy/module/binascii/ version takes precedence if it is enabled. -""" - -from pypy.interpreter.mixedmodule import MixedModule - - -class Module(MixedModule): - """binascii - Conversion between binary data and ASCII""" - - appleveldefs = { - } - - interpleveldefs = { - 'a2b_uu': 'interp_uu.a2b_uu', - 'b2a_uu': 'interp_uu.b2a_uu', - 'a2b_base64': 'interp_base64.a2b_base64', - 'b2a_base64': 'interp_base64.b2a_base64', - 'a2b_qp': 'interp_qp.a2b_qp', - 'b2a_qp': 'interp_qp.b2a_qp', - 'a2b_hqx': 'interp_hqx.a2b_hqx', - 'b2a_hqx': 'interp_hqx.b2a_hqx', - 'rledecode_hqx': 'interp_hqx.rledecode_hqx', - 'rlecode_hqx': 'interp_hqx.rlecode_hqx', - 'crc_hqx': 'interp_hqx.crc_hqx', - 'crc32': 'interp_crc32.crc32', - 'b2a_hex': 'interp_hexlify.hexlify', - 'hexlify': 'interp_hexlify.hexlify', - 'a2b_hex': 'interp_hexlify.unhexlify', - 'unhexlify': 'interp_hexlify.unhexlify', - 'Error' : 'space.fromcache(interp_binascii.Cache).w_error', - 'Incomplete': 'space.fromcache(interp_binascii.Cache).w_incomplete', - } diff --git a/pypy/module/binascii/__init__.py b/pypy/module/binascii/moduledef.py copy from pypy/module/binascii/__init__.py copy to pypy/module/binascii/moduledef.py diff --git a/pypy/module/bz2/__init__.py b/pypy/module/bz2/__init__.py --- a/pypy/module/bz2/__init__.py +++ b/pypy/module/bz2/__init__.py @@ -1,14 +0,0 @@ -from pypy.interpreter.mixedmodule import MixedModule - -class Module(MixedModule): - # The private part of the bz2 module. - - applevel_name = '_bz2' - - interpleveldefs = { - 'BZ2Compressor': 'interp_bz2.W_BZ2Compressor', - 'BZ2Decompressor': 'interp_bz2.W_BZ2Decompressor', - } - - appleveldefs = { - } diff --git a/pypy/module/bz2/__init__.py b/pypy/module/bz2/moduledef.py copy from pypy/module/bz2/__init__.py copy to pypy/module/bz2/moduledef.py diff --git a/pypy/module/cmath/__init__.py b/pypy/module/cmath/__init__.py --- a/pypy/module/cmath/__init__.py +++ b/pypy/module/cmath/__init__.py @@ -1,46 +0,0 @@ - -# Package initialisation -from pypy.interpreter.mixedmodule import MixedModule - -names_and_docstrings = { - 'sqrt': "Return the square root of x.", - 'acos': "Return the arc cosine of x.", - 'acosh': "Return the hyperbolic arc cosine of x.", - 'asin': "Return the arc sine of x.", - 'asinh': "Return the hyperbolic arc sine of x.", - 'atan': "Return the arc tangent of x.", - 'atanh': "Return the hyperbolic arc tangent of x.", - 'log': ("log(x[, base]) -> the logarithm of x to the given base.\n" - "If the base not specified, returns the natural logarithm " - "(base e) of x."), - 'log10': "Return the base-10 logarithm of x.", - 'exp': "Return the exponential value e**x.", - 'cosh': "Return the hyperbolic cosine of x.", - 'sinh': "Return the hyperbolic sine of x.", - 'tanh': "Return the hyperbolic tangent of x.", - 'cos': "Return the cosine of x.", - 'sin': "Return the sine of x.", - 'tan': "Return the tangent of x.", - 'rect': "Convert from polar coordinates to rectangular coordinates.", - 'polar': ("polar(z) -> r: float, phi: float\n" - "Convert a complex from rectangular coordinates " - "to polar coordinates. r is\n" - "the distance from 0 and phi the phase angle."), - 'phase': "Return argument, also known as the phase angle, of a complex.", - 'isinf': "Checks if the real or imaginary part of z is infinite.", - 'isnan': "Checks if the real or imaginary part of z is not a number (NaN)", - 'isfinite': "isfinite(z) -> bool\nReturn True if both the real and imaginary parts of z are finite, else False.", -} - - -class Module(MixedModule): - appleveldefs = { - } - - interpleveldefs = { - 'pi': 'space.newfloat(interp_cmath.pi)', - 'e': 'space.newfloat(interp_cmath.e)', - 'isclose': 'interp_cmath.isclose', - } - interpleveldefs.update(dict([(name, 'interp_cmath.wrapped_' + name) - for name in names_and_docstrings])) diff --git a/pypy/module/cmath/__init__.py b/pypy/module/cmath/moduledef.py copy from pypy/module/cmath/__init__.py copy to pypy/module/cmath/moduledef.py diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py --- a/pypy/module/cpyext/__init__.py +++ b/pypy/module/cpyext/__init__.py @@ -1,83 +0,0 @@ -from pypy.interpreter.mixedmodule import MixedModule -from pypy.module.cpyext.state import State -from pypy.module.cpyext import api - -class Module(MixedModule): - interpleveldefs = { - 'is_cpyext_function': 'interp_cpyext.is_cpyext_function', - } - - appleveldefs = { - } - - atexit_funcs = [] - - def startup(self, space): - space.fromcache(State).startup(space) - method = pypy.module.cpyext.typeobject.get_new_method_def(space) - # the w_self argument here is a dummy, the only thing done with w_obj - # is call type() on it - w_obj = pypy.module.cpyext.methodobject.W_PyCFunctionObject(space, - method, space.w_None) - space.appexec([w_obj], """(meth): - from pickle import Pickler - Pickler.dispatch[type(meth)] = Pickler.save_global - """) - - def register_atexit(self, function): - if len(self.atexit_funcs) >= 32: - raise ValueError("cannot register more than 32 atexit functions") - self.atexit_funcs.append(function) - - def shutdown(self, space): - for func in self.atexit_funcs: - func() - - -# import these modules to register api functions by side-effect -import pypy.module.cpyext.pyobject -import pypy.module.cpyext.boolobject -import pypy.module.cpyext.floatobject -import pypy.module.cpyext.modsupport -import pypy.module.cpyext.pythonrun -import pypy.module.cpyext.pyerrors -import pypy.module.cpyext.typeobject -import pypy.module.cpyext.object -import pypy.module.cpyext.bytesobject -import pypy.module.cpyext.bytearrayobject -import pypy.module.cpyext.tupleobject -import pypy.module.cpyext.setobject -import pypy.module.cpyext.dictobject -import pypy.module.cpyext.longobject -import pypy.module.cpyext.listobject -import pypy.module.cpyext.sequence -import pypy.module.cpyext.buffer -import pypy.module.cpyext.eval -import pypy.module.cpyext.import_ -import pypy.module.cpyext.mapping -import pypy.module.cpyext.iterator -import pypy.module.cpyext.unicodeobject -import pypy.module.cpyext.sysmodule -import pypy.module.cpyext.number -import pypy.module.cpyext.sliceobject -import pypy.module.cpyext.stubsactive -import pypy.module.cpyext.pystate -import pypy.module.cpyext.cdatetime -import pypy.module.cpyext.complexobject -import pypy.module.cpyext.weakrefobject -import pypy.module.cpyext.funcobject -import pypy.module.cpyext.frameobject -import pypy.module.cpyext.classobject -import pypy.module.cpyext.exception -import pypy.module.cpyext.memoryobject -import pypy.module.cpyext.codecs -import pypy.module.cpyext.pyfile -import pypy.module.cpyext.pystrtod -import pypy.module.cpyext.pytraceback -import pypy.module.cpyext.methodobject -import pypy.module.cpyext.dictproxyobject -import pypy.module.cpyext.genobject -import pypy.module.cpyext.namespaceobject - -# now that all rffi_platform.Struct types are registered, configure them -api.configure_types() diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/moduledef.py copy from pypy/module/cpyext/__init__.py copy to pypy/module/cpyext/moduledef.py diff --git a/pypy/module/crypt/__init__.py b/pypy/module/crypt/__init__.py --- a/pypy/module/crypt/__init__.py +++ b/pypy/module/crypt/__init__.py @@ -1,11 +0,0 @@ -from pypy.interpreter.mixedmodule import MixedModule - -class Module(MixedModule): - applevel_name = '_crypt' - - interpleveldefs = { - 'crypt' : 'interp_crypt.crypt', - } - - appleveldefs = { - } diff --git a/pypy/module/crypt/__init__.py b/pypy/module/crypt/moduledef.py copy from pypy/module/crypt/__init__.py copy to pypy/module/crypt/moduledef.py diff --git a/pypy/module/errno/__init__.py b/pypy/module/errno/__init__.py --- a/pypy/module/errno/__init__.py +++ b/pypy/module/errno/__init__.py @@ -1,23 +0,0 @@ -from pypy.interpreter.mixedmodule import MixedModule -from pypy.module.errno.interp_errno import name2code - -class Module(MixedModule): - """This module makes available standard errno system symbols. - - The value of each symbol is the corresponding integer value, - e.g., on most systems, errno.ENOENT equals the integer 2. - - The dictionary errno.errorcode maps numeric codes to symbol names, - e.g., errno.errorcode[2] could be the string 'ENOENT'. - - Symbols that are not relevant to the underlying system are not defined. - - To map error codes to error messages, use the function os.strerror(), - e.g. os.strerror(2) could return 'No such file or directory'.""" - - appleveldefs = {} - interpleveldefs = {"errorcode": "interp_errno.get_errorcode(space)"} - -for name, code in name2code.iteritems(): - if code is not None: - Module.interpleveldefs[name] = ("space.newint(%s)" % code) diff --git a/pypy/module/errno/__init__.py b/pypy/module/errno/moduledef.py copy from pypy/module/errno/__init__.py copy to pypy/module/errno/moduledef.py diff --git a/pypy/module/exceptions/__init__.py b/pypy/module/exceptions/__init__.py --- a/pypy/module/exceptions/__init__.py +++ b/pypy/module/exceptions/__init__.py @@ -1,85 +0,0 @@ -from rpython.rlib import rwin32 -from pypy.interpreter.mixedmodule import MixedModule - -class Module(MixedModule): - applevel_name = '__exceptions__' - appleveldefs = {} - - interpleveldefs = { - 'ArithmeticError' : 'interp_exceptions.W_ArithmeticError', - 'AssertionError' : 'interp_exceptions.W_AssertionError', - 'AttributeError' : 'interp_exceptions.W_AttributeError', - 'BaseException' : 'interp_exceptions.W_BaseException', - 'BlockingIOError': 'interp_exceptions.W_BlockingIOError', - 'BrokenPipeError': 'interp_exceptions.W_BrokenPipeError', - 'BufferError' : 'interp_exceptions.W_BufferError', - 'BytesWarning' : 'interp_exceptions.W_BytesWarning', - 'ChildProcessError': 'interp_exceptions.W_ChildProcessError', - 'ConnectionAbortedError': 'interp_exceptions.W_ConnectionAbortedError', - 'ConnectionError': 'interp_exceptions.W_ConnectionError', - 'ConnectionRefusedError': 'interp_exceptions.W_ConnectionRefusedError', - 'ConnectionResetError': 'interp_exceptions.W_ConnectionResetError', - 'DeprecationWarning' : 'interp_exceptions.W_DeprecationWarning', - 'EOFError' : 'interp_exceptions.W_EOFError', - 'EnvironmentError' : 'interp_exceptions.W_OSError', - 'Exception' : 'interp_exceptions.W_Exception', - 'FileExistsError': 'interp_exceptions.W_FileExistsError', - 'FileNotFoundError': 'interp_exceptions.W_FileNotFoundError', - 'FloatingPointError' : 'interp_exceptions.W_FloatingPointError', - 'FutureWarning' : 'interp_exceptions.W_FutureWarning', - 'GeneratorExit' : 'interp_exceptions.W_GeneratorExit', - 'IOError' : 'interp_exceptions.W_OSError', - 'ImportError' : 'interp_exceptions.W_ImportError', - 'ImportWarning' : 'interp_exceptions.W_ImportWarning', - 'IndentationError' : 'interp_exceptions.W_IndentationError', - 'IndexError' : 'interp_exceptions.W_IndexError', - 'InterruptedError': 'interp_exceptions.W_InterruptedError', - 'IsADirectoryError': 'interp_exceptions.W_IsADirectoryError', - 'KeyError' : 'interp_exceptions.W_KeyError', - 'KeyboardInterrupt' : 'interp_exceptions.W_KeyboardInterrupt', - 'LookupError' : 'interp_exceptions.W_LookupError', - 'MemoryError' : 'interp_exceptions.W_MemoryError', - 'NameError' : 'interp_exceptions.W_NameError', - 'NotADirectoryError': 'interp_exceptions.W_NotADirectoryError', - 'NotImplementedError' : 'interp_exceptions.W_NotImplementedError', - 'OSError' : 'interp_exceptions.W_OSError', - 'OverflowError' : 'interp_exceptions.W_OverflowError', - 'PendingDeprecationWarning' : 'interp_exceptions.W_PendingDeprecationWarning', - 'PermissionError': 'interp_exceptions.W_PermissionError', - 'ProcessLookupError': 'interp_exceptions.W_ProcessLookupError', - 'RecursionError' : 'interp_exceptions.W_RecursionError', - 'ReferenceError' : 'interp_exceptions.W_ReferenceError', - 'ResourceWarning' : 'interp_exceptions.W_ResourceWarning', - 'RuntimeError' : 'interp_exceptions.W_RuntimeError', - 'RuntimeWarning' : 'interp_exceptions.W_RuntimeWarning', - 'StopAsyncIteration' : 'interp_exceptions.W_StopAsyncIteration', - 'StopIteration' : 'interp_exceptions.W_StopIteration', - 'SyntaxError' : 'interp_exceptions.W_SyntaxError', - 'SyntaxWarning' : 'interp_exceptions.W_SyntaxWarning', - 'SystemError' : 'interp_exceptions.W_SystemError', - 'SystemExit' : 'interp_exceptions.W_SystemExit', - 'TabError' : 'interp_exceptions.W_TabError', - 'TimeoutError': 'interp_exceptions.W_TimeoutError', - 'TypeError' : 'interp_exceptions.W_TypeError', - 'UnboundLocalError' : 'interp_exceptions.W_UnboundLocalError', - 'UnicodeDecodeError' : 'interp_exceptions.W_UnicodeDecodeError', - 'UnicodeEncodeError' : 'interp_exceptions.W_UnicodeEncodeError', - 'UnicodeError' : 'interp_exceptions.W_UnicodeError', - 'UnicodeTranslateError' : 'interp_exceptions.W_UnicodeTranslateError', - 'UnicodeWarning' : 'interp_exceptions.W_UnicodeWarning', - 'UserWarning' : 'interp_exceptions.W_UserWarning', - 'ValueError' : 'interp_exceptions.W_ValueError', - 'Warning' : 'interp_exceptions.W_Warning', - 'ZeroDivisionError' : 'interp_exceptions.W_ZeroDivisionError', - } - - if rwin32.WIN32: - interpleveldefs['WindowsError'] = 'interp_exceptions.W_OSError' - - def setup_after_space_initialization(self): - from pypy.objspace.std.transparent import register_proxyable - from pypy.module.exceptions import interp_exceptions - - for name, exc in interp_exceptions.__dict__.items(): - if isinstance(exc, type) and issubclass(exc, interp_exceptions.W_BaseException): - register_proxyable(self.space, exc) diff --git a/pypy/module/exceptions/__init__.py b/pypy/module/exceptions/moduledef.py copy from pypy/module/exceptions/__init__.py copy to pypy/module/exceptions/moduledef.py diff --git a/pypy/module/faulthandler/__init__.py b/pypy/module/faulthandler/__init__.py --- a/pypy/module/faulthandler/__init__.py +++ b/pypy/module/faulthandler/__init__.py @@ -1,38 +0,0 @@ -import sys -from pypy.interpreter.mixedmodule import MixedModule - - -class Module(MixedModule): - appleveldefs = { - } From pypy.commits at gmail.com Mon Apr 16 22:48:36 2018 From: pypy.commits at gmail.com (rlamy) Date: Mon, 16 Apr 2018 19:48:36 -0700 (PDT) Subject: [pypy-commit] pypy py3tests: apptest_ files do need to be collected when running without -A Message-ID: <5ad56084.c2091c0a.19ae1.1472@mx.google.com> Author: Ronan Lamy Branch: py3tests Changeset: r94357:88f2cbae0c90 Date: 2018-04-17 03:45 +0100 http://bitbucket.org/pypy/pypy/changeset/88f2cbae0c90/ Log: apptest_ files do need to be collected when running without -A diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -48,7 +48,7 @@ def py3k_skip(message): py.test.skip('[py3k] %s' % message) py.test.py3k_skip = py3k_skip - if HOST_IS_PY3: + if HOST_IS_PY3 or not config.getoption('runappdirect'): config.addinivalue_line('python_files', APPLEVEL_FN) def pytest_addoption(parser): From pypy.commits at gmail.com Tue Apr 17 05:24:31 2018 From: pypy.commits at gmail.com (antocuni) Date: Tue, 17 Apr 2018 02:24:31 -0700 (PDT) Subject: [pypy-commit] pypy gc-hooks: add docs about GC hooks Message-ID: <5ad5bd4f.138f1c0a.c4860.c745@mx.google.com> Author: Antonio Cuni Branch: gc-hooks Changeset: r94358:3fe7e9dc4d49 Date: 2018-04-17 11:22 +0200 http://bitbucket.org/pypy/pypy/changeset/3fe7e9dc4d49/ Log: add docs about GC hooks diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst --- a/pypy/doc/gc_info.rst +++ b/pypy/doc/gc_info.rst @@ -121,6 +121,160 @@ alive by GC objects, but not accounted in the GC +GC Hooks +-------- + +GC hooks are user-defined functions which are called whenever a specific GC +event occur, and can be used to monitor GC activity and pauses. You can +install the hooks by setting the following attributes: + +``gc.hook.on_gc_minor`` + Called whenever a minor collection occurs. It corresponds to + ``gc-minor`` sections inside ``PYPYLOG``. + +``gc.hook.on_gc_collect_step`` + Called whenever an incremental step of a major collection occurs. It + corresponds to ``gc-collect-step`` sections inside ``PYPYLOG``. + +``gc.hook.on_gc_collect`` + Called after the last incremental step, when a major collection is fully + done. It corresponds to ``gc-collect-done`` sections inside ``PYPYLOG``. + +To uninstall a hook, simply set the corresponding attribute to ``None``. To +install all hooks at once, you can call ``gc.hooks.set(obj)``, which will look +for methods ``on_gc_*`` on ``obj``. To uninstall all the hooks at once, you +can call ``gc.hooks.reset()``. + +The functions called by the hooks receive a single ``stats`` argument, which +contains various statistics about the event. + +Note that PyPy cannot call the hooks immediately after a GC event, but it has +to wait until it reaches a point in which the interpreter is in a known state +and calling user-defined code is harmless. It might happen that multiple +events occur before the hook is invoked: in this case, you can inspect the +value ``stats.count`` to know how many times the event occured since the last +time the hook was called. Similarly, ``stats.duration`` contains the +**total** time spent by the GC for this specific event since the last time the +hook was called. + +On the other hand, all the other fields of the ``stats`` object are relative +only to the **last** event of the series. + +The attributes for ``GcMinorStats`` are: + +``count`` + The number of minor collections occured since the last hook call. + +``duration`` + The total time spent inside minor collections since the last hook + call. See below for more information on the unit. + + ``total_memory_used`` + The amount of memory used at the end of the minor collection, in + bytes. This include the memory used in arenas (for GC-managed memory) and + raw-malloced memory (e.g., the content of numpy arrays). + +``pinned_objects`` + the number of pinned objects. + + +The attributes for ``GcCollectStepStats`` are: + +``count``, ``duration`` + See above. + +``oldstate``, ``newstate`` + Integers which indicate the state of the GC before and after the step. + +The value of ``oldstate`` and ``newstate`` is one of these constants, defined +inside ``gc.GcCollectStepStats``: ``STATE_SCANNING``, ``STATE_MARKING``, +``STATE_SWEEPING``, ``STATE_FINALIZING``. It is possible to get a string +representation of it by indexing the ``GC_STATS`` tuple. + + +The attributes for ``GcCollectStats`` are: + +``count`` + See above. + +``num_major_collects`` + The total number of major collections which have been done since the + start. Contrarily to ``count``, this is an always-growing counter and it's + not reset between invocations. + +``arenas_count_before``, ``arenas_count_after`` + Number of arenas used before and after the major collection. + +``arenas_bytes`` + Total number of bytes used by GC-managed objects. + +``rawmalloc_bytes_before``, ``rawmalloc_bytes_after`` + Total number of bytes used by raw-malloced objects, before and after the + major collection. + +Note that ``GcCollectStats`` has **not** got a ``duration`` field. This is +because all the GC work is done inside ``gc-collect-step``: +``gc-collect-done`` is used only to give additional stats, but doesn't do any +actual work. + +A note about the ``duration`` field: depending on the architecture and +operating system, PyPy uses different ways to read timestamps, so ``duration`` +is expressed in varying units. It is possible to know which by calling +``__pypy__.debug_get_timestamp_unit()``, which can be one of the following +values: + +``tsc`` + The default on ``x86`` machines: timestamps are expressed in CPU ticks, as + read by the `Time Stamp Counter`_. + +``ns`` + Timestamps are expressed in nanoseconds. + +``QueryPerformanceCounter`` + On Windows, in case for some reason ``tsc`` is not available: timestamps + are read using the win API ``QueryPerformanceCounter()``. + + +Unfortunately, there does not seem to be a reliable standard way for +converting ``tsc`` ticks into nanoseconds, although in practice on modern CPUs +it is enough to divide the ticks by the maximum nominal frequency of the CPU. +For this reason, PyPy gives the raw value, and leaves the job of doing the +conversion to external libraries. + +Here is an example of GC hooks in use:: + + import sys + import gc + + class MyHooks(object): + done = False + + def on_gc_minor(self, stats): + print 'gc-minor: count = %02d, duration = %d' % (stats.count, + stats.duration) + + def on_gc_collect_step(self, stats): + old = gc.GcCollectStepStats.GC_STATES[stats.oldstate] + new = gc.GcCollectStepStats.GC_STATES[stats.newstate] + print 'gc-collect-step: %s --> %s' % (old, new) + print ' count = %02d, duration = %d' % (stats.count, + stats.duration) + + def on_gc_collect(self, stats): + print 'gc-collect-done: count = %02d' % stats.count + self.done = True + + hooks = MyHooks() + gc.hooks.set(hooks) + + # simulate some GC activity + lst = [] + while not hooks.done: + lst = [lst, 1, 2, 3] + + +.. _`Time Stamp Counter`: https://en.wikipedia.org/wiki/Time_Stamp_Counter + .. _minimark-environment-variables: Environment variables From pypy.commits at gmail.com Tue Apr 17 06:09:45 2018 From: pypy.commits at gmail.com (antocuni) Date: Tue, 17 Apr 2018 03:09:45 -0700 (PDT) Subject: [pypy-commit] pypy gc-hooks: update whatsnew Message-ID: <5ad5c7e9.88e51c0a.c9070.b3e4@mx.google.com> Author: Antonio Cuni Branch: gc-hooks Changeset: r94359:781821b4228a Date: 2018-04-17 12:07 +0200 http://bitbucket.org/pypy/pypy/changeset/781821b4228a/ Log: update whatsnew diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -10,3 +10,7 @@ Fix a rare GC bug that was introduced more than one year ago, but was not diagnosed before issue #2752. + +.. branch: gc-hooks + +Introduce GC hooks, as documented in doc/gc_info.rst From pypy.commits at gmail.com Tue Apr 17 06:09:47 2018 From: pypy.commits at gmail.com (antocuni) Date: Tue, 17 Apr 2018 03:09:47 -0700 (PDT) Subject: [pypy-commit] pypy gc-hooks: close merged branch Message-ID: <5ad5c7eb.8fc9df0a.45012.0d64@mx.google.com> Author: Antonio Cuni Branch: gc-hooks Changeset: r94360:7af841458a81 Date: 2018-04-17 12:08 +0200 http://bitbucket.org/pypy/pypy/changeset/7af841458a81/ Log: close merged branch From pypy.commits at gmail.com Tue Apr 17 06:09:49 2018 From: pypy.commits at gmail.com (antocuni) Date: Tue, 17 Apr 2018 03:09:49 -0700 (PDT) Subject: [pypy-commit] pypy default: merge the gc-hooks branch: it is now possible to install app-level hooks which are triggered whenever a specific GC activity occurs Message-ID: <5ad5c7ed.1887df0a.2d928.0900@mx.google.com> Author: Antonio Cuni Branch: Changeset: r94361:5bd740a5496c Date: 2018-04-17 12:09 +0200 http://bitbucket.org/pypy/pypy/changeset/5bd740a5496c/ Log: merge the gc-hooks branch: it is now possible to install app-level hooks which are triggered whenever a specific GC activity occurs diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst --- a/pypy/doc/gc_info.rst +++ b/pypy/doc/gc_info.rst @@ -121,6 +121,160 @@ alive by GC objects, but not accounted in the GC +GC Hooks +-------- + +GC hooks are user-defined functions which are called whenever a specific GC +event occur, and can be used to monitor GC activity and pauses. You can +install the hooks by setting the following attributes: + +``gc.hook.on_gc_minor`` + Called whenever a minor collection occurs. It corresponds to + ``gc-minor`` sections inside ``PYPYLOG``. + +``gc.hook.on_gc_collect_step`` + Called whenever an incremental step of a major collection occurs. It + corresponds to ``gc-collect-step`` sections inside ``PYPYLOG``. + +``gc.hook.on_gc_collect`` + Called after the last incremental step, when a major collection is fully + done. It corresponds to ``gc-collect-done`` sections inside ``PYPYLOG``. + +To uninstall a hook, simply set the corresponding attribute to ``None``. To +install all hooks at once, you can call ``gc.hooks.set(obj)``, which will look +for methods ``on_gc_*`` on ``obj``. To uninstall all the hooks at once, you +can call ``gc.hooks.reset()``. + +The functions called by the hooks receive a single ``stats`` argument, which +contains various statistics about the event. + +Note that PyPy cannot call the hooks immediately after a GC event, but it has +to wait until it reaches a point in which the interpreter is in a known state +and calling user-defined code is harmless. It might happen that multiple +events occur before the hook is invoked: in this case, you can inspect the +value ``stats.count`` to know how many times the event occured since the last +time the hook was called. Similarly, ``stats.duration`` contains the +**total** time spent by the GC for this specific event since the last time the +hook was called. + +On the other hand, all the other fields of the ``stats`` object are relative +only to the **last** event of the series. + +The attributes for ``GcMinorStats`` are: + +``count`` + The number of minor collections occured since the last hook call. + +``duration`` + The total time spent inside minor collections since the last hook + call. See below for more information on the unit. + + ``total_memory_used`` + The amount of memory used at the end of the minor collection, in + bytes. This include the memory used in arenas (for GC-managed memory) and + raw-malloced memory (e.g., the content of numpy arrays). + +``pinned_objects`` + the number of pinned objects. + + +The attributes for ``GcCollectStepStats`` are: + +``count``, ``duration`` + See above. + +``oldstate``, ``newstate`` + Integers which indicate the state of the GC before and after the step. + +The value of ``oldstate`` and ``newstate`` is one of these constants, defined +inside ``gc.GcCollectStepStats``: ``STATE_SCANNING``, ``STATE_MARKING``, +``STATE_SWEEPING``, ``STATE_FINALIZING``. It is possible to get a string +representation of it by indexing the ``GC_STATS`` tuple. + + +The attributes for ``GcCollectStats`` are: + +``count`` + See above. + +``num_major_collects`` + The total number of major collections which have been done since the + start. Contrarily to ``count``, this is an always-growing counter and it's + not reset between invocations. + +``arenas_count_before``, ``arenas_count_after`` + Number of arenas used before and after the major collection. + +``arenas_bytes`` + Total number of bytes used by GC-managed objects. + +``rawmalloc_bytes_before``, ``rawmalloc_bytes_after`` + Total number of bytes used by raw-malloced objects, before and after the + major collection. + +Note that ``GcCollectStats`` has **not** got a ``duration`` field. This is +because all the GC work is done inside ``gc-collect-step``: +``gc-collect-done`` is used only to give additional stats, but doesn't do any +actual work. + +A note about the ``duration`` field: depending on the architecture and +operating system, PyPy uses different ways to read timestamps, so ``duration`` +is expressed in varying units. It is possible to know which by calling +``__pypy__.debug_get_timestamp_unit()``, which can be one of the following +values: + +``tsc`` + The default on ``x86`` machines: timestamps are expressed in CPU ticks, as + read by the `Time Stamp Counter`_. + +``ns`` + Timestamps are expressed in nanoseconds. + +``QueryPerformanceCounter`` + On Windows, in case for some reason ``tsc`` is not available: timestamps + are read using the win API ``QueryPerformanceCounter()``. + + +Unfortunately, there does not seem to be a reliable standard way for +converting ``tsc`` ticks into nanoseconds, although in practice on modern CPUs +it is enough to divide the ticks by the maximum nominal frequency of the CPU. +For this reason, PyPy gives the raw value, and leaves the job of doing the +conversion to external libraries. + +Here is an example of GC hooks in use:: + + import sys + import gc + + class MyHooks(object): + done = False + + def on_gc_minor(self, stats): + print 'gc-minor: count = %02d, duration = %d' % (stats.count, + stats.duration) + + def on_gc_collect_step(self, stats): + old = gc.GcCollectStepStats.GC_STATES[stats.oldstate] + new = gc.GcCollectStepStats.GC_STATES[stats.newstate] + print 'gc-collect-step: %s --> %s' % (old, new) + print ' count = %02d, duration = %d' % (stats.count, + stats.duration) + + def on_gc_collect(self, stats): + print 'gc-collect-done: count = %02d' % stats.count + self.done = True + + hooks = MyHooks() + gc.hooks.set(hooks) + + # simulate some GC activity + lst = [] + while not hooks.done: + lst = [lst, 1, 2, 3] + + +.. _`Time Stamp Counter`: https://en.wikipedia.org/wiki/Time_Stamp_Counter + .. _minimark-environment-variables: Environment variables diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -10,3 +10,7 @@ Fix a rare GC bug that was introduced more than one year ago, but was not diagnosed before issue #2752. + +.. branch: gc-hooks + +Introduce GC hooks, as documented in doc/gc_info.rst diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -215,6 +215,7 @@ usage = SUPPRESS_USAGE take_options = True + space = None def opt_parser(self, config): parser = to_optparse(config, useoptions=["objspace.*"], @@ -364,15 +365,21 @@ from pypy.module.pypyjit.hooks import pypy_hooks return PyPyJitPolicy(pypy_hooks) + def get_gchooks(self): + from pypy.module.gc.hook import LowLevelGcHooks + if self.space is None: + raise Exception("get_gchooks must be called afeter get_entry_point") + return self.space.fromcache(LowLevelGcHooks) + def get_entry_point(self, config): - space = make_objspace(config) + self.space = make_objspace(config) # manually imports app_main.py filename = os.path.join(pypydir, 'interpreter', 'app_main.py') app = gateway.applevel(open(filename).read(), 'app_main.py', 'app_main') app.hidden_applevel = False - w_dict = app.getwdict(space) - entry_point, _ = create_entry_point(space, w_dict) + w_dict = app.getwdict(self.space) + entry_point, _ = create_entry_point(self.space, w_dict) return entry_point, None, PyPyAnnotatorPolicy() @@ -381,7 +388,7 @@ 'jitpolicy', 'get_entry_point', 'get_additional_config_options']: ns[name] = getattr(self, name) - + ns['get_gchooks'] = self.get_gchooks PyPyTarget().interface(globals()) diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -404,7 +404,7 @@ self._periodic_actions = [] self._nonperiodic_actions = [] self.has_bytecode_counter = False - self.fired_actions = None + self._fired_actions_reset() # the default value is not 100, unlike CPython 2.7, but a much # larger value, because we use a technique that not only allows # but actually *forces* another thread to run whenever the counter @@ -416,13 +416,28 @@ """Request for the action to be run before the next opcode.""" if not action._fired: action._fired = True - if self.fired_actions is None: - self.fired_actions = [] - self.fired_actions.append(action) + self._fired_actions_append(action) # set the ticker to -1 in order to force action_dispatcher() # to run at the next possible bytecode self.reset_ticker(-1) + def _fired_actions_reset(self): + # linked list of actions. We cannot use a normal RPython list because + # we want AsyncAction.fire() to be marked as @rgc.collect: this way, + # we can call it from e.g. GcHooks or cpyext's dealloc_trigger. + self._fired_actions_first = None + self._fired_actions_last = None + + @rgc.no_collect + def _fired_actions_append(self, action): + assert action._next is None + if self._fired_actions_first is None: + self._fired_actions_first = action + self._fired_actions_last = action + else: + self._fired_actions_last._next = action + self._fired_actions_last = action + @not_rpython def register_periodic_action(self, action, use_bytecode_counter): """ @@ -467,9 +482,9 @@ action.perform(ec, frame) # nonperiodic actions - list = self.fired_actions - if list is not None: - self.fired_actions = None + action = self._fired_actions_first + if action: + self._fired_actions_reset() # NB. in case there are several actions, we reset each # 'action._fired' to false only when we're about to call # 'action.perform()'. This means that if @@ -477,9 +492,10 @@ # the corresponding perform(), the fire() has no # effect---which is the effect we want, because # perform() will be called anyway. - for action in list: + while action is not None: action._fired = False action.perform(ec, frame) + action._next, action = None, action._next self.action_dispatcher = action_dispatcher @@ -512,10 +528,12 @@ to occur between two opcodes, not at a completely random time. """ _fired = False + _next = None def __init__(self, space): self.space = space + @rgc.no_collect def fire(self): """Request for the action to be run before the next opcode. The action must have been registered at space initalization time.""" diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py --- a/pypy/interpreter/test/test_executioncontext.py +++ b/pypy/interpreter/test/test_executioncontext.py @@ -37,6 +37,37 @@ pass assert i == 9 + def test_action_queue(self): + events = [] + + class Action1(executioncontext.AsyncAction): + def perform(self, ec, frame): + events.append('one') + + class Action2(executioncontext.AsyncAction): + def perform(self, ec, frame): + events.append('two') + + space = self.space + a1 = Action1(space) + a2 = Action2(space) + a1.fire() + a2.fire() + space.appexec([], """(): + n = 5 + return n + 2 + """) + assert events == ['one', 'two'] + # + events[:] = [] + a1.fire() + space.appexec([], """(): + n = 5 + return n + 2 + """) + assert events == ['one'] + + def test_periodic_action(self): from pypy.interpreter.executioncontext import ActionFlag diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -82,6 +82,8 @@ 'debug_stop' : 'interp_debug.debug_stop', 'debug_print_once' : 'interp_debug.debug_print_once', 'debug_flush' : 'interp_debug.debug_flush', + 'debug_read_timestamp' : 'interp_debug.debug_read_timestamp', + 'debug_get_timestamp_unit' : 'interp_debug.debug_get_timestamp_unit', 'builtinify' : 'interp_magic.builtinify', 'hidden_applevel' : 'interp_magic.hidden_applevel', 'get_hidden_tb' : 'interp_magic.get_hidden_tb', diff --git a/pypy/module/__pypy__/interp_debug.py b/pypy/module/__pypy__/interp_debug.py --- a/pypy/module/__pypy__/interp_debug.py +++ b/pypy/module/__pypy__/interp_debug.py @@ -1,6 +1,6 @@ from pypy.interpreter.gateway import unwrap_spec from rpython.rlib import debug, jit - +from rpython.rlib import rtimer @jit.dont_look_inside @unwrap_spec(category='text') @@ -28,3 +28,18 @@ @jit.dont_look_inside def debug_flush(space): debug.debug_flush() + +def debug_read_timestamp(space): + return space.newint(rtimer.read_timestamp()) + +def debug_get_timestamp_unit(space): + unit = rtimer.get_timestamp_unit() + if unit == rtimer.UNIT_TSC: + unit_str = 'tsc' + elif unit == rtimer.UNIT_NS: + unit_str = 'ns' + elif unit == rtimer.UNIT_QUERY_PERFORMANCE_COUNTER: + unit_str = 'QueryPerformanceCounter' + else: + unit_str = 'UNKNOWN(%d)' % unit + return space.newtext(unit_str) diff --git a/pypy/module/__pypy__/test/test_debug.py b/pypy/module/__pypy__/test/test_debug.py --- a/pypy/module/__pypy__/test/test_debug.py +++ b/pypy/module/__pypy__/test/test_debug.py @@ -48,3 +48,14 @@ from __pypy__ import debug_flush debug_flush() # assert did not crash + + def test_debug_read_timestamp(self): + from __pypy__ import debug_read_timestamp + a = debug_read_timestamp() + b = debug_read_timestamp() + assert b > a + + def test_debug_get_timestamp_unit(self): + from __pypy__ import debug_get_timestamp_unit + unit = debug_get_timestamp_unit() + assert unit in ('tsc', 'ns', 'QueryPerformanceCounter') diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py --- a/pypy/module/gc/__init__.py +++ b/pypy/module/gc/__init__.py @@ -34,5 +34,7 @@ 'get_typeids_z': 'referents.get_typeids_z', 'get_typeids_list': 'referents.get_typeids_list', 'GcRef': 'referents.W_GcRef', + 'hooks': 'space.fromcache(hook.W_AppLevelHooks)', + 'GcCollectStepStats': 'hook.W_GcCollectStepStats', }) MixedModule.__init__(self, space, w_name) diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py new file mode 100644 --- /dev/null +++ b/pypy/module/gc/hook.py @@ -0,0 +1,317 @@ +from rpython.memory.gc.hook import GcHooks +from rpython.memory.gc import incminimark +from rpython.rlib.nonconst import NonConstant +from rpython.rlib.rarithmetic import r_uint, r_longlong +from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault +from pypy.interpreter.baseobjspace import W_Root +from pypy.interpreter.typedef import TypeDef, interp_attrproperty, GetSetProperty +from pypy.interpreter.executioncontext import AsyncAction + +class LowLevelGcHooks(GcHooks): + """ + These are the low-level hooks which are called directly from the GC. + + They can't do much, because the base class marks the methods as + @rgc.no_collect. + + This is expected to be a singleton, created by space.fromcache, and it is + integrated with the translation by targetpypystandalone.get_gchooks + """ + + def __init__(self, space): + self.space = space + self.w_hooks = space.fromcache(W_AppLevelHooks) + + def is_gc_minor_enabled(self): + return self.w_hooks.gc_minor_enabled + + def is_gc_collect_step_enabled(self): + return self.w_hooks.gc_collect_step_enabled + + def is_gc_collect_enabled(self): + return self.w_hooks.gc_collect_enabled + + def on_gc_minor(self, duration, total_memory_used, pinned_objects): + action = self.w_hooks.gc_minor + action.count += 1 + action.duration += duration + action.total_memory_used = total_memory_used + action.pinned_objects = pinned_objects + action.fire() + + def on_gc_collect_step(self, duration, oldstate, newstate): + action = self.w_hooks.gc_collect_step + action.count += 1 + action.duration += duration + action.oldstate = oldstate + action.newstate = newstate + action.fire() + + def on_gc_collect(self, num_major_collects, + arenas_count_before, arenas_count_after, + arenas_bytes, rawmalloc_bytes_before, + rawmalloc_bytes_after): + action = self.w_hooks.gc_collect + action.count += 1 + action.num_major_collects = num_major_collects + action.arenas_count_before = arenas_count_before + action.arenas_count_after = arenas_count_after + action.arenas_bytes = arenas_bytes + action.rawmalloc_bytes_before = rawmalloc_bytes_before + action.rawmalloc_bytes_after = rawmalloc_bytes_after + action.fire() + + +class W_AppLevelHooks(W_Root): + + def __init__(self, space): + self.space = space + self.gc_minor_enabled = False + self.gc_collect_step_enabled = False + self.gc_collect_enabled = False + self.gc_minor = GcMinorHookAction(space) + self.gc_collect_step = GcCollectStepHookAction(space) + self.gc_collect = GcCollectHookAction(space) + + def descr_get_on_gc_minor(self, space): + return self.gc_minor.w_callable + + def descr_set_on_gc_minor(self, space, w_obj): + self.gc_minor_enabled = not space.is_none(w_obj) + self.gc_minor.w_callable = w_obj + self.gc_minor.fix_annotation() + + def descr_get_on_gc_collect_step(self, space): + return self.gc_collect_step.w_callable + + def descr_set_on_gc_collect_step(self, space, w_obj): + self.gc_collect_step_enabled = not space.is_none(w_obj) + self.gc_collect_step.w_callable = w_obj + self.gc_collect_step.fix_annotation() + + def descr_get_on_gc_collect(self, space): + return self.gc_collect.w_callable + + def descr_set_on_gc_collect(self, space, w_obj): + self.gc_collect_enabled = not space.is_none(w_obj) + self.gc_collect.w_callable = w_obj + self.gc_collect.fix_annotation() + + def descr_set(self, space, w_obj): + w_a = space.getattr(w_obj, space.newtext('on_gc_minor')) + w_b = space.getattr(w_obj, space.newtext('on_gc_collect_step')) + w_c = space.getattr(w_obj, space.newtext('on_gc_collect')) + self.descr_set_on_gc_minor(space, w_a) + self.descr_set_on_gc_collect_step(space, w_b) + self.descr_set_on_gc_collect(space, w_c) + + def descr_reset(self, space): + self.descr_set_on_gc_minor(space, space.w_None) + self.descr_set_on_gc_collect_step(space, space.w_None) + self.descr_set_on_gc_collect(space, space.w_None) + + +class GcMinorHookAction(AsyncAction): + count = 0 + duration = r_longlong(0) + total_memory_used = 0 + pinned_objects = 0 + + def __init__(self, space): + AsyncAction.__init__(self, space) + self.w_callable = space.w_None + + def reset(self): + self.count = 0 + self.duration = r_longlong(0) + + def fix_annotation(self): + # the annotation of the class and its attributes must be completed + # BEFORE we do the gc transform; this makes sure that everything is + # annotated with the correct types + if NonConstant(False): + self.count = NonConstant(-42) + self.duration = NonConstant(r_longlong(-42)) + self.total_memory_used = NonConstant(r_uint(42)) + self.pinned_objects = NonConstant(-42) + self.fire() + + def perform(self, ec, frame): + w_stats = W_GcMinorStats( + self.count, + self.duration, + self.total_memory_used, + self.pinned_objects) + self.reset() + self.space.call_function(self.w_callable, w_stats) + + +class GcCollectStepHookAction(AsyncAction): + count = 0 + duration = r_longlong(0) + oldstate = 0 + newstate = 0 + + def __init__(self, space): + AsyncAction.__init__(self, space) + self.w_callable = space.w_None + + def reset(self): + self.count = 0 + self.duration = r_longlong(0) + + def fix_annotation(self): + # the annotation of the class and its attributes must be completed + # BEFORE we do the gc transform; this makes sure that everything is + # annotated with the correct types + if NonConstant(False): + self.count = NonConstant(-42) + self.duration = NonConstant(r_longlong(-42)) + self.oldstate = NonConstant(-42) + self.newstate = NonConstant(-42) + self.fire() + + def perform(self, ec, frame): + w_stats = W_GcCollectStepStats( + self.count, + self.duration, + self.oldstate, + self.newstate) + self.reset() + self.space.call_function(self.w_callable, w_stats) + + +class GcCollectHookAction(AsyncAction): + count = 0 + num_major_collects = 0 + arenas_count_before = 0 + arenas_count_after = 0 + arenas_bytes = 0 + rawmalloc_bytes_before = 0 + rawmalloc_bytes_after = 0 + + def __init__(self, space): + AsyncAction.__init__(self, space) + self.w_callable = space.w_None + + def reset(self): + self.count = 0 + + def fix_annotation(self): + # the annotation of the class and its attributes must be completed + # BEFORE we do the gc transform; this makes sure that everything is + # annotated with the correct types + if NonConstant(False): + self.count = NonConstant(-42) + self.num_major_collects = NonConstant(-42) + self.arenas_count_before = NonConstant(-42) + self.arenas_count_after = NonConstant(-42) + self.arenas_bytes = NonConstant(r_uint(42)) + self.rawmalloc_bytes_before = NonConstant(r_uint(42)) + self.rawmalloc_bytes_after = NonConstant(r_uint(42)) + self.fire() + + def perform(self, ec, frame): + w_stats = W_GcCollectStats(self.count, + self.num_major_collects, + self.arenas_count_before, + self.arenas_count_after, + self.arenas_bytes, + self.rawmalloc_bytes_before, + self.rawmalloc_bytes_after) + self.reset() + self.space.call_function(self.w_callable, w_stats) + + +class W_GcMinorStats(W_Root): + + def __init__(self, count, duration, total_memory_used, pinned_objects): + self.count = count + self.duration = duration + self.total_memory_used = total_memory_used + self.pinned_objects = pinned_objects + + +class W_GcCollectStepStats(W_Root): + + def __init__(self, count, duration, oldstate, newstate): + self.count = count + self.duration = duration + self.oldstate = oldstate + self.newstate = newstate + + +class W_GcCollectStats(W_Root): + def __init__(self, count, num_major_collects, + arenas_count_before, arenas_count_after, + arenas_bytes, rawmalloc_bytes_before, + rawmalloc_bytes_after): + self.count = count + self.num_major_collects = num_major_collects + self.arenas_count_before = arenas_count_before + self.arenas_count_after = arenas_count_after + self.arenas_bytes = arenas_bytes + self.rawmalloc_bytes_before = rawmalloc_bytes_before + self.rawmalloc_bytes_after = rawmalloc_bytes_after + + +# just a shortcut to make the typedefs shorter +def wrap_many_ints(cls, names): + d = {} + for name in names: + d[name] = interp_attrproperty(name, cls=cls, wrapfn="newint") + return d + + +W_AppLevelHooks.typedef = TypeDef( + "GcHooks", + on_gc_minor = GetSetProperty( + W_AppLevelHooks.descr_get_on_gc_minor, + W_AppLevelHooks.descr_set_on_gc_minor), + + on_gc_collect_step = GetSetProperty( + W_AppLevelHooks.descr_get_on_gc_collect_step, + W_AppLevelHooks.descr_set_on_gc_collect_step), + + on_gc_collect = GetSetProperty( + W_AppLevelHooks.descr_get_on_gc_collect, + W_AppLevelHooks.descr_set_on_gc_collect), + + set = interp2app(W_AppLevelHooks.descr_set), + reset = interp2app(W_AppLevelHooks.descr_reset), + ) + +W_GcMinorStats.typedef = TypeDef( + "GcMinorStats", + **wrap_many_ints(W_GcMinorStats, ( + "count", + "duration", + "total_memory_used", + "pinned_objects")) + ) + +W_GcCollectStepStats.typedef = TypeDef( + "GcCollectStepStats", + STATE_SCANNING = incminimark.STATE_SCANNING, + STATE_MARKING = incminimark.STATE_MARKING, + STATE_SWEEPING = incminimark.STATE_SWEEPING, + STATE_FINALIZING = incminimark.STATE_FINALIZING, + GC_STATES = tuple(incminimark.GC_STATES), + **wrap_many_ints(W_GcCollectStepStats, ( + "count", + "duration", + "oldstate", + "newstate")) + ) + +W_GcCollectStats.typedef = TypeDef( + "GcCollectStats", + **wrap_many_ints(W_GcCollectStats, ( + "count", + "num_major_collects", + "arenas_count_before", + "arenas_count_after", + "arenas_bytes", + "rawmalloc_bytes_before", + "rawmalloc_bytes_after")) + ) diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py new file mode 100644 --- /dev/null +++ b/pypy/module/gc/test/test_hook.py @@ -0,0 +1,176 @@ +import pytest +from rpython.rlib.rarithmetic import r_uint +from pypy.module.gc.hook import LowLevelGcHooks +from pypy.interpreter.baseobjspace import ObjSpace +from pypy.interpreter.gateway import interp2app, unwrap_spec + +class AppTestGcHooks(object): + + def setup_class(cls): + if cls.runappdirect: + pytest.skip("these tests cannot work with -A") + space = cls.space + gchooks = space.fromcache(LowLevelGcHooks) + + @unwrap_spec(ObjSpace, int, r_uint, int) + def fire_gc_minor(space, duration, total_memory_used, pinned_objects): + gchooks.fire_gc_minor(duration, total_memory_used, pinned_objects) + + @unwrap_spec(ObjSpace, int, int, int) + def fire_gc_collect_step(space, duration, oldstate, newstate): + gchooks.fire_gc_collect_step(duration, oldstate, newstate) + + @unwrap_spec(ObjSpace, int, int, int, r_uint, r_uint, r_uint) + def fire_gc_collect(space, a, b, c, d, e, f): + gchooks.fire_gc_collect(a, b, c, d, e, f) + + @unwrap_spec(ObjSpace) + def fire_many(space): + gchooks.fire_gc_minor(5, 0, 0) + gchooks.fire_gc_minor(7, 0, 0) + gchooks.fire_gc_collect_step(5, 0, 0) + gchooks.fire_gc_collect_step(15, 0, 0) + gchooks.fire_gc_collect_step(22, 0, 0) + gchooks.fire_gc_collect(1, 2, 3, 4, 5, 6) + + cls.w_fire_gc_minor = space.wrap(interp2app(fire_gc_minor)) + cls.w_fire_gc_collect_step = space.wrap(interp2app(fire_gc_collect_step)) + cls.w_fire_gc_collect = space.wrap(interp2app(fire_gc_collect)) + cls.w_fire_many = space.wrap(interp2app(fire_many)) + + def test_default(self): + import gc + assert gc.hooks.on_gc_minor is None + assert gc.hooks.on_gc_collect_step is None + assert gc.hooks.on_gc_collect is None + + def test_on_gc_minor(self): + import gc + lst = [] + def on_gc_minor(stats): + lst.append((stats.count, + stats.duration, + stats.total_memory_used, + stats.pinned_objects)) + gc.hooks.on_gc_minor = on_gc_minor + self.fire_gc_minor(10, 20, 30) + self.fire_gc_minor(40, 50, 60) + assert lst == [ + (1, 10, 20, 30), + (1, 40, 50, 60), + ] + # + gc.hooks.on_gc_minor = None + self.fire_gc_minor(70, 80, 90) # won't fire because the hooks is disabled + assert lst == [ + (1, 10, 20, 30), + (1, 40, 50, 60), + ] + + def test_on_gc_collect_step(self): + import gc + lst = [] + def on_gc_collect_step(stats): + lst.append((stats.count, + stats.duration, + stats.oldstate, + stats.newstate)) + gc.hooks.on_gc_collect_step = on_gc_collect_step + self.fire_gc_collect_step(10, 20, 30) + self.fire_gc_collect_step(40, 50, 60) + assert lst == [ + (1, 10, 20, 30), + (1, 40, 50, 60), + ] + # + gc.hooks.on_gc_collect_step = None + self.fire_gc_collect_step(70, 80, 90) # won't fire + assert lst == [ + (1, 10, 20, 30), + (1, 40, 50, 60), + ] + + def test_on_gc_collect(self): + import gc + lst = [] + def on_gc_collect(stats): + lst.append((stats.count, + stats.num_major_collects, + stats.arenas_count_before, + stats.arenas_count_after, + stats.arenas_bytes, + stats.rawmalloc_bytes_before, + stats.rawmalloc_bytes_after)) + gc.hooks.on_gc_collect = on_gc_collect + self.fire_gc_collect(1, 2, 3, 4, 5, 6) + self.fire_gc_collect(7, 8, 9, 10, 11, 12) + assert lst == [ + (1, 1, 2, 3, 4, 5, 6), + (1, 7, 8, 9, 10, 11, 12), + ] + # + gc.hooks.on_gc_collect = None + self.fire_gc_collect(42, 42, 42, 42, 42, 42) # won't fire + assert lst == [ + (1, 1, 2, 3, 4, 5, 6), + (1, 7, 8, 9, 10, 11, 12), + ] + + def test_consts(self): + import gc + S = gc.GcCollectStepStats + assert S.STATE_SCANNING == 0 + assert S.STATE_MARKING == 1 + assert S.STATE_SWEEPING == 2 + assert S.STATE_FINALIZING == 3 + assert S.GC_STATES == ('SCANNING', 'MARKING', 'SWEEPING', 'FINALIZING') + + def test_cumulative(self): + import gc + class MyHooks(object): + + def __init__(self): + self.minors = [] + self.steps = [] + + def on_gc_minor(self, stats): + self.minors.append((stats.count, stats.duration)) + + def on_gc_collect_step(self, stats): + self.steps.append((stats.count, stats.duration)) + + on_gc_collect = None + + myhooks = MyHooks() + gc.hooks.set(myhooks) + self.fire_many() + assert myhooks.minors == [(2, 12)] + assert myhooks.steps == [(3, 42)] + + def test_clear_queue(self): + import gc + class MyHooks(object): + + def __init__(self): + self.lst = [] + + def on_gc_minor(self, stats): + self.lst.append('minor') + + def on_gc_collect_step(self, stats): + self.lst.append('step') + + def on_gc_collect(self, stats): + self.lst.append('collect') + + myhooks = MyHooks() + gc.hooks.set(myhooks) + self.fire_many() + assert myhooks.lst == ['minor', 'step', 'collect'] + myhooks.lst[:] = [] + self.fire_gc_minor(0, 0, 0) + assert myhooks.lst == ['minor'] + gc.hooks.reset() + assert gc.hooks.on_gc_minor is None + assert gc.hooks.on_gc_collect_step is None + assert gc.hooks.on_gc_collect is None diff --git a/pypy/module/gc/test/test_ztranslation.py b/pypy/module/gc/test/test_ztranslation.py --- a/pypy/module/gc/test/test_ztranslation.py +++ b/pypy/module/gc/test/test_ztranslation.py @@ -1,4 +1,9 @@ from pypy.objspace.fake.checkmodule import checkmodule def test_checkmodule(): - checkmodule('gc') + # we need to ignore GcCollectStepStats, else checkmodule fails. I think + # this happens because W_GcCollectStepStats.__init__ is only called from + # GcCollectStepHookAction.perform() and the fake objspace doesn't know + # about those: so, perform() is never annotated and the annotator thinks + # W_GcCollectStepStats has no attributes + checkmodule('gc', ignore=['GcCollectStepStats']) diff --git a/pypy/objspace/fake/checkmodule.py b/pypy/objspace/fake/checkmodule.py --- a/pypy/objspace/fake/checkmodule.py +++ b/pypy/objspace/fake/checkmodule.py @@ -4,6 +4,7 @@ def checkmodule(*modnames, **kwds): translate_startup = kwds.pop('translate_startup', True) + ignore = set(kwds.pop('ignore', ())) assert not kwds config = get_pypy_config(translating=True) space = FakeObjSpace(config) @@ -17,6 +18,8 @@ module.init(space) modules.append(module) for name in module.loaders: + if name in ignore: + continue seeobj_w.append(module._load_lazily(space, name)) if hasattr(module, 'submodules'): for cls in module.submodules.itervalues(): diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -2164,6 +2164,11 @@ oopspecindex=EffectInfo.OS_MATH_READ_TIMESTAMP, extraeffect=EffectInfo.EF_CANNOT_RAISE) + def rewrite_op_ll_get_timestamp_unit(self, op): + op1 = self.prepare_builtin_call(op, "ll_get_timestamp_unit", []) + return self.handle_residual_call(op1, + extraeffect=EffectInfo.EF_CANNOT_RAISE) + def rewrite_op_jit_force_quasi_immutable(self, op): v_inst, c_fieldname = op.args descr1 = self.cpu.fielddescrof(v_inst.concretetype.TO, diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py --- a/rpython/jit/codewriter/support.py +++ b/rpython/jit/codewriter/support.py @@ -285,6 +285,9 @@ from rpython.rlib import rtimer return rtimer.read_timestamp() +def _ll_0_ll_get_timestamp_unit(): + from rpython.rlib import rtimer + return rtimer.get_timestamp_unit() # math support # ------------ diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py --- a/rpython/jit/metainterp/test/test_ajit.py +++ b/rpython/jit/metainterp/test/test_ajit.py @@ -2577,6 +2577,14 @@ res = self.interp_operations(f, []) assert res + def test_get_timestamp_unit(self): + import time + from rpython.rlib import rtimer + def f(): + return rtimer.get_timestamp_unit() + unit = self.interp_operations(f, []) + assert unit == rtimer.UNIT_NS + def test_bug688_multiple_immutable_fields(self): myjitdriver = JitDriver(greens=[], reds=['counter','context']) diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py --- a/rpython/memory/gc/base.py +++ b/rpython/memory/gc/base.py @@ -5,6 +5,7 @@ from rpython.memory.support import DEFAULT_CHUNK_SIZE from rpython.memory.support import get_address_stack, get_address_deque from rpython.memory.support import AddressDict, null_address_dict +from rpython.memory.gc.hook import GcHooks from rpython.rtyper.lltypesystem.llmemory import NULL, raw_malloc_usage from rpython.rtyper.annlowlevel import cast_adr_to_nongc_instance @@ -25,7 +26,7 @@ _totalroots_rpy = 0 # for inspector.py def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE, - translated_to_c=True): + translated_to_c=True, hooks=None): self.gcheaderbuilder = GCHeaderBuilder(self.HDR) self.AddressStack = get_address_stack(chunk_size) self.AddressDeque = get_address_deque(chunk_size) @@ -34,6 +35,9 @@ self.config = config assert isinstance(translated_to_c, bool) self.translated_to_c = translated_to_c + if hooks is None: + hooks = GcHooks() # the default hooks are empty + self.hooks = hooks def setup(self): # all runtime mutable values' setup should happen here diff --git a/rpython/memory/gc/hook.py b/rpython/memory/gc/hook.py new file mode 100644 --- /dev/null +++ b/rpython/memory/gc/hook.py @@ -0,0 +1,70 @@ +from rpython.rlib import rgc + +# WARNING: at the moment of writing, gc hooks are implemented only for +# incminimark. Please add calls to hooks to the other GCs if you need it. +class GcHooks(object): + """ + Base class to write your own GC hooks. + + Subclasses are expected to override the on_* methods. Note that such + methods can do only simple stuff such as updating statistics and/or + setting a flag: in particular, they cannot do anything which can possibly + trigger a GC collection. + """ + + def is_gc_minor_enabled(self): + return False + + def is_gc_collect_step_enabled(self): + return False + + def is_gc_collect_enabled(self): + return False + + def on_gc_minor(self, duration, total_memory_used, pinned_objects): + """ + Called after a minor collection + """ + + def on_gc_collect_step(self, duration, oldstate, newstate): + """ + Called after each individual step of a major collection, in case the GC is + incremental. + + ``oldstate`` and ``newstate`` are integers which indicate the GC + state; for incminimark, see incminimark.STATE_* and + incminimark.GC_STATES. + """ + + + def on_gc_collect(self, num_major_collects, + arenas_count_before, arenas_count_after, + arenas_bytes, rawmalloc_bytes_before, + rawmalloc_bytes_after): + """ + Called after a major collection is fully done + """ + + # the fire_* methods are meant to be called from the GC are should NOT be + # overridden + + @rgc.no_collect + def fire_gc_minor(self, duration, total_memory_used, pinned_objects): + if self.is_gc_minor_enabled(): + self.on_gc_minor(duration, total_memory_used, pinned_objects) + + @rgc.no_collect + def fire_gc_collect_step(self, duration, oldstate, newstate): + if self.is_gc_collect_step_enabled(): + self.on_gc_collect_step(duration, oldstate, newstate) + + @rgc.no_collect + def fire_gc_collect(self, num_major_collects, + arenas_count_before, arenas_count_after, + arenas_bytes, rawmalloc_bytes_before, + rawmalloc_bytes_after): + if self.is_gc_collect_enabled(): + self.on_gc_collect(num_major_collects, + arenas_count_before, arenas_count_after, + arenas_bytes, rawmalloc_bytes_before, + rawmalloc_bytes_after) diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -73,6 +73,7 @@ from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop from rpython.rlib.objectmodel import specialize from rpython.rlib import rgc +from rpython.rlib.rtimer import read_timestamp from rpython.memory.gc.minimarkpage import out_of_memory # @@ -1643,6 +1644,7 @@ """Perform a minor collection: find the objects from the nursery that remain alive and move them out.""" # + start = read_timestamp() debug_start("gc-minor") # # All nursery barriers are invalid from this point on. They @@ -1830,8 +1832,8 @@ # from the nursery that we just moved out. self.size_objects_made_old += r_uint(self.nursery_surviving_size) # - debug_print("minor collect, total memory used:", - self.get_total_memory_used()) + total_memory_used = self.get_total_memory_used() + debug_print("minor collect, total memory used:", total_memory_used) debug_print("number of pinned objects:", self.pinned_objects_in_nursery) if self.DEBUG >= 2: @@ -1840,6 +1842,11 @@ self.root_walker.finished_minor_collection() # debug_stop("gc-minor") + duration = read_timestamp() - start + self.hooks.fire_gc_minor( + duration=duration, + total_memory_used=total_memory_used, + pinned_objects=self.pinned_objects_in_nursery) def _reset_flag_old_objects_pointing_to_pinned(self, obj, ignore): ll_assert(self.header(obj).tid & GCFLAG_PINNED_OBJECT_PARENT_KNOWN != 0, @@ -2241,7 +2248,9 @@ # Note - minor collections seem fast enough so that one # is done before every major collection step def major_collection_step(self, reserving_size=0): + start = read_timestamp() debug_start("gc-collect-step") + oldstate = self.gc_state debug_print("starting gc state: ", GC_STATES[self.gc_state]) # Debugging checks if self.pinned_objects_in_nursery == 0: @@ -2421,6 +2430,13 @@ self.stat_rawmalloced_total_size, " => ", self.rawmalloced_total_size) debug_stop("gc-collect-done") + self.hooks.fire_gc_collect( + num_major_collects=self.num_major_collects, + arenas_count_before=self.stat_ac_arenas_count, + arenas_count_after=self.ac.arenas_count, + arenas_bytes=self.ac.total_memory_used, + rawmalloc_bytes_before=self.stat_rawmalloced_total_size, + rawmalloc_bytes_after=self.rawmalloced_total_size) # # Set the threshold for the next major collection to be when we # have allocated 'major_collection_threshold' times more than @@ -2472,6 +2488,11 @@ debug_print("stopping, now in gc state: ", GC_STATES[self.gc_state]) debug_stop("gc-collect-step") + duration = read_timestamp() - start + self.hooks.fire_gc_collect_step( + duration=duration, + oldstate=oldstate, + newstate=self.gc_state) def _sweep_old_objects_pointing_to_pinned(self, obj, new_list): if self.header(obj).tid & GCFLAG_VISITED: diff --git a/rpython/memory/gc/test/test_direct.py b/rpython/memory/gc/test/test_direct.py --- a/rpython/memory/gc/test/test_direct.py +++ b/rpython/memory/gc/test/test_direct.py @@ -70,6 +70,9 @@ class BaseDirectGCTest(object): GC_PARAMS = {} + def get_extra_gc_params(self): + return {} + def setup_method(self, meth): from rpython.config.translationoption import get_combined_translation_config config = get_combined_translation_config(translating=True).translation @@ -78,6 +81,7 @@ if hasattr(meth, 'GC_PARAMS'): GC_PARAMS.update(meth.GC_PARAMS) GC_PARAMS['translated_to_c'] = False + GC_PARAMS.update(self.get_extra_gc_params()) self.gc = self.GCClass(config, **GC_PARAMS) self.gc.DEBUG = True self.rootwalker = DirectRootWalker(self) diff --git a/rpython/memory/gc/test/test_hook.py b/rpython/memory/gc/test/test_hook.py new file mode 100644 --- /dev/null +++ b/rpython/memory/gc/test/test_hook.py @@ -0,0 +1,125 @@ +from rpython.rtyper.lltypesystem import lltype, llmemory +from rpython.memory.gc.hook import GcHooks +from rpython.memory.gc.test.test_direct import BaseDirectGCTest, S + + +class MyGcHooks(GcHooks): + + def __init__(self): + GcHooks.__init__(self) + self._gc_minor_enabled = False + self._gc_collect_step_enabled = False + self._gc_collect_enabled = False + self.reset() + + def is_gc_minor_enabled(self): + return self._gc_minor_enabled + + def is_gc_collect_step_enabled(self): + return self._gc_collect_step_enabled + + def is_gc_collect_enabled(self): + return self._gc_collect_enabled + + def reset(self): + self.minors = [] + self.steps = [] + self.collects = [] + self.durations = [] + + def on_gc_minor(self, duration, total_memory_used, pinned_objects): + self.durations.append(duration) + self.minors.append({ + 'total_memory_used': total_memory_used, + 'pinned_objects': pinned_objects}) + + def on_gc_collect_step(self, duration, oldstate, newstate): + self.durations.append(duration) + self.steps.append({ + 'oldstate': oldstate, + 'newstate': newstate}) + + def on_gc_collect(self, num_major_collects, + arenas_count_before, arenas_count_after, + arenas_bytes, rawmalloc_bytes_before, + rawmalloc_bytes_after): + self.collects.append({ + 'num_major_collects': num_major_collects, + 'arenas_count_before': arenas_count_before, + 'arenas_count_after': arenas_count_after, + 'arenas_bytes': arenas_bytes, + 'rawmalloc_bytes_before': rawmalloc_bytes_before, + 'rawmalloc_bytes_after': rawmalloc_bytes_after}) + + +class TestIncMiniMarkHooks(BaseDirectGCTest): + from rpython.memory.gc.incminimark import IncrementalMiniMarkGC as GCClass + + def get_extra_gc_params(self): + return {'hooks': MyGcHooks()} + + def setup_method(self, m): + BaseDirectGCTest.setup_method(self, m) + size = llmemory.sizeof(S) + self.gc.gcheaderbuilder.size_gc_header + self.size_of_S = llmemory.raw_malloc_usage(size) + + def test_on_gc_minor(self): + self.gc.hooks._gc_minor_enabled = True + self.malloc(S) + self.gc._minor_collection() + assert self.gc.hooks.minors == [ + {'total_memory_used': 0, 'pinned_objects': 0} + ] + assert self.gc.hooks.durations[0] > 0 + self.gc.hooks.reset() + # + # these objects survive, so the total_memory_used is > 0 + self.stackroots.append(self.malloc(S)) + self.stackroots.append(self.malloc(S)) + self.gc._minor_collection() + assert self.gc.hooks.minors == [ + {'total_memory_used': self.size_of_S*2, 'pinned_objects': 0} + ] + + def test_on_gc_collect(self): + from rpython.memory.gc import incminimark as m + self.gc.hooks._gc_collect_step_enabled = True + self.gc.hooks._gc_collect_enabled = True + self.malloc(S) + self.gc.collect() + assert self.gc.hooks.steps == [ + {'oldstate': m.STATE_SCANNING, 'newstate': m.STATE_MARKING}, + {'oldstate': m.STATE_MARKING, 'newstate': m.STATE_SWEEPING}, + {'oldstate': m.STATE_SWEEPING, 'newstate': m.STATE_FINALIZING}, + {'oldstate': m.STATE_FINALIZING, 'newstate': m.STATE_SCANNING} + ] + assert self.gc.hooks.collects == [ + {'num_major_collects': 1, + 'arenas_count_before': 0, + 'arenas_count_after': 0, + 'arenas_bytes': 0, + 'rawmalloc_bytes_after': 0, + 'rawmalloc_bytes_before': 0} + ] + assert len(self.gc.hooks.durations) == 4 # 4 steps + for d in self.gc.hooks.durations: + assert d > 0 + self.gc.hooks.reset() + # + self.stackroots.append(self.malloc(S)) + self.gc.collect() + assert self.gc.hooks.collects == [ + {'num_major_collects': 2, + 'arenas_count_before': 1, + 'arenas_count_after': 1, + 'arenas_bytes': self.size_of_S, + 'rawmalloc_bytes_after': 0, + 'rawmalloc_bytes_before': 0} + ] + + def test_hook_disabled(self): + self.gc._minor_collection() + self.gc.collect() + assert self.gc.hooks.minors == [] + assert self.gc.hooks.steps == [] + assert self.gc.hooks.collects == [] diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -116,7 +116,7 @@ class BaseFrameworkGCTransformer(GCTransformer): root_stack_depth = None # for tests to override - def __init__(self, translator): + def __init__(self, translator, gchooks=None): from rpython.memory.gc.base import choose_gc_from_config super(BaseFrameworkGCTransformer, self).__init__(translator, @@ -162,7 +162,8 @@ self.finalizer_queue_indexes = {} self.finalizer_handlers = [] - gcdata.gc = GCClass(translator.config.translation, **GC_PARAMS) + gcdata.gc = GCClass(translator.config.translation, hooks=gchooks, + **GC_PARAMS) root_walker = self.build_root_walker() root_walker.finished_minor_collection_func = finished_minor_collection self.root_walker = root_walker diff --git a/rpython/memory/test/test_transformed_gc.py b/rpython/memory/test/test_transformed_gc.py --- a/rpython/memory/test/test_transformed_gc.py +++ b/rpython/memory/test/test_transformed_gc.py @@ -14,7 +14,9 @@ from rpython.conftest import option from rpython.rlib.rstring import StringBuilder from rpython.rlib.rarithmetic import LONG_BIT +from rpython.rlib.nonconst import NonConstant from rpython.rtyper.rtyper import llinterp_backend +from rpython.memory.gc.hook import GcHooks WORD = LONG_BIT // 8 @@ -48,6 +50,7 @@ gcpolicy = None GC_CAN_MOVE = False taggedpointers = False + gchooks = None def setup_class(cls): cls.marker = lltype.malloc(rffi.CArray(lltype.Signed), 1, @@ -112,7 +115,8 @@ fixup(t) cbuild = CStandaloneBuilder(t, entrypoint, config=t.config, - gcpolicy=cls.gcpolicy) + gcpolicy=cls.gcpolicy, + gchooks=cls.gchooks) cbuild.make_entrypoint_wrapper = False db = cbuild.build_database() entrypointptr = cbuild.getentrypointptr() @@ -1388,6 +1392,48 @@ assert res([]) == 0 +class GcHooksStats(object): + minors = 0 + steps = 0 + collects = 0 + + def reset(self): + # the NonConstant are needed so that the annotator annotates the + # fields as a generic SomeInteger(), instead of a constant 0. A call + # to this method MUST be seen during normal annotation, else the class + # is annotated only during GC transform, when it's too late + self.minors = NonConstant(0) + self.steps = NonConstant(0) + self.collects = NonConstant(0) + + +class MyGcHooks(GcHooks): + + def __init__(self, stats=None): + self.stats = stats or GcHooksStats() + + def is_gc_minor_enabled(self): + return True + + def is_gc_collect_step_enabled(self): + return True + + def is_gc_collect_enabled(self): + return True + + def on_gc_minor(self, duration, total_memory_used, pinned_objects): + self.stats.minors += 1 + + def on_gc_collect_step(self, duration, oldstate, newstate): + self.stats.steps += 1 + + def on_gc_collect(self, num_major_collects, + arenas_count_before, arenas_count_after, + arenas_bytes, rawmalloc_bytes_before, + rawmalloc_bytes_after): + self.stats.collects += 1 + + class TestIncrementalMiniMarkGC(TestMiniMarkGC): gcname = "incminimark" @@ -1405,6 +1451,8 @@ } root_stack_depth = 200 + gchooks = MyGcHooks() + def define_malloc_array_of_gcptr(self): S = lltype.GcStruct('S', ('x', lltype.Signed)) A = lltype.GcArray(lltype.Ptr(S)) @@ -1438,6 +1486,34 @@ res = run([]) assert res + def define_gc_hooks(cls): + gchooks = cls.gchooks + # it is important that we fish .stats OUTSIDE f(); we cannot see + # gchooks from within RPython code + stats = gchooks.stats + def f(): + stats.reset() + # trigger two major collections + llop.gc__collect(lltype.Void) + llop.gc__collect(lltype.Void) + return (10000 * stats.collects + + 100 * stats.steps + + 1 * stats.minors) + return f + + def test_gc_hooks(self): + run = self.runner("gc_hooks") + count = run([]) + collects, count = divmod(count, 10000) + steps, minors = divmod(count, 100) + # + # note: the following asserts are slightly fragile, as they assume + # that we do NOT run any minor collection apart the ones triggered by + # major_collection_step + assert collects == 2 # 2 collections, manually triggered + assert steps == 4 * collects # 4 steps for each major collection + assert minors == steps # one minor collection for each step + # ________________________________________________________________ # tagged pointers diff --git a/rpython/rlib/rtimer.py b/rpython/rlib/rtimer.py --- a/rpython/rlib/rtimer.py +++ b/rpython/rlib/rtimer.py @@ -7,6 +7,11 @@ _is_64_bit = r_uint.BITS > 32 +# unit of values returned by read_timestamp. Should be in sync with the ones +# defined in translator/c/debug_print.h +UNIT_TSC = 0 +UNIT_NS = 1 # nanoseconds +UNIT_QUERY_PERFORMANCE_COUNTER = 2 def read_timestamp(): # Returns a longlong on 32-bit, and a regular int on 64-bit. @@ -17,6 +22,11 @@ else: return longlongmask(x) +def get_timestamp_unit(): + # an unit which is as arbitrary as the way we build the result of + # read_timestamp :) + return UNIT_NS + class ReadTimestampEntry(ExtRegistryEntry): _about_ = read_timestamp @@ -35,3 +45,15 @@ else: resulttype = rffi.LONGLONG return hop.genop("ll_read_timestamp", [], resulttype=resulttype) + + +class ReadTimestampEntry(ExtRegistryEntry): + _about_ = get_timestamp_unit + + def compute_result_annotation(self): + from rpython.annotator.model import SomeInteger + return SomeInteger(nonneg=True) + + def specialize_call(self, hop): + hop.exception_cannot_occur() + return hop.genop("ll_get_timestamp_unit", [], resulttype=lltype.Signed) diff --git a/rpython/rlib/test/test_rtimer.py b/rpython/rlib/test/test_rtimer.py --- a/rpython/rlib/test/test_rtimer.py +++ b/rpython/rlib/test/test_rtimer.py @@ -1,28 +1,56 @@ import time - -from rpython.rlib.rtimer import read_timestamp +import platform +from rpython.rlib import rtimer from rpython.rtyper.test.test_llinterp import interpret from rpython.translator.c.test.test_genc import compile -def timer(): - t1 = read_timestamp() - start = time.time() - while time.time() - start < 0.1: - # busy wait - pass - t2 = read_timestamp() - return t2 - t1 +class TestTimer(object): -def test_timer(): - diff = timer() - # We're counting ticks, verify they look correct - assert diff > 1000 + @staticmethod + def timer(): + t1 = rtimer.read_timestamp() + start = time.time() + while time.time() - start < 0.1: + # busy wait + pass + t2 = rtimer.read_timestamp() + return t2 - t1 -def test_annotation(): - diff = interpret(timer, []) - assert diff > 1000 + def test_direct(self): + diff = self.timer() + # We're counting ticks, verify they look correct + assert diff > 1000 -def test_compile_c(): - function = compile(timer, []) - diff = function() - assert diff > 1000 \ No newline at end of file + def test_annotation(self): + diff = interpret(self.timer, []) + assert diff > 1000 + + def test_compile_c(self): + function = compile(self.timer, []) + diff = function() + assert diff > 1000 + + +class TestGetUnit(object): + + @staticmethod + def get_unit(): + return rtimer.get_timestamp_unit() + + def test_direct(self): + unit = self.get_unit() + assert unit == rtimer.UNIT_NS + + def test_annotation(self): + unit = interpret(self.get_unit, []) + assert unit == rtimer.UNIT_NS + + def test_compile_c(self): + function = compile(self.get_unit, []) + unit = function() + if platform.processor() in ('x86', 'x86_64'): + assert unit == rtimer.UNIT_TSC + else: + assert unit in (rtimer.UNIT_TSC, + rtimer.UNIT_NS, + rtimer.UNIT_QUERY_PERFORMANCE_COUNTER) diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -445,6 +445,7 @@ 'get_write_barrier_from_array_failing_case': LLOp(sideeffects=False), 'gc_get_type_info_group': LLOp(sideeffects=False), 'll_read_timestamp': LLOp(canrun=True), + 'll_get_timestamp_unit': LLOp(canrun=True), # __________ GC operations __________ diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py --- a/rpython/rtyper/lltypesystem/opimpl.py +++ b/rpython/rtyper/lltypesystem/opimpl.py @@ -696,6 +696,10 @@ from rpython.rlib.rtimer import read_timestamp return read_timestamp() +def op_ll_get_timestamp_unit(): + from rpython.rlib.rtimer import get_timestamp_unit + return get_timestamp_unit() + def op_debug_fatalerror(ll_msg): from rpython.rtyper.lltypesystem import lltype, rstr from rpython.rtyper.llinterp import LLFatalError diff --git a/rpython/translator/c/database.py b/rpython/translator/c/database.py --- a/rpython/translator/c/database.py +++ b/rpython/translator/c/database.py @@ -29,6 +29,7 @@ def __init__(self, translator=None, standalone=False, gcpolicyclass=None, + gchooks=None, exctransformer=None, thread_enabled=False, sandbox=False): @@ -56,7 +57,7 @@ self.namespace = CNameManager() if translator is not None: - self.gctransformer = self.gcpolicy.gettransformer(translator) + self.gctransformer = self.gcpolicy.gettransformer(translator, gchooks) self.completed = False self.instrument_ncounter = 0 diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py --- a/rpython/translator/c/gc.py +++ b/rpython/translator/c/gc.py @@ -94,7 +94,7 @@ class RefcountingGcPolicy(BasicGcPolicy): - def gettransformer(self, translator): + def gettransformer(self, translator, gchooks): from rpython.memory.gctransform import refcounting return refcounting.RefcountingGCTransformer(translator) @@ -175,7 +175,7 @@ class BoehmGcPolicy(BasicGcPolicy): - def gettransformer(self, translator): + def gettransformer(self, translator, gchooks): from rpython.memory.gctransform import boehm return boehm.BoehmGCTransformer(translator) @@ -302,9 +302,9 @@ class BasicFrameworkGcPolicy(BasicGcPolicy): - def gettransformer(self, translator): + def gettransformer(self, translator, gchooks): if hasattr(self, 'transformerclass'): # for rpython/memory tests - return self.transformerclass(translator) + return self.transformerclass(translator, gchooks=gchooks) raise NotImplementedError def struct_setup(self, structdefnode, rtti): @@ -439,9 +439,9 @@ class ShadowStackFrameworkGcPolicy(BasicFrameworkGcPolicy): - def gettransformer(self, translator): + def gettransformer(self, translator, gchooks): from rpython.memory.gctransform import shadowstack - return shadowstack.ShadowStackFrameworkGCTransformer(translator) + return shadowstack.ShadowStackFrameworkGCTransformer(translator, gchooks) def enter_roots_frame(self, funcgen, (c_gcdata, c_numcolors)): numcolors = c_numcolors.value @@ -484,9 +484,9 @@ class AsmGcRootFrameworkGcPolicy(BasicFrameworkGcPolicy): - def gettransformer(self, translator): + def gettransformer(self, translator, gchooks): from rpython.memory.gctransform import asmgcroot - return asmgcroot.AsmGcRootFrameworkGCTransformer(translator) + return asmgcroot.AsmGcRootFrameworkGCTransformer(translator, gchooks) def GC_KEEPALIVE(self, funcgen, v): return 'pypy_asm_keepalive(%s);' % funcgen.expr(v) diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py --- a/rpython/translator/c/genc.py +++ b/rpython/translator/c/genc.py @@ -64,13 +64,14 @@ split = False def __init__(self, translator, entrypoint, config, gcpolicy=None, - secondary_entrypoints=()): + gchooks=None, secondary_entrypoints=()): self.translator = translator self.entrypoint = entrypoint self.entrypoint_name = getattr(self.entrypoint, 'func_name', None) self.originalentrypoint = entrypoint self.config = config self.gcpolicy = gcpolicy # for tests only, e.g. rpython/memory/ + self.gchooks = gchooks self.eci = self.get_eci() self.secondary_entrypoints = secondary_entrypoints @@ -91,6 +92,7 @@ exctransformer = translator.getexceptiontransformer() db = LowLevelDatabase(translator, standalone=self.standalone, gcpolicyclass=gcpolicyclass, + gchooks=self.gchooks, exctransformer=exctransformer, thread_enabled=self.config.translation.thread, sandbox=self.config.translation.sandbox) diff --git a/rpython/translator/c/src/asm_gcc_x86.h b/rpython/translator/c/src/asm_gcc_x86.h --- a/rpython/translator/c/src/asm_gcc_x86.h +++ b/rpython/translator/c/src/asm_gcc_x86.h @@ -70,6 +70,7 @@ // lfence // I don't know how important it is, comment talks about time warps +#define READ_TIMESTAMP_UNIT TIMESTAMP_UNIT_TSC #ifndef PYPY_CPU_HAS_STANDARD_PRECISION /* On x86-32, we have to use the following hacks to set and restore diff --git a/rpython/translator/c/src/asm_gcc_x86_64.h b/rpython/translator/c/src/asm_gcc_x86_64.h --- a/rpython/translator/c/src/asm_gcc_x86_64.h +++ b/rpython/translator/c/src/asm_gcc_x86_64.h @@ -7,5 +7,6 @@ val = (_rdx << 32) | _rax; \ } while (0) +#define READ_TIMESTAMP_UNIT TIMESTAMP_UNIT_TSC #define RPy_YieldProcessor() asm("pause") diff --git a/rpython/translator/c/src/asm_msvc.h b/rpython/translator/c/src/asm_msvc.h --- a/rpython/translator/c/src/asm_msvc.h +++ b/rpython/translator/c/src/asm_msvc.h @@ -13,3 +13,4 @@ #include #pragma intrinsic(__rdtsc) #define READ_TIMESTAMP(val) do { val = (long long)__rdtsc(); } while (0) +#define READ_TIMESTAMP_UNIT TIMESTAMP_UNIT_TSC diff --git a/rpython/translator/c/src/debug_print.h b/rpython/translator/c/src/debug_print.h --- a/rpython/translator/c/src/debug_print.h +++ b/rpython/translator/c/src/debug_print.h @@ -51,6 +51,11 @@ RPY_EXTERN long pypy_have_debug_prints; RPY_EXPORTED FILE *pypy_debug_file; +/* these should be in sync with the values defined in rlib/rtimer.py */ +#define TIMESTAMP_UNIT_TSC 0 +#define TIMESTAMP_UNIT_NS 1 +#define TIMESTAMP_UNIT_QUERY_PERFORMANCE_COUNTER 2 + #define OP_LL_READ_TIMESTAMP(val) READ_TIMESTAMP(val) #include "src/asm.h" @@ -62,11 +67,15 @@ # ifdef _WIN32 # define READ_TIMESTAMP(val) QueryPerformanceCounter((LARGE_INTEGER*)&(val)) +# define READ_TIMESTAMP_UNIT TIMESTAMP_UNIT_QUERY_PERFORMANCE_COUNTER # else RPY_EXTERN long long pypy_read_timestamp(void); # define READ_TIMESTAMP(val) (val) = pypy_read_timestamp() +# define READ_TIMESTAMP_UNIT TIMESTAMP_UNIT_NS # endif #endif + +#define OP_LL_GET_TIMESTAMP_UNIT(res) res = READ_TIMESTAMP_UNIT diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py --- a/rpython/translator/driver.py +++ b/rpython/translator/driver.py @@ -413,11 +413,13 @@ translator.frozen = True standalone = self.standalone + get_gchooks = self.extra.get('get_gchooks', lambda: None) + gchooks = get_gchooks() if standalone: from rpython.translator.c.genc import CStandaloneBuilder cbuilder = CStandaloneBuilder(self.translator, self.entry_point, - config=self.config, + config=self.config, gchooks=gchooks, secondary_entrypoints= self.secondary_entrypoints + annotated_jit_entrypoints) else: @@ -426,7 +428,8 @@ cbuilder = CLibraryBuilder(self.translator, self.entry_point, functions=functions, name='libtesting', - config=self.config) + config=self.config, + gchooks=gchooks) if not standalone: # xxx more messy cbuilder.modulename = self.extmod_name database = cbuilder.build_database() diff --git a/rpython/translator/goal/targetgcbench.py b/rpython/translator/goal/targetgcbench.py --- a/rpython/translator/goal/targetgcbench.py +++ b/rpython/translator/goal/targetgcbench.py @@ -1,16 +1,25 @@ from rpython.translator.goal import gcbench +from rpython.memory.test.test_transformed_gc import MyGcHooks, GcHooksStats # _____ Define and setup target ___ +GC_HOOKS_STATS = GcHooksStats() + +def entry_point(argv): + GC_HOOKS_STATS.reset() + ret = gcbench.entry_point(argv) + minors = GC_HOOKS_STATS.minors + steps = GC_HOOKS_STATS.steps + collects = GC_HOOKS_STATS.collects + print 'GC hooks statistics' + print ' gc-minor: ', minors + print ' gc-collect-step: ', steps + print ' gc-collect: ', collects + return ret + +def get_gchooks(): + return MyGcHooks(GC_HOOKS_STATS) + def target(*args): gcbench.ENABLE_THREADS = False # not RPython - return gcbench.entry_point, None - -""" -Why is this a stand-alone target? - -The above target specifies None as the argument types list. -This is a case treated specially in the driver.py . If the list -of input types is empty, it is meant to be a list of strings, -actually implementing argv of the executable. -""" + return entry_point, None From pypy.commits at gmail.com Tue Apr 17 09:06:57 2018 From: pypy.commits at gmail.com (antocuni) Date: Tue, 17 Apr 2018 06:06:57 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: hg merge gc-hooks Message-ID: <5ad5f171.0fbcdf0a.5d62b.bde4@mx.google.com> Author: Antonio Cuni Branch: py3.5 Changeset: r94362:dcf6cc8d15a3 Date: 2018-04-17 14:06 +0100 http://bitbucket.org/pypy/pypy/changeset/dcf6cc8d15a3/ Log: hg merge gc-hooks diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst --- a/pypy/doc/gc_info.rst +++ b/pypy/doc/gc_info.rst @@ -121,6 +121,160 @@ alive by GC objects, but not accounted in the GC +GC Hooks +-------- + +GC hooks are user-defined functions which are called whenever a specific GC +event occur, and can be used to monitor GC activity and pauses. You can +install the hooks by setting the following attributes: + +``gc.hook.on_gc_minor`` + Called whenever a minor collection occurs. It corresponds to + ``gc-minor`` sections inside ``PYPYLOG``. + +``gc.hook.on_gc_collect_step`` + Called whenever an incremental step of a major collection occurs. It + corresponds to ``gc-collect-step`` sections inside ``PYPYLOG``. + +``gc.hook.on_gc_collect`` + Called after the last incremental step, when a major collection is fully + done. It corresponds to ``gc-collect-done`` sections inside ``PYPYLOG``. + +To uninstall a hook, simply set the corresponding attribute to ``None``. To +install all hooks at once, you can call ``gc.hooks.set(obj)``, which will look +for methods ``on_gc_*`` on ``obj``. To uninstall all the hooks at once, you +can call ``gc.hooks.reset()``. + +The functions called by the hooks receive a single ``stats`` argument, which +contains various statistics about the event. + +Note that PyPy cannot call the hooks immediately after a GC event, but it has +to wait until it reaches a point in which the interpreter is in a known state +and calling user-defined code is harmless. It might happen that multiple +events occur before the hook is invoked: in this case, you can inspect the +value ``stats.count`` to know how many times the event occured since the last +time the hook was called. Similarly, ``stats.duration`` contains the +**total** time spent by the GC for this specific event since the last time the +hook was called. + +On the other hand, all the other fields of the ``stats`` object are relative +only to the **last** event of the series. + +The attributes for ``GcMinorStats`` are: + +``count`` + The number of minor collections occured since the last hook call. + +``duration`` + The total time spent inside minor collections since the last hook + call. See below for more information on the unit. + + ``total_memory_used`` + The amount of memory used at the end of the minor collection, in + bytes. This include the memory used in arenas (for GC-managed memory) and + raw-malloced memory (e.g., the content of numpy arrays). + +``pinned_objects`` + the number of pinned objects. + + +The attributes for ``GcCollectStepStats`` are: + +``count``, ``duration`` + See above. + +``oldstate``, ``newstate`` + Integers which indicate the state of the GC before and after the step. + +The value of ``oldstate`` and ``newstate`` is one of these constants, defined +inside ``gc.GcCollectStepStats``: ``STATE_SCANNING``, ``STATE_MARKING``, +``STATE_SWEEPING``, ``STATE_FINALIZING``. It is possible to get a string +representation of it by indexing the ``GC_STATS`` tuple. + + +The attributes for ``GcCollectStats`` are: + +``count`` + See above. + +``num_major_collects`` + The total number of major collections which have been done since the + start. Contrarily to ``count``, this is an always-growing counter and it's + not reset between invocations. + +``arenas_count_before``, ``arenas_count_after`` + Number of arenas used before and after the major collection. + +``arenas_bytes`` + Total number of bytes used by GC-managed objects. + +``rawmalloc_bytes_before``, ``rawmalloc_bytes_after`` + Total number of bytes used by raw-malloced objects, before and after the + major collection. + +Note that ``GcCollectStats`` has **not** got a ``duration`` field. This is +because all the GC work is done inside ``gc-collect-step``: +``gc-collect-done`` is used only to give additional stats, but doesn't do any +actual work. + +A note about the ``duration`` field: depending on the architecture and +operating system, PyPy uses different ways to read timestamps, so ``duration`` +is expressed in varying units. It is possible to know which by calling +``__pypy__.debug_get_timestamp_unit()``, which can be one of the following +values: + +``tsc`` + The default on ``x86`` machines: timestamps are expressed in CPU ticks, as + read by the `Time Stamp Counter`_. + +``ns`` + Timestamps are expressed in nanoseconds. + +``QueryPerformanceCounter`` + On Windows, in case for some reason ``tsc`` is not available: timestamps + are read using the win API ``QueryPerformanceCounter()``. + + +Unfortunately, there does not seem to be a reliable standard way for +converting ``tsc`` ticks into nanoseconds, although in practice on modern CPUs +it is enough to divide the ticks by the maximum nominal frequency of the CPU. +For this reason, PyPy gives the raw value, and leaves the job of doing the +conversion to external libraries. + +Here is an example of GC hooks in use:: + + import sys + import gc + + class MyHooks(object): + done = False + + def on_gc_minor(self, stats): + print 'gc-minor: count = %02d, duration = %d' % (stats.count, + stats.duration) + + def on_gc_collect_step(self, stats): + old = gc.GcCollectStepStats.GC_STATES[stats.oldstate] + new = gc.GcCollectStepStats.GC_STATES[stats.newstate] + print 'gc-collect-step: %s --> %s' % (old, new) + print ' count = %02d, duration = %d' % (stats.count, + stats.duration) + + def on_gc_collect(self, stats): + print 'gc-collect-done: count = %02d' % stats.count + self.done = True + + hooks = MyHooks() + gc.hooks.set(hooks) + + # simulate some GC activity + lst = [] + while not hooks.done: + lst = [lst, 1, 2, 3] + + +.. _`Time Stamp Counter`: https://en.wikipedia.org/wiki/Time_Stamp_Counter + .. _minimark-environment-variables: Environment variables diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -10,3 +10,7 @@ Fix a rare GC bug that was introduced more than one year ago, but was not diagnosed before issue #2752. + +.. branch: gc-hooks + +Introduce GC hooks, as documented in doc/gc_info.rst diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -223,6 +223,7 @@ usage = SUPPRESS_USAGE take_options = True + space = None def opt_parser(self, config): parser = to_optparse(config, useoptions=["objspace.*"], @@ -382,15 +383,21 @@ from pypy.module.pypyjit.hooks import pypy_hooks return PyPyJitPolicy(pypy_hooks) + def get_gchooks(self): + from pypy.module.gc.hook import LowLevelGcHooks + if self.space is None: + raise Exception("get_gchooks must be called afeter get_entry_point") + return self.space.fromcache(LowLevelGcHooks) + def get_entry_point(self, config): - space = make_objspace(config) + self.space = make_objspace(config) # manually imports app_main.py filename = os.path.join(pypydir, 'interpreter', 'app_main.py') app = gateway.applevel(open(filename).read(), 'app_main.py', 'app_main') app.hidden_applevel = False - w_dict = app.getwdict(space) - entry_point, _ = create_entry_point(space, w_dict) + w_dict = app.getwdict(self.space) + entry_point, _ = create_entry_point(self.space, w_dict) return entry_point, None, PyPyAnnotatorPolicy() @@ -399,7 +406,7 @@ 'jitpolicy', 'get_entry_point', 'get_additional_config_options']: ns[name] = getattr(self, name) - + ns['get_gchooks'] = self.get_gchooks PyPyTarget().interface(globals()) diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -392,7 +392,7 @@ self._periodic_actions = [] self._nonperiodic_actions = [] self.has_bytecode_counter = False - self.fired_actions = None + self._fired_actions_reset() # the default value is not 100, unlike CPython 2.7, but a much # larger value, because we use a technique that not only allows # but actually *forces* another thread to run whenever the counter @@ -404,13 +404,28 @@ """Request for the action to be run before the next opcode.""" if not action._fired: action._fired = True - if self.fired_actions is None: - self.fired_actions = [] - self.fired_actions.append(action) + self._fired_actions_append(action) # set the ticker to -1 in order to force action_dispatcher() # to run at the next possible bytecode self.reset_ticker(-1) + def _fired_actions_reset(self): + # linked list of actions. We cannot use a normal RPython list because + # we want AsyncAction.fire() to be marked as @rgc.collect: this way, + # we can call it from e.g. GcHooks or cpyext's dealloc_trigger. + self._fired_actions_first = None + self._fired_actions_last = None + + @rgc.no_collect + def _fired_actions_append(self, action): + assert action._next is None + if self._fired_actions_first is None: + self._fired_actions_first = action + self._fired_actions_last = action + else: + self._fired_actions_last._next = action + self._fired_actions_last = action + @not_rpython def register_periodic_action(self, action, use_bytecode_counter): """ @@ -455,9 +470,9 @@ action.perform(ec, frame) # nonperiodic actions - list = self.fired_actions - if list is not None: - self.fired_actions = None + action = self._fired_actions_first + if action: + self._fired_actions_reset() # NB. in case there are several actions, we reset each # 'action._fired' to false only when we're about to call # 'action.perform()'. This means that if @@ -465,9 +480,10 @@ # the corresponding perform(), the fire() has no # effect---which is the effect we want, because # perform() will be called anyway. - for action in list: + while action is not None: action._fired = False action.perform(ec, frame) + action._next, action = None, action._next self.action_dispatcher = action_dispatcher @@ -500,10 +516,12 @@ to occur between two opcodes, not at a completely random time. """ _fired = False + _next = None def __init__(self, space): self.space = space + @rgc.no_collect def fire(self): """Request for the action to be run before the next opcode. The action must have been registered at space initalization time.""" diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py --- a/pypy/interpreter/test/test_executioncontext.py +++ b/pypy/interpreter/test/test_executioncontext.py @@ -36,6 +36,37 @@ pass assert i == 9 + def test_action_queue(self): + events = [] + + class Action1(executioncontext.AsyncAction): + def perform(self, ec, frame): + events.append('one') + + class Action2(executioncontext.AsyncAction): + def perform(self, ec, frame): + events.append('two') + + space = self.space + a1 = Action1(space) + a2 = Action2(space) + a1.fire() + a2.fire() + space.appexec([], """(): + n = 5 + return n + 2 + """) + assert events == ['one', 'two'] + # + events[:] = [] + a1.fire() + space.appexec([], """(): + n = 5 + return n + 2 + """) + assert events == ['one'] + + def test_periodic_action(self): from pypy.interpreter.executioncontext import ActionFlag diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -83,6 +83,8 @@ 'debug_stop' : 'interp_debug.debug_stop', 'debug_print_once' : 'interp_debug.debug_print_once', 'debug_flush' : 'interp_debug.debug_flush', + 'debug_read_timestamp' : 'interp_debug.debug_read_timestamp', + 'debug_get_timestamp_unit' : 'interp_debug.debug_get_timestamp_unit', 'builtinify' : 'interp_magic.builtinify', 'hidden_applevel' : 'interp_magic.hidden_applevel', 'lookup_special' : 'interp_magic.lookup_special', diff --git a/pypy/module/__pypy__/interp_debug.py b/pypy/module/__pypy__/interp_debug.py --- a/pypy/module/__pypy__/interp_debug.py +++ b/pypy/module/__pypy__/interp_debug.py @@ -1,6 +1,6 @@ from pypy.interpreter.gateway import unwrap_spec from rpython.rlib import debug, jit - +from rpython.rlib import rtimer @jit.dont_look_inside @unwrap_spec(category='text') @@ -28,3 +28,18 @@ @jit.dont_look_inside def debug_flush(space): debug.debug_flush() + +def debug_read_timestamp(space): + return space.newint(rtimer.read_timestamp()) + +def debug_get_timestamp_unit(space): + unit = rtimer.get_timestamp_unit() + if unit == rtimer.UNIT_TSC: + unit_str = 'tsc' + elif unit == rtimer.UNIT_NS: + unit_str = 'ns' + elif unit == rtimer.UNIT_QUERY_PERFORMANCE_COUNTER: + unit_str = 'QueryPerformanceCounter' + else: + unit_str = 'UNKNOWN(%d)' % unit + return space.newtext(unit_str) diff --git a/pypy/module/__pypy__/test/test_debug.py b/pypy/module/__pypy__/test/test_debug.py --- a/pypy/module/__pypy__/test/test_debug.py +++ b/pypy/module/__pypy__/test/test_debug.py @@ -48,3 +48,14 @@ from __pypy__ import debug_flush debug_flush() # assert did not crash + + def test_debug_read_timestamp(self): + from __pypy__ import debug_read_timestamp + a = debug_read_timestamp() + b = debug_read_timestamp() + assert b > a + + def test_debug_get_timestamp_unit(self): + from __pypy__ import debug_get_timestamp_unit + unit = debug_get_timestamp_unit() + assert unit in ('tsc', 'ns', 'QueryPerformanceCounter') diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py --- a/pypy/module/gc/__init__.py +++ b/pypy/module/gc/__init__.py @@ -34,5 +34,7 @@ 'get_typeids_z': 'referents.get_typeids_z', 'get_typeids_list': 'referents.get_typeids_list', 'GcRef': 'referents.W_GcRef', + 'hooks': 'space.fromcache(hook.W_AppLevelHooks)', + 'GcCollectStepStats': 'hook.W_GcCollectStepStats', }) MixedModule.__init__(self, space, w_name) diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py new file mode 100644 --- /dev/null +++ b/pypy/module/gc/hook.py @@ -0,0 +1,317 @@ +from rpython.memory.gc.hook import GcHooks +from rpython.memory.gc import incminimark +from rpython.rlib.nonconst import NonConstant +from rpython.rlib.rarithmetic import r_uint, r_longlong +from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault +from pypy.interpreter.baseobjspace import W_Root +from pypy.interpreter.typedef import TypeDef, interp_attrproperty, GetSetProperty +from pypy.interpreter.executioncontext import AsyncAction + +class LowLevelGcHooks(GcHooks): + """ + These are the low-level hooks which are called directly from the GC. + + They can't do much, because the base class marks the methods as + @rgc.no_collect. + + This is expected to be a singleton, created by space.fromcache, and it is + integrated with the translation by targetpypystandalone.get_gchooks + """ + + def __init__(self, space): + self.space = space + self.w_hooks = space.fromcache(W_AppLevelHooks) + + def is_gc_minor_enabled(self): + return self.w_hooks.gc_minor_enabled + + def is_gc_collect_step_enabled(self): + return self.w_hooks.gc_collect_step_enabled + + def is_gc_collect_enabled(self): + return self.w_hooks.gc_collect_enabled + + def on_gc_minor(self, duration, total_memory_used, pinned_objects): + action = self.w_hooks.gc_minor + action.count += 1 + action.duration += duration + action.total_memory_used = total_memory_used + action.pinned_objects = pinned_objects + action.fire() + + def on_gc_collect_step(self, duration, oldstate, newstate): + action = self.w_hooks.gc_collect_step + action.count += 1 + action.duration += duration + action.oldstate = oldstate + action.newstate = newstate + action.fire() + + def on_gc_collect(self, num_major_collects, + arenas_count_before, arenas_count_after, + arenas_bytes, rawmalloc_bytes_before, + rawmalloc_bytes_after): + action = self.w_hooks.gc_collect + action.count += 1 + action.num_major_collects = num_major_collects + action.arenas_count_before = arenas_count_before + action.arenas_count_after = arenas_count_after + action.arenas_bytes = arenas_bytes + action.rawmalloc_bytes_before = rawmalloc_bytes_before + action.rawmalloc_bytes_after = rawmalloc_bytes_after + action.fire() + + +class W_AppLevelHooks(W_Root): + + def __init__(self, space): + self.space = space + self.gc_minor_enabled = False + self.gc_collect_step_enabled = False + self.gc_collect_enabled = False + self.gc_minor = GcMinorHookAction(space) + self.gc_collect_step = GcCollectStepHookAction(space) + self.gc_collect = GcCollectHookAction(space) + + def descr_get_on_gc_minor(self, space): + return self.gc_minor.w_callable + + def descr_set_on_gc_minor(self, space, w_obj): + self.gc_minor_enabled = not space.is_none(w_obj) + self.gc_minor.w_callable = w_obj + self.gc_minor.fix_annotation() + + def descr_get_on_gc_collect_step(self, space): + return self.gc_collect_step.w_callable + + def descr_set_on_gc_collect_step(self, space, w_obj): + self.gc_collect_step_enabled = not space.is_none(w_obj) + self.gc_collect_step.w_callable = w_obj + self.gc_collect_step.fix_annotation() + + def descr_get_on_gc_collect(self, space): + return self.gc_collect.w_callable + + def descr_set_on_gc_collect(self, space, w_obj): + self.gc_collect_enabled = not space.is_none(w_obj) + self.gc_collect.w_callable = w_obj + self.gc_collect.fix_annotation() + + def descr_set(self, space, w_obj): + w_a = space.getattr(w_obj, space.newtext('on_gc_minor')) + w_b = space.getattr(w_obj, space.newtext('on_gc_collect_step')) + w_c = space.getattr(w_obj, space.newtext('on_gc_collect')) + self.descr_set_on_gc_minor(space, w_a) + self.descr_set_on_gc_collect_step(space, w_b) + self.descr_set_on_gc_collect(space, w_c) + + def descr_reset(self, space): + self.descr_set_on_gc_minor(space, space.w_None) + self.descr_set_on_gc_collect_step(space, space.w_None) + self.descr_set_on_gc_collect(space, space.w_None) + + +class GcMinorHookAction(AsyncAction): + count = 0 + duration = r_longlong(0) + total_memory_used = 0 + pinned_objects = 0 + + def __init__(self, space): + AsyncAction.__init__(self, space) + self.w_callable = space.w_None + + def reset(self): + self.count = 0 + self.duration = r_longlong(0) + + def fix_annotation(self): + # the annotation of the class and its attributes must be completed + # BEFORE we do the gc transform; this makes sure that everything is + # annotated with the correct types + if NonConstant(False): + self.count = NonConstant(-42) + self.duration = NonConstant(r_longlong(-42)) + self.total_memory_used = NonConstant(r_uint(42)) + self.pinned_objects = NonConstant(-42) + self.fire() + + def perform(self, ec, frame): + w_stats = W_GcMinorStats( + self.count, + self.duration, + self.total_memory_used, + self.pinned_objects) + self.reset() + self.space.call_function(self.w_callable, w_stats) + + +class GcCollectStepHookAction(AsyncAction): + count = 0 + duration = r_longlong(0) + oldstate = 0 + newstate = 0 + + def __init__(self, space): + AsyncAction.__init__(self, space) + self.w_callable = space.w_None + + def reset(self): + self.count = 0 + self.duration = r_longlong(0) + + def fix_annotation(self): + # the annotation of the class and its attributes must be completed + # BEFORE we do the gc transform; this makes sure that everything is + # annotated with the correct types + if NonConstant(False): + self.count = NonConstant(-42) + self.duration = NonConstant(r_longlong(-42)) + self.oldstate = NonConstant(-42) + self.newstate = NonConstant(-42) + self.fire() + + def perform(self, ec, frame): + w_stats = W_GcCollectStepStats( + self.count, + self.duration, + self.oldstate, + self.newstate) + self.reset() + self.space.call_function(self.w_callable, w_stats) + + +class GcCollectHookAction(AsyncAction): + count = 0 + num_major_collects = 0 + arenas_count_before = 0 + arenas_count_after = 0 + arenas_bytes = 0 + rawmalloc_bytes_before = 0 + rawmalloc_bytes_after = 0 + + def __init__(self, space): + AsyncAction.__init__(self, space) + self.w_callable = space.w_None + + def reset(self): + self.count = 0 + + def fix_annotation(self): + # the annotation of the class and its attributes must be completed + # BEFORE we do the gc transform; this makes sure that everything is + # annotated with the correct types + if NonConstant(False): + self.count = NonConstant(-42) + self.num_major_collects = NonConstant(-42) + self.arenas_count_before = NonConstant(-42) + self.arenas_count_after = NonConstant(-42) + self.arenas_bytes = NonConstant(r_uint(42)) + self.rawmalloc_bytes_before = NonConstant(r_uint(42)) + self.rawmalloc_bytes_after = NonConstant(r_uint(42)) + self.fire() + + def perform(self, ec, frame): + w_stats = W_GcCollectStats(self.count, + self.num_major_collects, + self.arenas_count_before, + self.arenas_count_after, + self.arenas_bytes, + self.rawmalloc_bytes_before, + self.rawmalloc_bytes_after) + self.reset() + self.space.call_function(self.w_callable, w_stats) + + +class W_GcMinorStats(W_Root): + + def __init__(self, count, duration, total_memory_used, pinned_objects): + self.count = count + self.duration = duration + self.total_memory_used = total_memory_used + self.pinned_objects = pinned_objects + + +class W_GcCollectStepStats(W_Root): + + def __init__(self, count, duration, oldstate, newstate): + self.count = count + self.duration = duration + self.oldstate = oldstate + self.newstate = newstate + + +class W_GcCollectStats(W_Root): + def __init__(self, count, num_major_collects, + arenas_count_before, arenas_count_after, + arenas_bytes, rawmalloc_bytes_before, + rawmalloc_bytes_after): + self.count = count + self.num_major_collects = num_major_collects + self.arenas_count_before = arenas_count_before + self.arenas_count_after = arenas_count_after + self.arenas_bytes = arenas_bytes + self.rawmalloc_bytes_before = rawmalloc_bytes_before + self.rawmalloc_bytes_after = rawmalloc_bytes_after + + +# just a shortcut to make the typedefs shorter +def wrap_many_ints(cls, names): + d = {} + for name in names: + d[name] = interp_attrproperty(name, cls=cls, wrapfn="newint") + return d + + +W_AppLevelHooks.typedef = TypeDef( + "GcHooks", + on_gc_minor = GetSetProperty( + W_AppLevelHooks.descr_get_on_gc_minor, + W_AppLevelHooks.descr_set_on_gc_minor), + + on_gc_collect_step = GetSetProperty( + W_AppLevelHooks.descr_get_on_gc_collect_step, + W_AppLevelHooks.descr_set_on_gc_collect_step), + + on_gc_collect = GetSetProperty( + W_AppLevelHooks.descr_get_on_gc_collect, + W_AppLevelHooks.descr_set_on_gc_collect), + + set = interp2app(W_AppLevelHooks.descr_set), + reset = interp2app(W_AppLevelHooks.descr_reset), + ) + +W_GcMinorStats.typedef = TypeDef( + "GcMinorStats", + **wrap_many_ints(W_GcMinorStats, ( + "count", + "duration", + "total_memory_used", + "pinned_objects")) + ) + +W_GcCollectStepStats.typedef = TypeDef( + "GcCollectStepStats", + STATE_SCANNING = incminimark.STATE_SCANNING, + STATE_MARKING = incminimark.STATE_MARKING, + STATE_SWEEPING = incminimark.STATE_SWEEPING, + STATE_FINALIZING = incminimark.STATE_FINALIZING, + GC_STATES = tuple(incminimark.GC_STATES), + **wrap_many_ints(W_GcCollectStepStats, ( + "count", + "duration", + "oldstate", + "newstate")) + ) + +W_GcCollectStats.typedef = TypeDef( + "GcCollectStats", + **wrap_many_ints(W_GcCollectStats, ( + "count", + "num_major_collects", + "arenas_count_before", + "arenas_count_after", + "arenas_bytes", + "rawmalloc_bytes_before", + "rawmalloc_bytes_after")) + ) diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py new file mode 100644 --- /dev/null +++ b/pypy/module/gc/test/test_hook.py @@ -0,0 +1,176 @@ +import pytest +from rpython.rlib.rarithmetic import r_uint +from pypy.module.gc.hook import LowLevelGcHooks +from pypy.interpreter.baseobjspace import ObjSpace +from pypy.interpreter.gateway import interp2app, unwrap_spec + +class AppTestGcHooks(object): + + def setup_class(cls): + if cls.runappdirect: + pytest.skip("these tests cannot work with -A") + space = cls.space + gchooks = space.fromcache(LowLevelGcHooks) + + @unwrap_spec(ObjSpace, int, r_uint, int) + def fire_gc_minor(space, duration, total_memory_used, pinned_objects): + gchooks.fire_gc_minor(duration, total_memory_used, pinned_objects) + + @unwrap_spec(ObjSpace, int, int, int) + def fire_gc_collect_step(space, duration, oldstate, newstate): + gchooks.fire_gc_collect_step(duration, oldstate, newstate) + + @unwrap_spec(ObjSpace, int, int, int, r_uint, r_uint, r_uint) + def fire_gc_collect(space, a, b, c, d, e, f): + gchooks.fire_gc_collect(a, b, c, d, e, f) + + @unwrap_spec(ObjSpace) + def fire_many(space): + gchooks.fire_gc_minor(5, 0, 0) + gchooks.fire_gc_minor(7, 0, 0) + gchooks.fire_gc_collect_step(5, 0, 0) + gchooks.fire_gc_collect_step(15, 0, 0) + gchooks.fire_gc_collect_step(22, 0, 0) + gchooks.fire_gc_collect(1, 2, 3, 4, 5, 6) + + cls.w_fire_gc_minor = space.wrap(interp2app(fire_gc_minor)) + cls.w_fire_gc_collect_step = space.wrap(interp2app(fire_gc_collect_step)) + cls.w_fire_gc_collect = space.wrap(interp2app(fire_gc_collect)) + cls.w_fire_many = space.wrap(interp2app(fire_many)) + + def test_default(self): + import gc + assert gc.hooks.on_gc_minor is None + assert gc.hooks.on_gc_collect_step is None + assert gc.hooks.on_gc_collect is None + + def test_on_gc_minor(self): + import gc + lst = [] + def on_gc_minor(stats): + lst.append((stats.count, + stats.duration, + stats.total_memory_used, + stats.pinned_objects)) + gc.hooks.on_gc_minor = on_gc_minor + self.fire_gc_minor(10, 20, 30) + self.fire_gc_minor(40, 50, 60) + assert lst == [ + (1, 10, 20, 30), + (1, 40, 50, 60), + ] + # + gc.hooks.on_gc_minor = None + self.fire_gc_minor(70, 80, 90) # won't fire because the hooks is disabled + assert lst == [ + (1, 10, 20, 30), + (1, 40, 50, 60), + ] + + def test_on_gc_collect_step(self): + import gc + lst = [] + def on_gc_collect_step(stats): + lst.append((stats.count, + stats.duration, + stats.oldstate, + stats.newstate)) + gc.hooks.on_gc_collect_step = on_gc_collect_step + self.fire_gc_collect_step(10, 20, 30) + self.fire_gc_collect_step(40, 50, 60) + assert lst == [ + (1, 10, 20, 30), + (1, 40, 50, 60), + ] + # + gc.hooks.on_gc_collect_step = None + self.fire_gc_collect_step(70, 80, 90) # won't fire + assert lst == [ + (1, 10, 20, 30), + (1, 40, 50, 60), + ] + + def test_on_gc_collect(self): + import gc + lst = [] + def on_gc_collect(stats): + lst.append((stats.count, + stats.num_major_collects, + stats.arenas_count_before, + stats.arenas_count_after, + stats.arenas_bytes, + stats.rawmalloc_bytes_before, + stats.rawmalloc_bytes_after)) + gc.hooks.on_gc_collect = on_gc_collect + self.fire_gc_collect(1, 2, 3, 4, 5, 6) + self.fire_gc_collect(7, 8, 9, 10, 11, 12) + assert lst == [ + (1, 1, 2, 3, 4, 5, 6), + (1, 7, 8, 9, 10, 11, 12), + ] + # + gc.hooks.on_gc_collect = None + self.fire_gc_collect(42, 42, 42, 42, 42, 42) # won't fire + assert lst == [ + (1, 1, 2, 3, 4, 5, 6), + (1, 7, 8, 9, 10, 11, 12), + ] + + def test_consts(self): + import gc + S = gc.GcCollectStepStats + assert S.STATE_SCANNING == 0 + assert S.STATE_MARKING == 1 + assert S.STATE_SWEEPING == 2 + assert S.STATE_FINALIZING == 3 + assert S.GC_STATES == ('SCANNING', 'MARKING', 'SWEEPING', 'FINALIZING') + + def test_cumulative(self): + import gc + class MyHooks(object): + + def __init__(self): + self.minors = [] + self.steps = [] + + def on_gc_minor(self, stats): + self.minors.append((stats.count, stats.duration)) + + def on_gc_collect_step(self, stats): + self.steps.append((stats.count, stats.duration)) + + on_gc_collect = None + + myhooks = MyHooks() + gc.hooks.set(myhooks) + self.fire_many() + assert myhooks.minors == [(2, 12)] + assert myhooks.steps == [(3, 42)] + + def test_clear_queue(self): + import gc + class MyHooks(object): + + def __init__(self): + self.lst = [] + + def on_gc_minor(self, stats): + self.lst.append('minor') + + def on_gc_collect_step(self, stats): + self.lst.append('step') + + def on_gc_collect(self, stats): + self.lst.append('collect') + + myhooks = MyHooks() + gc.hooks.set(myhooks) + self.fire_many() + assert myhooks.lst == ['minor', 'step', 'collect'] + myhooks.lst[:] = [] + self.fire_gc_minor(0, 0, 0) + assert myhooks.lst == ['minor'] + gc.hooks.reset() + assert gc.hooks.on_gc_minor is None + assert gc.hooks.on_gc_collect_step is None + assert gc.hooks.on_gc_collect is None diff --git a/pypy/module/gc/test/test_ztranslation.py b/pypy/module/gc/test/test_ztranslation.py --- a/pypy/module/gc/test/test_ztranslation.py +++ b/pypy/module/gc/test/test_ztranslation.py @@ -1,4 +1,9 @@ from pypy.objspace.fake.checkmodule import checkmodule def test_checkmodule(): - checkmodule('gc') + # we need to ignore GcCollectStepStats, else checkmodule fails. I think + # this happens because W_GcCollectStepStats.__init__ is only called from + # GcCollectStepHookAction.perform() and the fake objspace doesn't know + # about those: so, perform() is never annotated and the annotator thinks + # W_GcCollectStepStats has no attributes + checkmodule('gc', ignore=['GcCollectStepStats']) diff --git a/pypy/objspace/fake/checkmodule.py b/pypy/objspace/fake/checkmodule.py --- a/pypy/objspace/fake/checkmodule.py +++ b/pypy/objspace/fake/checkmodule.py @@ -4,6 +4,7 @@ def checkmodule(*modnames, **kwds): translate_startup = kwds.pop('translate_startup', True) + ignore = set(kwds.pop('ignore', ())) assert not kwds config = get_pypy_config(translating=True) space = FakeObjSpace(config) @@ -17,6 +18,8 @@ module.init(space) modules.append(module) for name in module.loaders: + if name in ignore: + continue seeobj_w.append(module._load_lazily(space, name)) if hasattr(module, 'submodules'): for cls in module.submodules.itervalues(): diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -2164,6 +2164,11 @@ oopspecindex=EffectInfo.OS_MATH_READ_TIMESTAMP, extraeffect=EffectInfo.EF_CANNOT_RAISE) + def rewrite_op_ll_get_timestamp_unit(self, op): + op1 = self.prepare_builtin_call(op, "ll_get_timestamp_unit", []) + return self.handle_residual_call(op1, + extraeffect=EffectInfo.EF_CANNOT_RAISE) + def rewrite_op_jit_force_quasi_immutable(self, op): v_inst, c_fieldname = op.args descr1 = self.cpu.fielddescrof(v_inst.concretetype.TO, diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py --- a/rpython/jit/codewriter/support.py +++ b/rpython/jit/codewriter/support.py @@ -285,6 +285,9 @@ from rpython.rlib import rtimer return rtimer.read_timestamp() +def _ll_0_ll_get_timestamp_unit(): + from rpython.rlib import rtimer + return rtimer.get_timestamp_unit() # math support # ------------ diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py --- a/rpython/jit/metainterp/test/test_ajit.py +++ b/rpython/jit/metainterp/test/test_ajit.py @@ -2577,6 +2577,14 @@ res = self.interp_operations(f, []) assert res + def test_get_timestamp_unit(self): + import time + from rpython.rlib import rtimer + def f(): + return rtimer.get_timestamp_unit() + unit = self.interp_operations(f, []) + assert unit == rtimer.UNIT_NS + def test_bug688_multiple_immutable_fields(self): myjitdriver = JitDriver(greens=[], reds=['counter','context']) diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py --- a/rpython/memory/gc/base.py +++ b/rpython/memory/gc/base.py @@ -5,6 +5,7 @@ from rpython.memory.support import DEFAULT_CHUNK_SIZE from rpython.memory.support import get_address_stack, get_address_deque from rpython.memory.support import AddressDict, null_address_dict +from rpython.memory.gc.hook import GcHooks from rpython.rtyper.lltypesystem.llmemory import NULL, raw_malloc_usage from rpython.rtyper.annlowlevel import cast_adr_to_nongc_instance @@ -25,7 +26,7 @@ _totalroots_rpy = 0 # for inspector.py def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE, - translated_to_c=True): + translated_to_c=True, hooks=None): self.gcheaderbuilder = GCHeaderBuilder(self.HDR) self.AddressStack = get_address_stack(chunk_size) self.AddressDeque = get_address_deque(chunk_size) @@ -34,6 +35,9 @@ self.config = config assert isinstance(translated_to_c, bool) self.translated_to_c = translated_to_c + if hooks is None: + hooks = GcHooks() # the default hooks are empty + self.hooks = hooks def setup(self): # all runtime mutable values' setup should happen here diff --git a/rpython/memory/gc/hook.py b/rpython/memory/gc/hook.py new file mode 100644 --- /dev/null +++ b/rpython/memory/gc/hook.py @@ -0,0 +1,70 @@ +from rpython.rlib import rgc + +# WARNING: at the moment of writing, gc hooks are implemented only for +# incminimark. Please add calls to hooks to the other GCs if you need it. +class GcHooks(object): + """ + Base class to write your own GC hooks. + + Subclasses are expected to override the on_* methods. Note that such + methods can do only simple stuff such as updating statistics and/or + setting a flag: in particular, they cannot do anything which can possibly + trigger a GC collection. + """ + + def is_gc_minor_enabled(self): + return False + + def is_gc_collect_step_enabled(self): + return False + + def is_gc_collect_enabled(self): + return False + + def on_gc_minor(self, duration, total_memory_used, pinned_objects): + """ + Called after a minor collection + """ + + def on_gc_collect_step(self, duration, oldstate, newstate): + """ + Called after each individual step of a major collection, in case the GC is + incremental. + + ``oldstate`` and ``newstate`` are integers which indicate the GC + state; for incminimark, see incminimark.STATE_* and + incminimark.GC_STATES. + """ + + + def on_gc_collect(self, num_major_collects, + arenas_count_before, arenas_count_after, + arenas_bytes, rawmalloc_bytes_before, + rawmalloc_bytes_after): + """ + Called after a major collection is fully done + """ + + # the fire_* methods are meant to be called from the GC are should NOT be + # overridden + + @rgc.no_collect + def fire_gc_minor(self, duration, total_memory_used, pinned_objects): + if self.is_gc_minor_enabled(): + self.on_gc_minor(duration, total_memory_used, pinned_objects) + + @rgc.no_collect + def fire_gc_collect_step(self, duration, oldstate, newstate): + if self.is_gc_collect_step_enabled(): + self.on_gc_collect_step(duration, oldstate, newstate) + + @rgc.no_collect + def fire_gc_collect(self, num_major_collects, + arenas_count_before, arenas_count_after, + arenas_bytes, rawmalloc_bytes_before, + rawmalloc_bytes_after): + if self.is_gc_collect_enabled(): + self.on_gc_collect(num_major_collects, + arenas_count_before, arenas_count_after, + arenas_bytes, rawmalloc_bytes_before, + rawmalloc_bytes_after) diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -73,6 +73,7 @@ from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop from rpython.rlib.objectmodel import specialize from rpython.rlib import rgc +from rpython.rlib.rtimer import read_timestamp from rpython.memory.gc.minimarkpage import out_of_memory # @@ -1643,6 +1644,7 @@ """Perform a minor collection: find the objects from the nursery that remain alive and move them out.""" # + start = read_timestamp() debug_start("gc-minor") # # All nursery barriers are invalid from this point on. They @@ -1830,8 +1832,8 @@ # from the nursery that we just moved out. self.size_objects_made_old += r_uint(self.nursery_surviving_size) # - debug_print("minor collect, total memory used:", - self.get_total_memory_used()) + total_memory_used = self.get_total_memory_used() + debug_print("minor collect, total memory used:", total_memory_used) debug_print("number of pinned objects:", self.pinned_objects_in_nursery) if self.DEBUG >= 2: @@ -1840,6 +1842,11 @@ self.root_walker.finished_minor_collection() # debug_stop("gc-minor") + duration = read_timestamp() - start + self.hooks.fire_gc_minor( + duration=duration, + total_memory_used=total_memory_used, + pinned_objects=self.pinned_objects_in_nursery) def _reset_flag_old_objects_pointing_to_pinned(self, obj, ignore): ll_assert(self.header(obj).tid & GCFLAG_PINNED_OBJECT_PARENT_KNOWN != 0, @@ -2241,7 +2248,9 @@ # Note - minor collections seem fast enough so that one # is done before every major collection step def major_collection_step(self, reserving_size=0): + start = read_timestamp() debug_start("gc-collect-step") + oldstate = self.gc_state debug_print("starting gc state: ", GC_STATES[self.gc_state]) # Debugging checks if self.pinned_objects_in_nursery == 0: @@ -2421,6 +2430,13 @@ self.stat_rawmalloced_total_size, " => ", self.rawmalloced_total_size) debug_stop("gc-collect-done") + self.hooks.fire_gc_collect( + num_major_collects=self.num_major_collects, + arenas_count_before=self.stat_ac_arenas_count, + arenas_count_after=self.ac.arenas_count, + arenas_bytes=self.ac.total_memory_used, + rawmalloc_bytes_before=self.stat_rawmalloced_total_size, + rawmalloc_bytes_after=self.rawmalloced_total_size) # # Set the threshold for the next major collection to be when we # have allocated 'major_collection_threshold' times more than @@ -2472,6 +2488,11 @@ debug_print("stopping, now in gc state: ", GC_STATES[self.gc_state]) debug_stop("gc-collect-step") + duration = read_timestamp() - start + self.hooks.fire_gc_collect_step( + duration=duration, + oldstate=oldstate, + newstate=self.gc_state) def _sweep_old_objects_pointing_to_pinned(self, obj, new_list): if self.header(obj).tid & GCFLAG_VISITED: diff --git a/rpython/memory/gc/test/test_direct.py b/rpython/memory/gc/test/test_direct.py --- a/rpython/memory/gc/test/test_direct.py +++ b/rpython/memory/gc/test/test_direct.py @@ -70,6 +70,9 @@ class BaseDirectGCTest(object): GC_PARAMS = {} + def get_extra_gc_params(self): + return {} + def setup_method(self, meth): from rpython.config.translationoption import get_combined_translation_config config = get_combined_translation_config(translating=True).translation @@ -78,6 +81,7 @@ if hasattr(meth, 'GC_PARAMS'): GC_PARAMS.update(meth.GC_PARAMS) GC_PARAMS['translated_to_c'] = False + GC_PARAMS.update(self.get_extra_gc_params()) self.gc = self.GCClass(config, **GC_PARAMS) self.gc.DEBUG = True self.rootwalker = DirectRootWalker(self) diff --git a/rpython/memory/gc/test/test_hook.py b/rpython/memory/gc/test/test_hook.py new file mode 100644 --- /dev/null +++ b/rpython/memory/gc/test/test_hook.py @@ -0,0 +1,125 @@ +from rpython.rtyper.lltypesystem import lltype, llmemory +from rpython.memory.gc.hook import GcHooks +from rpython.memory.gc.test.test_direct import BaseDirectGCTest, S + + +class MyGcHooks(GcHooks): + + def __init__(self): + GcHooks.__init__(self) + self._gc_minor_enabled = False + self._gc_collect_step_enabled = False + self._gc_collect_enabled = False + self.reset() + + def is_gc_minor_enabled(self): + return self._gc_minor_enabled + + def is_gc_collect_step_enabled(self): + return self._gc_collect_step_enabled + + def is_gc_collect_enabled(self): + return self._gc_collect_enabled + + def reset(self): + self.minors = [] + self.steps = [] + self.collects = [] + self.durations = [] + + def on_gc_minor(self, duration, total_memory_used, pinned_objects): + self.durations.append(duration) + self.minors.append({ + 'total_memory_used': total_memory_used, + 'pinned_objects': pinned_objects}) + + def on_gc_collect_step(self, duration, oldstate, newstate): + self.durations.append(duration) + self.steps.append({ + 'oldstate': oldstate, + 'newstate': newstate}) + + def on_gc_collect(self, num_major_collects, + arenas_count_before, arenas_count_after, + arenas_bytes, rawmalloc_bytes_before, + rawmalloc_bytes_after): + self.collects.append({ + 'num_major_collects': num_major_collects, + 'arenas_count_before': arenas_count_before, + 'arenas_count_after': arenas_count_after, + 'arenas_bytes': arenas_bytes, + 'rawmalloc_bytes_before': rawmalloc_bytes_before, + 'rawmalloc_bytes_after': rawmalloc_bytes_after}) + + +class TestIncMiniMarkHooks(BaseDirectGCTest): + from rpython.memory.gc.incminimark import IncrementalMiniMarkGC as GCClass + + def get_extra_gc_params(self): + return {'hooks': MyGcHooks()} + + def setup_method(self, m): + BaseDirectGCTest.setup_method(self, m) + size = llmemory.sizeof(S) + self.gc.gcheaderbuilder.size_gc_header + self.size_of_S = llmemory.raw_malloc_usage(size) + + def test_on_gc_minor(self): + self.gc.hooks._gc_minor_enabled = True + self.malloc(S) + self.gc._minor_collection() + assert self.gc.hooks.minors == [ + {'total_memory_used': 0, 'pinned_objects': 0} + ] + assert self.gc.hooks.durations[0] > 0 + self.gc.hooks.reset() + # + # these objects survive, so the total_memory_used is > 0 + self.stackroots.append(self.malloc(S)) + self.stackroots.append(self.malloc(S)) + self.gc._minor_collection() + assert self.gc.hooks.minors == [ + {'total_memory_used': self.size_of_S*2, 'pinned_objects': 0} + ] + + def test_on_gc_collect(self): + from rpython.memory.gc import incminimark as m + self.gc.hooks._gc_collect_step_enabled = True + self.gc.hooks._gc_collect_enabled = True + self.malloc(S) + self.gc.collect() + assert self.gc.hooks.steps == [ + {'oldstate': m.STATE_SCANNING, 'newstate': m.STATE_MARKING}, + {'oldstate': m.STATE_MARKING, 'newstate': m.STATE_SWEEPING}, + {'oldstate': m.STATE_SWEEPING, 'newstate': m.STATE_FINALIZING}, + {'oldstate': m.STATE_FINALIZING, 'newstate': m.STATE_SCANNING} + ] + assert self.gc.hooks.collects == [ + {'num_major_collects': 1, + 'arenas_count_before': 0, + 'arenas_count_after': 0, + 'arenas_bytes': 0, + 'rawmalloc_bytes_after': 0, + 'rawmalloc_bytes_before': 0} + ] + assert len(self.gc.hooks.durations) == 4 # 4 steps + for d in self.gc.hooks.durations: + assert d > 0 + self.gc.hooks.reset() + # + self.stackroots.append(self.malloc(S)) + self.gc.collect() + assert self.gc.hooks.collects == [ + {'num_major_collects': 2, + 'arenas_count_before': 1, + 'arenas_count_after': 1, + 'arenas_bytes': self.size_of_S, + 'rawmalloc_bytes_after': 0, + 'rawmalloc_bytes_before': 0} + ] + + def test_hook_disabled(self): + self.gc._minor_collection() + self.gc.collect() + assert self.gc.hooks.minors == [] + assert self.gc.hooks.steps == [] + assert self.gc.hooks.collects == [] diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -116,7 +116,7 @@ class BaseFrameworkGCTransformer(GCTransformer): root_stack_depth = None # for tests to override - def __init__(self, translator): + def __init__(self, translator, gchooks=None): from rpython.memory.gc.base import choose_gc_from_config super(BaseFrameworkGCTransformer, self).__init__(translator, @@ -162,7 +162,8 @@ self.finalizer_queue_indexes = {} self.finalizer_handlers = [] - gcdata.gc = GCClass(translator.config.translation, **GC_PARAMS) + gcdata.gc = GCClass(translator.config.translation, hooks=gchooks, + **GC_PARAMS) root_walker = self.build_root_walker() root_walker.finished_minor_collection_func = finished_minor_collection self.root_walker = root_walker diff --git a/rpython/memory/test/test_transformed_gc.py b/rpython/memory/test/test_transformed_gc.py --- a/rpython/memory/test/test_transformed_gc.py +++ b/rpython/memory/test/test_transformed_gc.py @@ -14,7 +14,9 @@ from rpython.conftest import option from rpython.rlib.rstring import StringBuilder from rpython.rlib.rarithmetic import LONG_BIT +from rpython.rlib.nonconst import NonConstant from rpython.rtyper.rtyper import llinterp_backend +from rpython.memory.gc.hook import GcHooks WORD = LONG_BIT // 8 @@ -48,6 +50,7 @@ gcpolicy = None GC_CAN_MOVE = False taggedpointers = False + gchooks = None def setup_class(cls): cls.marker = lltype.malloc(rffi.CArray(lltype.Signed), 1, @@ -112,7 +115,8 @@ fixup(t) cbuild = CStandaloneBuilder(t, entrypoint, config=t.config, - gcpolicy=cls.gcpolicy) + gcpolicy=cls.gcpolicy, + gchooks=cls.gchooks) cbuild.make_entrypoint_wrapper = False db = cbuild.build_database() entrypointptr = cbuild.getentrypointptr() @@ -1388,6 +1392,48 @@ assert res([]) == 0 +class GcHooksStats(object): + minors = 0 + steps = 0 + collects = 0 + + def reset(self): + # the NonConstant are needed so that the annotator annotates the + # fields as a generic SomeInteger(), instead of a constant 0. A call + # to this method MUST be seen during normal annotation, else the class + # is annotated only during GC transform, when it's too late + self.minors = NonConstant(0) + self.steps = NonConstant(0) + self.collects = NonConstant(0) + + +class MyGcHooks(GcHooks): + + def __init__(self, stats=None): + self.stats = stats or GcHooksStats() + + def is_gc_minor_enabled(self): + return True + + def is_gc_collect_step_enabled(self): + return True + + def is_gc_collect_enabled(self): + return True + + def on_gc_minor(self, duration, total_memory_used, pinned_objects): + self.stats.minors += 1 + + def on_gc_collect_step(self, duration, oldstate, newstate): + self.stats.steps += 1 + + def on_gc_collect(self, num_major_collects, + arenas_count_before, arenas_count_after, + arenas_bytes, rawmalloc_bytes_before, + rawmalloc_bytes_after): + self.stats.collects += 1 + + class TestIncrementalMiniMarkGC(TestMiniMarkGC): gcname = "incminimark" @@ -1405,6 +1451,8 @@ } root_stack_depth = 200 + gchooks = MyGcHooks() + def define_malloc_array_of_gcptr(self): S = lltype.GcStruct('S', ('x', lltype.Signed)) A = lltype.GcArray(lltype.Ptr(S)) @@ -1438,6 +1486,34 @@ res = run([]) assert res + def define_gc_hooks(cls): + gchooks = cls.gchooks + # it is important that we fish .stats OUTSIDE f(); we cannot see + # gchooks from within RPython code + stats = gchooks.stats + def f(): + stats.reset() + # trigger two major collections + llop.gc__collect(lltype.Void) + llop.gc__collect(lltype.Void) + return (10000 * stats.collects + + 100 * stats.steps + + 1 * stats.minors) + return f + + def test_gc_hooks(self): + run = self.runner("gc_hooks") + count = run([]) + collects, count = divmod(count, 10000) + steps, minors = divmod(count, 100) + # + # note: the following asserts are slightly fragile, as they assume + # that we do NOT run any minor collection apart the ones triggered by + # major_collection_step + assert collects == 2 # 2 collections, manually triggered + assert steps == 4 * collects # 4 steps for each major collection + assert minors == steps # one minor collection for each step + # ________________________________________________________________ # tagged pointers diff --git a/rpython/rlib/rtimer.py b/rpython/rlib/rtimer.py --- a/rpython/rlib/rtimer.py +++ b/rpython/rlib/rtimer.py @@ -7,6 +7,11 @@ _is_64_bit = r_uint.BITS > 32 +# unit of values returned by read_timestamp. Should be in sync with the ones +# defined in translator/c/debug_print.h +UNIT_TSC = 0 +UNIT_NS = 1 # nanoseconds +UNIT_QUERY_PERFORMANCE_COUNTER = 2 def read_timestamp(): # Returns a longlong on 32-bit, and a regular int on 64-bit. @@ -17,6 +22,11 @@ else: return longlongmask(x) +def get_timestamp_unit(): + # an unit which is as arbitrary as the way we build the result of + # read_timestamp :) + return UNIT_NS + class ReadTimestampEntry(ExtRegistryEntry): _about_ = read_timestamp @@ -35,3 +45,15 @@ else: resulttype = rffi.LONGLONG return hop.genop("ll_read_timestamp", [], resulttype=resulttype) + + +class ReadTimestampEntry(ExtRegistryEntry): + _about_ = get_timestamp_unit + + def compute_result_annotation(self): + from rpython.annotator.model import SomeInteger + return SomeInteger(nonneg=True) + + def specialize_call(self, hop): + hop.exception_cannot_occur() + return hop.genop("ll_get_timestamp_unit", [], resulttype=lltype.Signed) diff --git a/rpython/rlib/test/test_rtimer.py b/rpython/rlib/test/test_rtimer.py --- a/rpython/rlib/test/test_rtimer.py +++ b/rpython/rlib/test/test_rtimer.py @@ -1,28 +1,56 @@ import time - -from rpython.rlib.rtimer import read_timestamp +import platform +from rpython.rlib import rtimer from rpython.rtyper.test.test_llinterp import interpret from rpython.translator.c.test.test_genc import compile -def timer(): - t1 = read_timestamp() - start = time.time() - while time.time() - start < 0.1: - # busy wait - pass - t2 = read_timestamp() - return t2 - t1 +class TestTimer(object): -def test_timer(): - diff = timer() - # We're counting ticks, verify they look correct - assert diff > 1000 + @staticmethod + def timer(): + t1 = rtimer.read_timestamp() + start = time.time() + while time.time() - start < 0.1: + # busy wait + pass + t2 = rtimer.read_timestamp() + return t2 - t1 -def test_annotation(): - diff = interpret(timer, []) - assert diff > 1000 + def test_direct(self): + diff = self.timer() + # We're counting ticks, verify they look correct + assert diff > 1000 -def test_compile_c(): - function = compile(timer, []) - diff = function() - assert diff > 1000 \ No newline at end of file + def test_annotation(self): + diff = interpret(self.timer, []) + assert diff > 1000 + + def test_compile_c(self): + function = compile(self.timer, []) + diff = function() + assert diff > 1000 + + +class TestGetUnit(object): + + @staticmethod + def get_unit(): + return rtimer.get_timestamp_unit() + + def test_direct(self): + unit = self.get_unit() + assert unit == rtimer.UNIT_NS + + def test_annotation(self): + unit = interpret(self.get_unit, []) + assert unit == rtimer.UNIT_NS + + def test_compile_c(self): + function = compile(self.get_unit, []) + unit = function() + if platform.processor() in ('x86', 'x86_64'): + assert unit == rtimer.UNIT_TSC + else: + assert unit in (rtimer.UNIT_TSC, + rtimer.UNIT_NS, + rtimer.UNIT_QUERY_PERFORMANCE_COUNTER) diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -445,6 +445,7 @@ 'get_write_barrier_from_array_failing_case': LLOp(sideeffects=False), 'gc_get_type_info_group': LLOp(sideeffects=False), 'll_read_timestamp': LLOp(canrun=True), + 'll_get_timestamp_unit': LLOp(canrun=True), # __________ GC operations __________ diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py --- a/rpython/rtyper/lltypesystem/opimpl.py +++ b/rpython/rtyper/lltypesystem/opimpl.py @@ -696,6 +696,10 @@ from rpython.rlib.rtimer import read_timestamp return read_timestamp() +def op_ll_get_timestamp_unit(): + from rpython.rlib.rtimer import get_timestamp_unit + return get_timestamp_unit() + def op_debug_fatalerror(ll_msg): from rpython.rtyper.lltypesystem import lltype, rstr from rpython.rtyper.llinterp import LLFatalError diff --git a/rpython/translator/c/database.py b/rpython/translator/c/database.py --- a/rpython/translator/c/database.py +++ b/rpython/translator/c/database.py @@ -29,6 +29,7 @@ def __init__(self, translator=None, standalone=False, gcpolicyclass=None, + gchooks=None, exctransformer=None, thread_enabled=False, sandbox=False): @@ -56,7 +57,7 @@ self.namespace = CNameManager() if translator is not None: - self.gctransformer = self.gcpolicy.gettransformer(translator) + self.gctransformer = self.gcpolicy.gettransformer(translator, gchooks) self.completed = False self.instrument_ncounter = 0 diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py --- a/rpython/translator/c/gc.py +++ b/rpython/translator/c/gc.py @@ -94,7 +94,7 @@ class RefcountingGcPolicy(BasicGcPolicy): - def gettransformer(self, translator): + def gettransformer(self, translator, gchooks): from rpython.memory.gctransform import refcounting return refcounting.RefcountingGCTransformer(translator) @@ -175,7 +175,7 @@ class BoehmGcPolicy(BasicGcPolicy): - def gettransformer(self, translator): + def gettransformer(self, translator, gchooks): from rpython.memory.gctransform import boehm return boehm.BoehmGCTransformer(translator) @@ -302,9 +302,9 @@ class BasicFrameworkGcPolicy(BasicGcPolicy): - def gettransformer(self, translator): + def gettransformer(self, translator, gchooks): if hasattr(self, 'transformerclass'): # for rpython/memory tests - return self.transformerclass(translator) + return self.transformerclass(translator, gchooks=gchooks) raise NotImplementedError def struct_setup(self, structdefnode, rtti): @@ -439,9 +439,9 @@ class ShadowStackFrameworkGcPolicy(BasicFrameworkGcPolicy): - def gettransformer(self, translator): + def gettransformer(self, translator, gchooks): from rpython.memory.gctransform import shadowstack - return shadowstack.ShadowStackFrameworkGCTransformer(translator) + return shadowstack.ShadowStackFrameworkGCTransformer(translator, gchooks) def enter_roots_frame(self, funcgen, (c_gcdata, c_numcolors)): numcolors = c_numcolors.value @@ -484,9 +484,9 @@ class AsmGcRootFrameworkGcPolicy(BasicFrameworkGcPolicy): - def gettransformer(self, translator): + def gettransformer(self, translator, gchooks): from rpython.memory.gctransform import asmgcroot - return asmgcroot.AsmGcRootFrameworkGCTransformer(translator) + return asmgcroot.AsmGcRootFrameworkGCTransformer(translator, gchooks) def GC_KEEPALIVE(self, funcgen, v): return 'pypy_asm_keepalive(%s);' % funcgen.expr(v) diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py --- a/rpython/translator/c/genc.py +++ b/rpython/translator/c/genc.py @@ -64,13 +64,14 @@ split = False def __init__(self, translator, entrypoint, config, gcpolicy=None, - secondary_entrypoints=()): + gchooks=None, secondary_entrypoints=()): self.translator = translator self.entrypoint = entrypoint self.entrypoint_name = getattr(self.entrypoint, 'func_name', None) self.originalentrypoint = entrypoint self.config = config self.gcpolicy = gcpolicy # for tests only, e.g. rpython/memory/ + self.gchooks = gchooks self.eci = self.get_eci() self.secondary_entrypoints = secondary_entrypoints @@ -91,6 +92,7 @@ exctransformer = translator.getexceptiontransformer() db = LowLevelDatabase(translator, standalone=self.standalone, gcpolicyclass=gcpolicyclass, + gchooks=self.gchooks, exctransformer=exctransformer, thread_enabled=self.config.translation.thread, sandbox=self.config.translation.sandbox) diff --git a/rpython/translator/c/src/asm_gcc_x86.h b/rpython/translator/c/src/asm_gcc_x86.h --- a/rpython/translator/c/src/asm_gcc_x86.h +++ b/rpython/translator/c/src/asm_gcc_x86.h @@ -70,6 +70,7 @@ // lfence // I don't know how important it is, comment talks about time warps +#define READ_TIMESTAMP_UNIT TIMESTAMP_UNIT_TSC #ifndef PYPY_CPU_HAS_STANDARD_PRECISION /* On x86-32, we have to use the following hacks to set and restore diff --git a/rpython/translator/c/src/asm_gcc_x86_64.h b/rpython/translator/c/src/asm_gcc_x86_64.h --- a/rpython/translator/c/src/asm_gcc_x86_64.h +++ b/rpython/translator/c/src/asm_gcc_x86_64.h @@ -7,5 +7,6 @@ val = (_rdx << 32) | _rax; \ } while (0) +#define READ_TIMESTAMP_UNIT TIMESTAMP_UNIT_TSC #define RPy_YieldProcessor() asm("pause") diff --git a/rpython/translator/c/src/asm_msvc.h b/rpython/translator/c/src/asm_msvc.h --- a/rpython/translator/c/src/asm_msvc.h +++ b/rpython/translator/c/src/asm_msvc.h @@ -13,3 +13,4 @@ #include #pragma intrinsic(__rdtsc) #define READ_TIMESTAMP(val) do { val = (long long)__rdtsc(); } while (0) +#define READ_TIMESTAMP_UNIT TIMESTAMP_UNIT_TSC diff --git a/rpython/translator/c/src/debug_print.h b/rpython/translator/c/src/debug_print.h --- a/rpython/translator/c/src/debug_print.h +++ b/rpython/translator/c/src/debug_print.h @@ -51,6 +51,11 @@ RPY_EXTERN long pypy_have_debug_prints; RPY_EXPORTED FILE *pypy_debug_file; +/* these should be in sync with the values defined in rlib/rtimer.py */ +#define TIMESTAMP_UNIT_TSC 0 +#define TIMESTAMP_UNIT_NS 1 +#define TIMESTAMP_UNIT_QUERY_PERFORMANCE_COUNTER 2 + #define OP_LL_READ_TIMESTAMP(val) READ_TIMESTAMP(val) #include "src/asm.h" @@ -62,11 +67,15 @@ # ifdef _WIN32 # define READ_TIMESTAMP(val) QueryPerformanceCounter((LARGE_INTEGER*)&(val)) +# define READ_TIMESTAMP_UNIT TIMESTAMP_UNIT_QUERY_PERFORMANCE_COUNTER # else RPY_EXTERN long long pypy_read_timestamp(void); # define READ_TIMESTAMP(val) (val) = pypy_read_timestamp() +# define READ_TIMESTAMP_UNIT TIMESTAMP_UNIT_NS # endif #endif + +#define OP_LL_GET_TIMESTAMP_UNIT(res) res = READ_TIMESTAMP_UNIT diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py --- a/rpython/translator/driver.py +++ b/rpython/translator/driver.py @@ -413,11 +413,13 @@ translator.frozen = True standalone = self.standalone + get_gchooks = self.extra.get('get_gchooks', lambda: None) + gchooks = get_gchooks() if standalone: from rpython.translator.c.genc import CStandaloneBuilder cbuilder = CStandaloneBuilder(self.translator, self.entry_point, - config=self.config, + config=self.config, gchooks=gchooks, secondary_entrypoints= self.secondary_entrypoints + annotated_jit_entrypoints) else: @@ -426,7 +428,8 @@ cbuilder = CLibraryBuilder(self.translator, self.entry_point, functions=functions, name='libtesting', - config=self.config) + config=self.config, + gchooks=gchooks) if not standalone: # xxx more messy cbuilder.modulename = self.extmod_name database = cbuilder.build_database() diff --git a/rpython/translator/goal/targetgcbench.py b/rpython/translator/goal/targetgcbench.py --- a/rpython/translator/goal/targetgcbench.py +++ b/rpython/translator/goal/targetgcbench.py @@ -1,16 +1,25 @@ from rpython.translator.goal import gcbench +from rpython.memory.test.test_transformed_gc import MyGcHooks, GcHooksStats # _____ Define and setup target ___ +GC_HOOKS_STATS = GcHooksStats() + +def entry_point(argv): + GC_HOOKS_STATS.reset() + ret = gcbench.entry_point(argv) + minors = GC_HOOKS_STATS.minors + steps = GC_HOOKS_STATS.steps + collects = GC_HOOKS_STATS.collects + print 'GC hooks statistics' + print ' gc-minor: ', minors + print ' gc-collect-step: ', steps + print ' gc-collect: ', collects + return ret + +def get_gchooks(): + return MyGcHooks(GC_HOOKS_STATS) + def target(*args): gcbench.ENABLE_THREADS = False # not RPython - return gcbench.entry_point, None - -""" -Why is this a stand-alone target? - -The above target specifies None as the argument types list. -This is a case treated specially in the driver.py . If the list -of input types is empty, it is meant to be a list of strings, -actually implementing argv of the executable. -""" + return entry_point, None From pypy.commits at gmail.com Tue Apr 17 17:23:00 2018 From: pypy.commits at gmail.com (rlamy) Date: Tue, 17 Apr 2018 14:23:00 -0700 (PDT) Subject: [pypy-commit] pypy py3tests: fix fix fix Message-ID: <5ad665b4.4f8edf0a.fe5ae.dded@mx.google.com> Author: Ronan Lamy Branch: py3tests Changeset: r94363:18e33a4f6b3e Date: 2018-04-17 22:22 +0100 http://bitbucket.org/pypy/pypy/changeset/18e33a4f6b3e/ Log: fix fix fix diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -566,22 +566,22 @@ def make_builtins(self): "only for initializing the space." - from pypy.module.exceptions import Module + from pypy.module.exceptions.moduledef import Module w_name = self.newtext('__exceptions__') self.exceptions_module = Module(self, w_name) self.exceptions_module.install() - from pypy.module.imp import Module + from pypy.module.imp.moduledef import Module w_name = self.newtext('_imp') mod = Module(self, w_name) mod.install() - from pypy.module.sys import Module + from pypy.module.sys.moduledef import Module w_name = self.newtext('sys') self.sys = Module(self, w_name) self.sys.install() - from pypy.module.__builtin__ import Module + from pypy.module.__builtin__.moduledef import Module w_name = self.newtext('builtins') self.builtin = Module(self, w_name) w_builtin = self.builtin diff --git a/pypy/interpreter/mixedmodule.py b/pypy/interpreter/mixedmodule.py --- a/pypy/interpreter/mixedmodule.py +++ b/pypy/interpreter/mixedmodule.py @@ -173,7 +173,7 @@ # build a constant dictionary out of # applevel/interplevel definitions cls.loaders = loaders = {} - pkgroot = cls.__module__ + pkgroot = cls.__module__.rsplit('.', 1)[0] appname = cls.get_applevel_name() if cls.submodule_name is not None: appname += '.%s' % (cls.submodule_name,) diff --git a/pypy/interpreter/test/test_extmodules.py b/pypy/interpreter/test/test_extmodules.py --- a/pypy/interpreter/test/test_extmodules.py +++ b/pypy/interpreter/test/test_extmodules.py @@ -5,7 +5,7 @@ from pypy.objspace.std.objspace import StdObjSpace from rpython.tool.udir import udir -mod_init = """ +mod_def = """ from pypy.interpreter.mixedmodule import MixedModule import time @@ -45,14 +45,15 @@ pkg.join("__init__.py").write("# package") mod = pkg.join("extmod") mod.ensure(dir=True) - mod.join("__init__.py").write(mod_init) + mod.join("__init__.py").write("#") mod.join("interp_time.py").write(mod_interp) + mod.join("moduledef.py").write(mod_def) class AppTestExtModules(object): def setup_class(cls): init_extmodule_code() conf = get_pypy_config() - conf.objspace.extmodules = 'testext.extmod' + conf.objspace.extmodules = 'testext.extmod.moduledef' old_sys_path[:] = sys.path[:] sys.path.insert(0, str(udir)) space = StdObjSpace(conf) diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -10,7 +10,7 @@ from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault from pypy.interpreter import unicodehelper -from pypy.module.unicodedata import unicodedb +from pypy.module.unicodedata.interp_ucd import unicodedb class VersionTag(object): diff --git a/pypy/module/unicodedata/interp_ucd.py b/pypy/module/unicodedata/interp_ucd.py --- a/pypy/module/unicodedata/interp_ucd.py +++ b/pypy/module/unicodedata/interp_ucd.py @@ -342,3 +342,8 @@ ucd_3_2_0 = UCD(unicodedb_3_2_0) ucd_8_0_0 = UCD(unicodedb_8_0_0) ucd = ucd_8_0_0 + +# This is the default unicodedb used in various places: +# - the unicode type +# - the regular expression engine +unicodedb = ucd._unicodedb diff --git a/pypy/module/unicodedata/moduledef.py b/pypy/module/unicodedata/moduledef.py --- a/pypy/module/unicodedata/moduledef.py +++ b/pypy/module/unicodedata/moduledef.py @@ -1,14 +1,5 @@ from pypy.interpreter.mixedmodule import MixedModule -# This is the default unicodedb used in various places: -# - the unicode type -# - the regular expression engine -from pypy.module.unicodedata.interp_ucd import ucd as _ucd -unicodedb = _ucd._unicodedb - -# to get information about individual unicode chars look at: -# http://www.fileformat.info/info/unicode/char/search.htm - class Module(MixedModule): appleveldefs = { } diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -15,7 +15,7 @@ from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.gateway import WrappedDefault, interp2app, unwrap_spec from pypy.interpreter.typedef import TypeDef -from pypy.module.unicodedata import unicodedb +from pypy.module.unicodedata.interp_ucd import unicodedb from pypy.objspace.std import newformat from pypy.objspace.std.formatting import mod_format, FORMAT_UNICODE from pypy.objspace.std.stringmethods import StringMethods diff --git a/pypy/tool/pytest/fake_pytest/__init__.py b/pypy/tool/pytest/fake_pytest/__init__.py --- a/pypy/tool/pytest/fake_pytest/__init__.py +++ b/pypy/tool/pytest/fake_pytest/__init__.py @@ -1,12 +0,0 @@ -from pypy.interpreter.mixedmodule import MixedModule - -class Module(MixedModule): - applevel_name = 'pytest' - interpleveldefs = { - 'raises': 'interp_pytest.pypyraises', - 'skip': 'interp_pytest.pypyskip', - 'fixture': 'interp_pytest.fake_fixture', - } - appleveldefs = { - 'importorskip': 'app_pytest.importorskip', - } diff --git a/pypy/tool/pytest/fake_pytest/__init__.py b/pypy/tool/pytest/fake_pytest/moduledef.py copy from pypy/tool/pytest/fake_pytest/__init__.py copy to pypy/tool/pytest/fake_pytest/moduledef.py diff --git a/pypy/tool/pytest/objspace.py b/pypy/tool/pytest/objspace.py --- a/pypy/tool/pytest/objspace.py +++ b/pypy/tool/pytest/objspace.py @@ -30,7 +30,7 @@ config = make_config(option) if config.objspace.usemodules.thread: config.translation.thread = True - config.objspace.extmodules = 'pypy.tool.pytest.fake_pytest' + config.objspace.extmodules = 'pypy.tool.pytest.fake_pytest.moduledef' space = make_objspace(config) space.startup() # Initialize all builtin modules if config.objspace.std.reinterpretasserts: From pypy.commits at gmail.com Tue Apr 17 18:01:16 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 17 Apr 2018 15:01:16 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: merge default into py3.5 Message-ID: <5ad66eac.0bda500a.1c9d5.c18b@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r94364:5a7eeaf512fc Date: 2018-04-18 01:00 +0300 http://bitbucket.org/pypy/pypy/changeset/5a7eeaf512fc/ Log: merge default into py3.5 diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py --- a/pypy/module/_cffi_backend/ctypearray.py +++ b/pypy/module/_cffi_backend/ctypearray.py @@ -70,7 +70,15 @@ length = wchar_helper.unicode_size_as_char32(u) return (w_value, length + 1) else: - explicitlength = space.getindex_w(w_value, space.w_OverflowError) + try: + explicitlength = space.getindex_w(w_value, + space.w_OverflowError) + except OperationError as e: + if e.match(space, space.w_TypeError): + raise oefmt(space.w_TypeError, + "expected new array length or list/tuple/str, " + "not %T", w_value) + raise if explicitlength < 0: raise oefmt(space.w_ValueError, "negative array length") return (space.w_None, explicitlength) diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -278,6 +278,9 @@ assert repr(q).startswith(" Author: Ronan Lamy Branch: py3tests Changeset: r94366:b29602856871 Date: 2018-04-18 01:38 +0100 http://bitbucket.org/pypy/pypy/changeset/b29602856871/ Log: Fix token module diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py --- a/pypy/module/_cffi_backend/ctypefunc.py +++ b/pypy/module/_cffi_backend/ctypefunc.py @@ -12,7 +12,8 @@ from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from pypy.interpreter.error import OperationError, oefmt -from pypy.module import _cffi_backend +from pypy.module._cffi_backend.moduledef import ( + FFI_DEFAULT_ABI, has_stdcall, FFI_STDCALL) from pypy.module._cffi_backend import ctypearray, cdataobj, cerrno from pypy.module._cffi_backend.ctypeobj import W_CType from pypy.module._cffi_backend.ctypeptr import W_CTypePtrBase, W_CTypePointer @@ -30,8 +31,7 @@ cif_descr = lltype.nullptr(CIF_DESCRIPTION) - def __init__(self, space, fargs, fresult, ellipsis, - abi=_cffi_backend.FFI_DEFAULT_ABI): + def __init__(self, space, fargs, fresult, ellipsis, abi=FFI_DEFAULT_ABI): assert isinstance(ellipsis, bool) extra, xpos = self._compute_extra_text(fargs, fresult, ellipsis, abi) size = rffi.sizeof(rffi.VOIDP) @@ -101,7 +101,7 @@ from pypy.module._cffi_backend import newtype argnames = ['(*)('] xpos = 2 - if _cffi_backend.has_stdcall and abi == _cffi_backend.FFI_STDCALL: + if has_stdcall and abi == FFI_STDCALL: argnames[0] = '(__stdcall *)(' xpos += len('__stdcall ') for i, farg in enumerate(fargs): diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py --- a/pypy/module/_cffi_backend/ffi_obj.py +++ b/pypy/module/_cffi_backend/ffi_obj.py @@ -6,7 +6,8 @@ from rpython.rlib import jit, rgc from rpython.rtyper.lltypesystem import lltype, rffi -from pypy.module._cffi_backend import get_dict_rtld_constants +from pypy.module._cffi_backend.moduledef import ( + get_dict_rtld_constants) from pypy.module._cffi_backend import parse_c_type, realize_c_type from pypy.module._cffi_backend import newtype, cerrno, ccallback, ctypearray from pypy.module._cffi_backend import ctypestruct, ctypeptr, handle diff --git a/pypy/module/_cffi_backend/moduledef.py b/pypy/module/_cffi_backend/moduledef.py --- a/pypy/module/_cffi_backend/moduledef.py +++ b/pypy/module/_cffi_backend/moduledef.py @@ -11,6 +11,7 @@ has_stdcall = True except AttributeError: has_stdcall = False + FFI_STDCALL = None class Module(MixedModule): diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py --- a/pypy/module/_cffi_backend/newtype.py +++ b/pypy/module/_cffi_backend/newtype.py @@ -8,7 +8,7 @@ from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rtyper.tool import rffi_platform -from pypy.module import _cffi_backend +from pypy.module._cffi_backend.moduledef import FFI_DEFAULT_ABI from pypy.module._cffi_backend import (ctypeobj, ctypeprim, ctypeptr, ctypearray, ctypestruct, ctypevoid, ctypeenum) @@ -633,7 +633,7 @@ @unwrap_spec(w_fresult=ctypeobj.W_CType, ellipsis=int, abi=int) def new_function_type(space, w_fargs, w_fresult, ellipsis=0, - abi=_cffi_backend.FFI_DEFAULT_ABI): + abi=FFI_DEFAULT_ABI): fargs = [] for w_farg in space.fixedview(w_fargs): if not isinstance(w_farg, ctypeobj.W_CType): diff --git a/pypy/module/_cffi_backend/test/test_c.py b/pypy/module/_cffi_backend/test/test_c.py --- a/pypy/module/_cffi_backend/test/test_c.py +++ b/pypy/module/_cffi_backend/test/test_c.py @@ -17,12 +17,9 @@ """ import py, sys, ctypes -if sys.version_info < (2, 6): - py.test.skip("requires the b'' literal syntax") - from rpython.tool.udir import udir from pypy.interpreter import gateway -from pypy.module._cffi_backend import Module +from pypy.module._cffi_backend.moduledef import Module from pypy.module._cffi_backend.newtype import _clean_cache, UniqueCache from rpython.translator import cdir from rpython.translator.platform import host diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -34,7 +34,6 @@ from rpython.rlib.rposix import FdValidator from rpython.rlib.unroll import unrolling_iterable from rpython.rlib.objectmodel import specialize -from pypy.module import exceptions from pypy.module.exceptions import interp_exceptions from rpython.tool.sourcetools import func_with_new_name from rpython.rtyper.lltypesystem.lloperation import llop @@ -686,8 +685,9 @@ # PyExc_AttributeError, PyExc_OverflowError, PyExc_ImportError, # PyExc_NameError, PyExc_MemoryError, PyExc_RuntimeError, # PyExc_UnicodeEncodeError, PyExc_UnicodeDecodeError, ... + from pypy.module.exceptions.moduledef import Module global all_exceptions - all_exceptions = list(exceptions.Module.interpleveldefs) + all_exceptions = list(Module.interpleveldefs) for exc_name in all_exceptions: if exc_name in ('EnvironmentError', 'IOError', 'WindowsError'): # FIXME: aliases of OSError cause a clash of names via diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -3,7 +3,7 @@ from rpython.rlib.runicode import unicode_encode_latin_1, unicode_encode_utf_16_helper from rpython.rlib.rarithmetic import widen -from pypy.module.unicodedata import unicodedb +from pypy.objspace.std.unicodeobject import unicodedb from pypy.module.cpyext.api import ( CANNOT_FAIL, Py_ssize_t, build_type_checkers, cpython_api, bootstrap_function, CONST_STRING, diff --git a/pypy/module/micronumpy/test/conftest.py b/pypy/module/micronumpy/test/conftest.py --- a/pypy/module/micronumpy/test/conftest.py +++ b/pypy/module/micronumpy/test/conftest.py @@ -1,5 +1,2 @@ -import py - -def pytest_collect_directory(path, parent): - py.test.skip("micronumpy tests skipped for now on py3.5") -pytest_collect_file = pytest_collect_directory +def pytest_ignore_collect(): + return True diff --git a/pypy/module/token/interp_token.py b/pypy/module/token/interp_token.py new file mode 100644 --- /dev/null +++ b/pypy/module/token/interp_token.py @@ -0,0 +1,14 @@ +from pypy.interpreter.gateway import unwrap_spec +from pypy.interpreter.pyparser import pygram + + at unwrap_spec(tok=int) +def isterminal(space, tok): + return space.newbool(tok < 256) + + at unwrap_spec(tok=int) +def isnonterminal(space, tok): + return space.newbool(tok >= 256) + + at unwrap_spec(tok=int) +def iseof(space, tok): + return space.newbool(tok == pygram.tokens.ENDMARKER) diff --git a/pypy/module/token/moduledef.py b/pypy/module/token/moduledef.py --- a/pypy/module/token/moduledef.py +++ b/pypy/module/token/moduledef.py @@ -1,6 +1,5 @@ from pypy.interpreter.mixedmodule import MixedModule -from pypy.interpreter.gateway import unwrap_spec -from pypy.interpreter.pyparser import pytoken, pygram +from pypy.interpreter.pyparser import pytoken class Module(MixedModule): @@ -8,9 +7,9 @@ appleveldefs = {} interpleveldefs = { "NT_OFFSET" : "space.newint(256)", - "ISTERMINAL" : "__init__.isterminal", - "ISNONTERMINAL" : "__init__.isnonterminal", - "ISEOF" : "__init__.iseof" + "ISTERMINAL" : "interp_token.isterminal", + "ISNONTERMINAL" : "interp_token.isnonterminal", + "ISEOF" : "interp_token.iseof" } @@ -30,16 +29,3 @@ Module.interpleveldefs["__all__"] = "space.wrap(%r)" % (all_names,) _init_tokens() - - - at unwrap_spec(tok=int) -def isterminal(space, tok): - return space.newbool(tok < 256) - - at unwrap_spec(tok=int) -def isnonterminal(space, tok): - return space.newbool(tok >= 256) - - at unwrap_spec(tok=int) -def iseof(space, tok): - return space.newbool(tok == pygram.tokens.ENDMARKER) From pypy.commits at gmail.com Tue Apr 17 21:13:49 2018 From: pypy.commits at gmail.com (rlamy) Date: Tue, 17 Apr 2018 18:13:49 -0700 (PDT) Subject: [pypy-commit] pypy py3tests: fix _cffi_backend Message-ID: <5ad69bcd.cb8e1c0a.18cde.2727@mx.google.com> Author: Ronan Lamy Branch: py3tests Changeset: r94367:9b98fe93055e Date: 2018-04-18 02:12 +0100 http://bitbucket.org/pypy/pypy/changeset/9b98fe93055e/ Log: fix _cffi_backend diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -0,0 +1,1 @@ +VERSION = "1.11.5" diff --git a/pypy/module/_cffi_backend/moduledef.py b/pypy/module/_cffi_backend/moduledef.py --- a/pypy/module/_cffi_backend/moduledef.py +++ b/pypy/module/_cffi_backend/moduledef.py @@ -2,8 +2,7 @@ from pypy.interpreter.mixedmodule import MixedModule from rpython.rlib import rdynload, clibffi from rpython.rtyper.lltypesystem import rffi - -VERSION = "1.11.5" +from pypy.module._cffi_backend import VERSION FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI try: diff --git a/pypy/module/_cffi_backend/realize_c_type.py b/pypy/module/_cffi_backend/realize_c_type.py --- a/pypy/module/_cffi_backend/realize_c_type.py +++ b/pypy/module/_cffi_backend/realize_c_type.py @@ -5,7 +5,8 @@ from rpython.rtyper.lltypesystem import lltype, rffi from pypy.interpreter.error import oefmt from pypy.interpreter.baseobjspace import W_Root -from pypy.module import _cffi_backend +from pypy.module._cffi_backend.moduledef import ( + FFI_DEFAULT_ABI, has_stdcall, FFI_STDCALL) from pypy.module._cffi_backend.ctypeobj import W_CType from pypy.module._cffi_backend import cffi_opcode, newtype, ctypestruct from pypy.module._cffi_backend import ctypeprim @@ -182,12 +183,12 @@ ellipsis = (getarg(opcodes[base_index + num_args]) & 0x01) != 0 abi = (getarg(opcodes[base_index + num_args]) & 0xFE) if abi == 0: - abi = _cffi_backend.FFI_DEFAULT_ABI + abi = FFI_DEFAULT_ABI elif abi == 2: - if _cffi_backend.has_stdcall: - abi = _cffi_backend.FFI_STDCALL + if has_stdcall: + abi = FFI_STDCALL else: - abi = _cffi_backend.FFI_DEFAULT_ABI + abi = FFI_DEFAULT_ABI else: raise oefmt(ffi.w_FFIError, "abi number %d not supported", abi) # From pypy.commits at gmail.com Tue Apr 17 21:16:07 2018 From: pypy.commits at gmail.com (rlamy) Date: Tue, 17 Apr 2018 18:16:07 -0700 (PDT) Subject: [pypy-commit] pypy py3tests: fix checkmodule Message-ID: <5ad69c57.1c69fb81.e30c.01b6@mx.google.com> Author: Ronan Lamy Branch: py3tests Changeset: r94368:1c10c2181fc3 Date: 2018-04-18 02:15 +0100 http://bitbucket.org/pypy/pypy/changeset/1c10c2181fc3/ Log: fix checkmodule diff --git a/pypy/objspace/fake/checkmodule.py b/pypy/objspace/fake/checkmodule.py --- a/pypy/objspace/fake/checkmodule.py +++ b/pypy/objspace/fake/checkmodule.py @@ -10,7 +10,8 @@ seeobj_w = [] modules = [] for modname in modnames: - mod = __import__('pypy.module.%s' % modname, None, None, ['__doc__']) + mod = __import__( + 'pypy.module.%s.moduledef' % modname, None, None, ['__doc__']) # force computation and record what we wrap module = mod.Module(space, W_Root()) module.setup_after_space_initialization() From pypy.commits at gmail.com Tue Apr 17 21:21:51 2018 From: pypy.commits at gmail.com (rlamy) Date: Tue, 17 Apr 2018 18:21:51 -0700 (PDT) Subject: [pypy-commit] pypy py3tests: fix cmath Message-ID: <5ad69daf.1c69fb81.321ef.0ed7@mx.google.com> Author: Ronan Lamy Branch: py3tests Changeset: r94369:e388019b5e34 Date: 2018-04-18 02:21 +0100 http://bitbucket.org/pypy/pypy/changeset/e388019b5e34/ Log: fix cmath diff --git a/pypy/module/cmath/interp_cmath.py b/pypy/module/cmath/interp_cmath.py --- a/pypy/module/cmath/interp_cmath.py +++ b/pypy/module/cmath/interp_cmath.py @@ -3,7 +3,7 @@ from rpython.tool.sourcetools import func_with_new_name from pypy.interpreter.error import oefmt from pypy.interpreter.gateway import unwrap_spec -from pypy.module.cmath import names_and_docstrings +from pypy.module.cmath.moduledef import names_and_docstrings from rpython.rlib import rcomplex, rfloat pi = math.pi From pypy.commits at gmail.com Wed Apr 18 05:03:50 2018 From: pypy.commits at gmail.com (antocuni) Date: Wed, 18 Apr 2018 02:03:50 -0700 (PDT) Subject: [pypy-commit] pypy gc-hook-better-timestamp: bad me, no cookie; test_debug_print_start_stop_nonconst was broken, and translation as well Message-ID: <5ad709f6.48e8500a.492ea.286b@mx.google.com> Author: Antonio Cuni Branch: gc-hook-better-timestamp Changeset: r94373:de012a2f02b6 Date: 2018-04-18 11:03 +0200 http://bitbucket.org/pypy/pypy/changeset/de012a2f02b6/ Log: bad me, no cookie; test_debug_print_start_stop_nonconst was broken, and translation as well diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py --- a/rpython/translator/c/funcgen.py +++ b/rpython/translator/c/funcgen.py @@ -822,7 +822,7 @@ string_literal, self.expr(v_timestamp)) else: - x = "%s = %s(RPyString_AsCharP(%s, %s));\n" % (self.expr(op.result), + x = "%s = %s(RPyString_AsCharP(%s), %s);\n" % (self.expr(op.result), macro, self.expr(v_cat), self.expr(v_timestamp)) From pypy.commits at gmail.com Wed Apr 18 05:43:42 2018 From: pypy.commits at gmail.com (antocuni) Date: Wed, 18 Apr 2018 02:43:42 -0700 (PDT) Subject: [pypy-commit] pypy gc-hook-better-timestamp: expose the new debug_{start, stop} feature also to applevel, just because :) Message-ID: <5ad7134e.1c69fb81.bcf57.4136@mx.google.com> Author: Antonio Cuni Branch: gc-hook-better-timestamp Changeset: r94374:5e1e87b1839e Date: 2018-04-18 11:42 +0200 http://bitbucket.org/pypy/pypy/changeset/5e1e87b1839e/ Log: expose the new debug_{start,stop} feature also to applevel, just because :) diff --git a/pypy/module/__pypy__/interp_debug.py b/pypy/module/__pypy__/interp_debug.py --- a/pypy/module/__pypy__/interp_debug.py +++ b/pypy/module/__pypy__/interp_debug.py @@ -3,9 +3,12 @@ from rpython.rlib import rtimer @jit.dont_look_inside - at unwrap_spec(category='text') -def debug_start(space, category): - debug.debug_start(category) + at unwrap_spec(category='text', timestamp=bool) +def debug_start(space, category, timestamp=False): + res = debug.debug_start(category, timestamp=timestamp) + if timestamp: + return space.newint(res) + return space.w_None @jit.dont_look_inside def debug_print(space, args_w): @@ -13,10 +16,12 @@ debug.debug_print(' '.join(parts)) @jit.dont_look_inside - at unwrap_spec(category='text') -def debug_stop(space, category): - debug.debug_stop(category) - + at unwrap_spec(category='text', timestamp=bool) +def debug_stop(space, category, timestamp=False): + res = debug.debug_stop(category, timestamp=timestamp) + if timestamp: + return space.newint(res) + return space.w_None @unwrap_spec(category='text') def debug_print_once(space, category, args_w): diff --git a/pypy/module/__pypy__/test/test_debug.py b/pypy/module/__pypy__/test/test_debug.py --- a/pypy/module/__pypy__/test/test_debug.py +++ b/pypy/module/__pypy__/test/test_debug.py @@ -59,3 +59,15 @@ from __pypy__ import debug_get_timestamp_unit unit = debug_get_timestamp_unit() assert unit in ('tsc', 'ns', 'QueryPerformanceCounter') + + def test_debug_start_stop_timestamp(self): + import time + from __pypy__ import debug_start, debug_stop, debug_read_timestamp + assert debug_start('foo') is None + assert debug_stop('foo') is None + ts1 = debug_start('foo', timestamp=True) + t = time.time() + while time.time() - t < 0.02: + pass + ts2 = debug_stop('foo', timestamp=True) + assert ts2 > ts1 From pypy.commits at gmail.com Wed Apr 18 06:07:59 2018 From: pypy.commits at gmail.com (antocuni) Date: Wed, 18 Apr 2018 03:07:59 -0700 (PDT) Subject: [pypy-commit] pypy gc-hook-better-timestamp: add a way to know the fastest and the slowest gc-minor/gc-collect-step when we deliver many of them Message-ID: <5ad718ff.8bb71c0a.d3d49.8d2a@mx.google.com> Author: Antonio Cuni Branch: gc-hook-better-timestamp Changeset: r94375:f371797f2f7b Date: 2018-04-18 12:07 +0200 http://bitbucket.org/pypy/pypy/changeset/f371797f2f7b/ Log: add a way to know the fastest and the slowest gc-minor/gc-collect- step when we deliver many of them diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst --- a/pypy/doc/gc_info.rst +++ b/pypy/doc/gc_info.rst @@ -169,6 +169,12 @@ The total time spent inside minor collections since the last hook call. See below for more information on the unit. +``duration_min`` + The duration of the fastest minor collection since the last hook call. + +``duration_max`` + The duration of the slowest minor collection since the last hook call. + ``total_memory_used`` The amount of memory used at the end of the minor collection, in bytes. This include the memory used in arenas (for GC-managed memory) and @@ -180,7 +186,7 @@ The attributes for ``GcCollectStepStats`` are: -``count``, ``duration`` +``count``, ``duration``, ``duration_min``, ``duration_max`` See above. ``oldstate``, ``newstate`` diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py --- a/pypy/module/gc/hook.py +++ b/pypy/module/gc/hook.py @@ -1,7 +1,7 @@ from rpython.memory.gc.hook import GcHooks from rpython.memory.gc import incminimark from rpython.rlib.nonconst import NonConstant -from rpython.rlib.rarithmetic import r_uint, r_longlong +from rpython.rlib.rarithmetic import r_uint, r_longlong, longlongmax from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.typedef import TypeDef, interp_attrproperty, GetSetProperty @@ -35,6 +35,8 @@ action = self.w_hooks.gc_minor action.count += 1 action.duration += duration + action.duration_min = min(action.duration_min, duration) + action.duration_max = max(action.duration_max, duration) action.total_memory_used = total_memory_used action.pinned_objects = pinned_objects action.fire() @@ -43,6 +45,8 @@ action = self.w_hooks.gc_collect_step action.count += 1 action.duration += duration + action.duration_min = min(action.duration_min, duration) + action.duration_max = max(action.duration_max, duration) action.oldstate = oldstate action.newstate = newstate action.fire() @@ -112,18 +116,19 @@ class GcMinorHookAction(AsyncAction): - count = 0 - duration = r_longlong(0) total_memory_used = 0 pinned_objects = 0 def __init__(self, space): AsyncAction.__init__(self, space) self.w_callable = space.w_None + self.reset() def reset(self): self.count = 0 self.duration = r_longlong(0) + self.duration_min = r_longlong(longlongmax) + self.duration_max = r_longlong(0) def fix_annotation(self): # the annotation of the class and its attributes must be completed @@ -132,6 +137,8 @@ if NonConstant(False): self.count = NonConstant(-42) self.duration = NonConstant(r_longlong(-42)) + self.duration_min = NonConstant(r_longlong(-42)) + self.duration_max = NonConstant(r_longlong(-42)) self.total_memory_used = NonConstant(r_uint(42)) self.pinned_objects = NonConstant(-42) self.fire() @@ -140,6 +147,8 @@ w_stats = W_GcMinorStats( self.count, self.duration, + self.duration_min, + self.duration_max, self.total_memory_used, self.pinned_objects) self.reset() @@ -147,18 +156,19 @@ class GcCollectStepHookAction(AsyncAction): - count = 0 - duration = r_longlong(0) oldstate = 0 newstate = 0 def __init__(self, space): AsyncAction.__init__(self, space) self.w_callable = space.w_None + self.reset() def reset(self): self.count = 0 self.duration = r_longlong(0) + self.duration_min = r_longlong(longlongmax) + self.duration_max = r_longlong(0) def fix_annotation(self): # the annotation of the class and its attributes must be completed @@ -167,6 +177,8 @@ if NonConstant(False): self.count = NonConstant(-42) self.duration = NonConstant(r_longlong(-42)) + self.duration_min = NonConstant(r_longlong(-42)) + self.duration_max = NonConstant(r_longlong(-42)) self.oldstate = NonConstant(-42) self.newstate = NonConstant(-42) self.fire() @@ -175,6 +187,8 @@ w_stats = W_GcCollectStepStats( self.count, self.duration, + self.duration_min, + self.duration_max, self.oldstate, self.newstate) self.reset() @@ -182,7 +196,6 @@ class GcCollectHookAction(AsyncAction): - count = 0 num_major_collects = 0 arenas_count_before = 0 arenas_count_after = 0 @@ -193,6 +206,7 @@ def __init__(self, space): AsyncAction.__init__(self, space) self.w_callable = space.w_None + self.reset() def reset(self): self.count = 0 @@ -225,18 +239,24 @@ class W_GcMinorStats(W_Root): - def __init__(self, count, duration, total_memory_used, pinned_objects): + def __init__(self, count, duration, duration_min, duration_max, + total_memory_used, pinned_objects): self.count = count self.duration = duration + self.duration_min = duration_min + self.duration_max = duration_max self.total_memory_used = total_memory_used self.pinned_objects = pinned_objects class W_GcCollectStepStats(W_Root): - def __init__(self, count, duration, oldstate, newstate): + def __init__(self, count, duration, duration_min, duration_max, + oldstate, newstate): self.count = count self.duration = duration + self.duration_min = duration_min + self.duration_max = duration_max self.oldstate = oldstate self.newstate = newstate @@ -286,6 +306,8 @@ **wrap_many_ints(W_GcMinorStats, ( "count", "duration", + "duration_min", + "duration_max", "total_memory_used", "pinned_objects")) ) @@ -300,6 +322,8 @@ **wrap_many_ints(W_GcCollectStepStats, ( "count", "duration", + "duration_min", + "duration_max", "oldstate", "newstate")) ) diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py --- a/pypy/module/gc/test/test_hook.py +++ b/pypy/module/gc/test/test_hook.py @@ -134,18 +134,20 @@ self.steps = [] def on_gc_minor(self, stats): - self.minors.append((stats.count, stats.duration)) + self.minors.append((stats.count, stats.duration, + stats.duration_min, stats.duration_max)) def on_gc_collect_step(self, stats): - self.steps.append((stats.count, stats.duration)) + self.steps.append((stats.count, stats.duration, + stats.duration_min, stats.duration_max)) on_gc_collect = None myhooks = MyHooks() gc.hooks.set(myhooks) self.fire_many() - assert myhooks.minors == [(2, 12)] - assert myhooks.steps == [(3, 42)] + assert myhooks.minors == [(2, 12, 5, 7)] + assert myhooks.steps == [(3, 42, 5, 22)] def test_clear_queue(self): import gc From pypy.commits at gmail.com Wed Apr 18 09:08:28 2018 From: pypy.commits at gmail.com (antocuni) Date: Wed, 18 Apr 2018 06:08:28 -0700 (PDT) Subject: [pypy-commit] pypy gc-hook-better-timestamp: fix test_whatsnew Message-ID: <5ad7434c.1c69fb81.144b8.bb52@mx.google.com> Author: Antonio Cuni Branch: gc-hook-better-timestamp Changeset: r94376:09308ea0ff83 Date: 2018-04-18 15:05 +0200 http://bitbucket.org/pypy/pypy/changeset/09308ea0ff83/ Log: fix test_whatsnew diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -14,3 +14,7 @@ .. branch: gc-hooks Introduce GC hooks, as documented in doc/gc_info.rst + +.. branch: gc-hook-better-timestamp + +Improve GC hooks From pypy.commits at gmail.com Wed Apr 18 09:08:30 2018 From: pypy.commits at gmail.com (antocuni) Date: Wed, 18 Apr 2018 06:08:30 -0700 (PDT) Subject: [pypy-commit] pypy gc-hook-better-timestamp: close merged branch Message-ID: <5ad7434e.82811c0a.b9dce.8670@mx.google.com> Author: Antonio Cuni Branch: gc-hook-better-timestamp Changeset: r94377:042fd84032e6 Date: 2018-04-18 15:05 +0200 http://bitbucket.org/pypy/pypy/changeset/042fd84032e6/ Log: close merged branch From pypy.commits at gmail.com Wed Apr 18 09:08:32 2018 From: pypy.commits at gmail.com (antocuni) Date: Wed, 18 Apr 2018 06:08:32 -0700 (PDT) Subject: [pypy-commit] pypy default: Merge the gc-hook-better-timestamp branch. It improves gc hooks in two ways: Message-ID: <5ad74350.1c69fb81.15b6f.6c00@mx.google.com> Author: Antonio Cuni Branch: Changeset: r94378:e7b972c672e6 Date: 2018-04-18 15:07 +0200 http://bitbucket.org/pypy/pypy/changeset/e7b972c672e6/ Log: Merge the gc-hook-better-timestamp branch. It improves gc hooks in two ways: - the duration field is computed using the very same timestamps which are used to generate PYPYLOG, so that it is easier to correlate the two, if needed - the stats now report duration_min and duration_max, to give an idea of how much the fastest and slowest event took diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst --- a/pypy/doc/gc_info.rst +++ b/pypy/doc/gc_info.rst @@ -169,6 +169,12 @@ The total time spent inside minor collections since the last hook call. See below for more information on the unit. +``duration_min`` + The duration of the fastest minor collection since the last hook call. + +``duration_max`` + The duration of the slowest minor collection since the last hook call. + ``total_memory_used`` The amount of memory used at the end of the minor collection, in bytes. This include the memory used in arenas (for GC-managed memory) and @@ -180,7 +186,7 @@ The attributes for ``GcCollectStepStats`` are: -``count``, ``duration`` +``count``, ``duration``, ``duration_min``, ``duration_max`` See above. ``oldstate``, ``newstate`` diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -14,3 +14,7 @@ .. branch: gc-hooks Introduce GC hooks, as documented in doc/gc_info.rst + +.. branch: gc-hook-better-timestamp + +Improve GC hooks diff --git a/pypy/module/__pypy__/interp_debug.py b/pypy/module/__pypy__/interp_debug.py --- a/pypy/module/__pypy__/interp_debug.py +++ b/pypy/module/__pypy__/interp_debug.py @@ -3,9 +3,12 @@ from rpython.rlib import rtimer @jit.dont_look_inside - at unwrap_spec(category='text') -def debug_start(space, category): - debug.debug_start(category) + at unwrap_spec(category='text', timestamp=bool) +def debug_start(space, category, timestamp=False): + res = debug.debug_start(category, timestamp=timestamp) + if timestamp: + return space.newint(res) + return space.w_None @jit.dont_look_inside def debug_print(space, args_w): @@ -13,10 +16,12 @@ debug.debug_print(' '.join(parts)) @jit.dont_look_inside - at unwrap_spec(category='text') -def debug_stop(space, category): - debug.debug_stop(category) - + at unwrap_spec(category='text', timestamp=bool) +def debug_stop(space, category, timestamp=False): + res = debug.debug_stop(category, timestamp=timestamp) + if timestamp: + return space.newint(res) + return space.w_None @unwrap_spec(category='text') def debug_print_once(space, category, args_w): diff --git a/pypy/module/__pypy__/test/test_debug.py b/pypy/module/__pypy__/test/test_debug.py --- a/pypy/module/__pypy__/test/test_debug.py +++ b/pypy/module/__pypy__/test/test_debug.py @@ -59,3 +59,15 @@ from __pypy__ import debug_get_timestamp_unit unit = debug_get_timestamp_unit() assert unit in ('tsc', 'ns', 'QueryPerformanceCounter') + + def test_debug_start_stop_timestamp(self): + import time + from __pypy__ import debug_start, debug_stop, debug_read_timestamp + assert debug_start('foo') is None + assert debug_stop('foo') is None + ts1 = debug_start('foo', timestamp=True) + t = time.time() + while time.time() - t < 0.02: + pass + ts2 = debug_stop('foo', timestamp=True) + assert ts2 > ts1 diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py --- a/pypy/module/gc/hook.py +++ b/pypy/module/gc/hook.py @@ -1,7 +1,7 @@ from rpython.memory.gc.hook import GcHooks from rpython.memory.gc import incminimark from rpython.rlib.nonconst import NonConstant -from rpython.rlib.rarithmetic import r_uint, r_longlong +from rpython.rlib.rarithmetic import r_uint, r_longlong, longlongmax from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.typedef import TypeDef, interp_attrproperty, GetSetProperty @@ -35,6 +35,8 @@ action = self.w_hooks.gc_minor action.count += 1 action.duration += duration + action.duration_min = min(action.duration_min, duration) + action.duration_max = max(action.duration_max, duration) action.total_memory_used = total_memory_used action.pinned_objects = pinned_objects action.fire() @@ -43,6 +45,8 @@ action = self.w_hooks.gc_collect_step action.count += 1 action.duration += duration + action.duration_min = min(action.duration_min, duration) + action.duration_max = max(action.duration_max, duration) action.oldstate = oldstate action.newstate = newstate action.fire() @@ -112,18 +116,19 @@ class GcMinorHookAction(AsyncAction): - count = 0 - duration = r_longlong(0) total_memory_used = 0 pinned_objects = 0 def __init__(self, space): AsyncAction.__init__(self, space) self.w_callable = space.w_None + self.reset() def reset(self): self.count = 0 self.duration = r_longlong(0) + self.duration_min = r_longlong(longlongmax) + self.duration_max = r_longlong(0) def fix_annotation(self): # the annotation of the class and its attributes must be completed @@ -132,6 +137,8 @@ if NonConstant(False): self.count = NonConstant(-42) self.duration = NonConstant(r_longlong(-42)) + self.duration_min = NonConstant(r_longlong(-42)) + self.duration_max = NonConstant(r_longlong(-42)) self.total_memory_used = NonConstant(r_uint(42)) self.pinned_objects = NonConstant(-42) self.fire() @@ -140,6 +147,8 @@ w_stats = W_GcMinorStats( self.count, self.duration, + self.duration_min, + self.duration_max, self.total_memory_used, self.pinned_objects) self.reset() @@ -147,18 +156,19 @@ class GcCollectStepHookAction(AsyncAction): - count = 0 - duration = r_longlong(0) oldstate = 0 newstate = 0 def __init__(self, space): AsyncAction.__init__(self, space) self.w_callable = space.w_None + self.reset() def reset(self): self.count = 0 self.duration = r_longlong(0) + self.duration_min = r_longlong(longlongmax) + self.duration_max = r_longlong(0) def fix_annotation(self): # the annotation of the class and its attributes must be completed @@ -167,6 +177,8 @@ if NonConstant(False): self.count = NonConstant(-42) self.duration = NonConstant(r_longlong(-42)) + self.duration_min = NonConstant(r_longlong(-42)) + self.duration_max = NonConstant(r_longlong(-42)) self.oldstate = NonConstant(-42) self.newstate = NonConstant(-42) self.fire() @@ -175,6 +187,8 @@ w_stats = W_GcCollectStepStats( self.count, self.duration, + self.duration_min, + self.duration_max, self.oldstate, self.newstate) self.reset() @@ -182,7 +196,6 @@ class GcCollectHookAction(AsyncAction): - count = 0 num_major_collects = 0 arenas_count_before = 0 arenas_count_after = 0 @@ -193,6 +206,7 @@ def __init__(self, space): AsyncAction.__init__(self, space) self.w_callable = space.w_None + self.reset() def reset(self): self.count = 0 @@ -225,18 +239,24 @@ class W_GcMinorStats(W_Root): - def __init__(self, count, duration, total_memory_used, pinned_objects): + def __init__(self, count, duration, duration_min, duration_max, + total_memory_used, pinned_objects): self.count = count self.duration = duration + self.duration_min = duration_min + self.duration_max = duration_max self.total_memory_used = total_memory_used self.pinned_objects = pinned_objects class W_GcCollectStepStats(W_Root): - def __init__(self, count, duration, oldstate, newstate): + def __init__(self, count, duration, duration_min, duration_max, + oldstate, newstate): self.count = count self.duration = duration + self.duration_min = duration_min + self.duration_max = duration_max self.oldstate = oldstate self.newstate = newstate @@ -286,6 +306,8 @@ **wrap_many_ints(W_GcMinorStats, ( "count", "duration", + "duration_min", + "duration_max", "total_memory_used", "pinned_objects")) ) @@ -300,6 +322,8 @@ **wrap_many_ints(W_GcCollectStepStats, ( "count", "duration", + "duration_min", + "duration_max", "oldstate", "newstate")) ) diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py --- a/pypy/module/gc/test/test_hook.py +++ b/pypy/module/gc/test/test_hook.py @@ -134,18 +134,20 @@ self.steps = [] def on_gc_minor(self, stats): - self.minors.append((stats.count, stats.duration)) + self.minors.append((stats.count, stats.duration, + stats.duration_min, stats.duration_max)) def on_gc_collect_step(self, stats): - self.steps.append((stats.count, stats.duration)) + self.steps.append((stats.count, stats.duration, + stats.duration_min, stats.duration_max)) on_gc_collect = None myhooks = MyHooks() gc.hooks.set(myhooks) self.fire_many() - assert myhooks.minors == [(2, 12)] - assert myhooks.steps == [(3, 42)] + assert myhooks.minors == [(2, 12, 5, 7)] + assert myhooks.steps == [(3, 42, 5, 22)] def test_clear_queue(self): import gc diff --git a/rpython/rlib/debug.py b/rpython/rlib/debug.py --- a/rpython/rlib/debug.py +++ b/rpython/rlib/debug.py @@ -2,7 +2,7 @@ import time from rpython.rtyper.extregistry import ExtRegistryEntry -from rpython.rlib.objectmodel import we_are_translated +from rpython.rlib.objectmodel import we_are_translated, always_inline from rpython.rlib.rarithmetic import is_valid_int from rpython.rtyper.extfunc import register_external from rpython.rtyper.lltypesystem import lltype @@ -74,34 +74,66 @@ _start_colors_2 = "" _stop_colors = "" -def debug_start(category): + at always_inline +def debug_start(category, timestamp=False): + """ + Start a PYPYLOG section. + + By default, the return value is undefined. If timestamp is True, always + return the current timestamp, even if PYPYLOG is not set. + """ + return _debug_start(category, timestamp) + + at always_inline +def debug_stop(category, timestamp=False): + """ + Stop a PYPYLOG section. See debug_start for docs about timestamp + """ + return _debug_stop(category, timestamp) + + +def _debug_start(category, timestamp): c = int(time.clock() * 100) print >> sys.stderr, '%s[%x] {%s%s' % (_start_colors_1, c, category, _stop_colors) if _log is not None: _log.debug_start(category) -def debug_stop(category): + if timestamp: + return c + return -42 # random undefined value + +def _debug_stop(category, timestamp): c = int(time.clock() * 100) print >> sys.stderr, '%s[%x] %s}%s' % (_start_colors_2, c, category, _stop_colors) if _log is not None: _log.debug_stop(category) + if timestamp: + return c + return -42 # random undefined value + class Entry(ExtRegistryEntry): - _about_ = debug_start, debug_stop + _about_ = _debug_start, _debug_stop - def compute_result_annotation(self, s_category): - return None + def compute_result_annotation(self, s_category, s_timestamp): + from rpython.rlib.rtimer import s_TIMESTAMP + return s_TIMESTAMP def specialize_call(self, hop): from rpython.rtyper.lltypesystem.rstr import string_repr + from rpython.rlib.rtimer import TIMESTAMP_type fn = self.instance - vlist = hop.inputargs(string_repr) + _, r_timestamp = hop.args_r + vlist = hop.inputargs(string_repr, r_timestamp) hop.exception_cannot_occur() t = hop.rtyper.annotator.translator if t.config.translation.log: - hop.genop(fn.__name__, vlist) + opname = fn.__name__[1:] # remove the '_' + return hop.genop(opname, vlist, resulttype=TIMESTAMP_type) + else: + return hop.inputconst(TIMESTAMP_type, 0) def have_debug_prints(): diff --git a/rpython/rlib/rtimer.py b/rpython/rlib/rtimer.py --- a/rpython/rlib/rtimer.py +++ b/rpython/rlib/rtimer.py @@ -7,6 +7,15 @@ _is_64_bit = r_uint.BITS > 32 +from rpython.annotator.model import SomeInteger +if _is_64_bit: + s_TIMESTAMP = SomeInteger() + TIMESTAMP_type = lltype.Signed +else: + s_TIMESTAMP = SomeInteger(knowntype=r_longlong) + TIMESTAMP_type = rffi.LONGLONG + + # unit of values returned by read_timestamp. Should be in sync with the ones # defined in translator/c/debug_print.h UNIT_TSC = 0 @@ -32,19 +41,11 @@ _about_ = read_timestamp def compute_result_annotation(self): - from rpython.annotator.model import SomeInteger - if _is_64_bit: - return SomeInteger() - else: - return SomeInteger(knowntype=r_longlong) + return s_TIMESTAMP def specialize_call(self, hop): hop.exception_cannot_occur() - if _is_64_bit: - resulttype = lltype.Signed - else: - resulttype = rffi.LONGLONG - return hop.genop("ll_read_timestamp", [], resulttype=resulttype) + return hop.genop("ll_read_timestamp", [], resulttype=TIMESTAMP_type) class ReadTimestampEntry(ExtRegistryEntry): diff --git a/rpython/rlib/test/test_debug.py b/rpython/rlib/test/test_debug.py --- a/rpython/rlib/test/test_debug.py +++ b/rpython/rlib/test/test_debug.py @@ -120,6 +120,26 @@ assert dlog == [("mycat", [('debug_print', 'foo', 2, 'bar', 3)])] +def test_debug_start_stop_timestamp(): + import time + def f(timestamp): + ts_a = debug_start('foo', timestamp=timestamp) + # simulate some CPU time + t = time.time() + while time.time()-t < 0.02: + pass + ts_b = debug_stop('foo', timestamp=timestamp) + return ts_b - ts_a + + assert f(False) == 0 + assert f(True) > 0 + # + res = interpret(f, [False]) + assert res == 0 + res = interpret(f, [True]) + assert res > 0 + + def test_debug_print_traceback(): from rpython.translator.c.test.test_genc import compile from rpython.rtyper.lltypesystem import lltype diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py --- a/rpython/rtyper/lltypesystem/opimpl.py +++ b/rpython/rtyper/lltypesystem/opimpl.py @@ -586,11 +586,11 @@ def op_debug_print(*args): debug.debug_print(*map(_normalize, args)) -def op_debug_start(category): - debug.debug_start(_normalize(category)) +def op_debug_start(category, timestamp): + return debug.debug_start(_normalize(category), timestamp) -def op_debug_stop(category): - debug.debug_stop(_normalize(category)) +def op_debug_stop(category, timestamp): + return debug.debug_stop(_normalize(category), timestamp) def op_debug_offset(): return debug.debug_offset() diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py --- a/rpython/translator/c/funcgen.py +++ b/rpython/translator/c/funcgen.py @@ -813,20 +813,27 @@ "if (PYPY_HAVE_DEBUG_PRINTS) { fprintf(PYPY_DEBUG_FILE, %s); %s}" % (', '.join(argv), free_line)) - def _op_debug(self, opname, arg): - if isinstance(arg, Constant): - string_literal = c_string_constant(''.join(arg.value.chars)) - return "%s(%s);" % (opname, string_literal) + def _op_debug(self, macro, op): + v_cat, v_timestamp = op.args + if isinstance(v_cat, Constant): + string_literal = c_string_constant(''.join(v_cat.value.chars)) + return "%s = %s(%s, %s);" % (self.expr(op.result), + macro, + string_literal, + self.expr(v_timestamp)) else: - x = "%s(RPyString_AsCharP(%s));\n" % (opname, self.expr(arg)) + x = "%s = %s(RPyString_AsCharP(%s), %s);\n" % (self.expr(op.result), + macro, + self.expr(v_cat), + self.expr(v_timestamp)) x += "RPyString_FreeCache();" return x def OP_DEBUG_START(self, op): - return self._op_debug('PYPY_DEBUG_START', op.args[0]) + return self._op_debug('PYPY_DEBUG_START', op) def OP_DEBUG_STOP(self, op): - return self._op_debug('PYPY_DEBUG_STOP', op.args[0]) + return self._op_debug('PYPY_DEBUG_STOP', op) def OP_HAVE_DEBUG_PRINTS_FOR(self, op): arg = op.args[0] diff --git a/rpython/translator/c/src/debug_print.c b/rpython/translator/c/src/debug_print.c --- a/rpython/translator/c/src/debug_print.c +++ b/rpython/translator/c/src/debug_print.c @@ -199,8 +199,8 @@ #define PYPY_LONG_LONG_PRINTF_FORMAT "ll" #endif -static void display_startstop(const char *prefix, const char *postfix, - const char *category, const char *colors) +static long long display_startstop(const char *prefix, const char *postfix, + const char *category, const char *colors) { long long timestamp; READ_TIMESTAMP(timestamp); @@ -208,10 +208,12 @@ colors, timestamp, prefix, category, postfix, debug_stop_colors); + return timestamp; } -void pypy_debug_start(const char *category) +long long pypy_debug_start(const char *category, long timestamp) { + long long result = 42; pypy_debug_ensure_opened(); /* Enter a nesting level. Nested debug_prints are disabled by default because the following left shift introduces a 0 in the last bit. @@ -224,22 +226,30 @@ if (!debug_prefix || !startswithoneof(category, debug_prefix)) { /* wrong section name, or no PYPYLOG at all, skip it */ - return; + if (timestamp) + READ_TIMESTAMP(result); + return result; } /* else make this subsection active */ pypy_have_debug_prints |= 1; } - display_startstop("{", "", category, debug_start_colors_1); + return display_startstop("{", "", category, debug_start_colors_1); } -void pypy_debug_stop(const char *category) +long long pypy_debug_stop(const char *category, long timestamp) { + long long result = 42; if (debug_profile | (pypy_have_debug_prints & 1)) { - display_startstop("", "}", category, debug_start_colors_2); + result = display_startstop("", "}", category, debug_start_colors_2); fflush(pypy_debug_file); } + else if (timestamp) + { + READ_TIMESTAMP(result); + } pypy_have_debug_prints >>= 1; + return result; } long pypy_have_debug_prints_for(const char *category_prefix) diff --git a/rpython/translator/c/src/debug_print.h b/rpython/translator/c/src/debug_print.h --- a/rpython/translator/c/src/debug_print.h +++ b/rpython/translator/c/src/debug_print.h @@ -31,8 +31,8 @@ #define PYPY_HAVE_DEBUG_PRINTS (pypy_have_debug_prints & 1 ? \ (pypy_debug_ensure_opened(), 1) : 0) #define PYPY_DEBUG_FILE pypy_debug_file -#define PYPY_DEBUG_START(cat) pypy_debug_start(cat) -#define PYPY_DEBUG_STOP(cat) pypy_debug_stop(cat) +#define PYPY_DEBUG_START(cat, ts) pypy_debug_start(cat, ts) +#define PYPY_DEBUG_STOP(cat, ts) pypy_debug_stop(cat, ts) #define OP_DEBUG_OFFSET(res) res = pypy_debug_offset() #define OP_DEBUG_FORKED(ofs, _) pypy_debug_forked(ofs) #define OP_HAVE_DEBUG_PRINTS(r) r = (pypy_have_debug_prints & 1) @@ -42,8 +42,8 @@ /* prototypes (internal use only) */ RPY_EXTERN void pypy_debug_ensure_opened(void); -RPY_EXTERN void pypy_debug_start(const char *category); -RPY_EXTERN void pypy_debug_stop(const char *category); +RPY_EXTERN long long pypy_debug_start(const char *category, long timestamp); +RPY_EXTERN long long pypy_debug_stop(const char *category, long timestamp); RPY_EXTERN long pypy_debug_offset(void); RPY_EXTERN void pypy_debug_forked(long original_offset); RPY_EXTERN long pypy_have_debug_prints_for(const char *category_prefix); diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py --- a/rpython/translator/c/test/test_standalone.py +++ b/rpython/translator/c/test/test_standalone.py @@ -1,5 +1,6 @@ import py import sys, os, re +import textwrap from rpython.config.translationoption import get_combined_translation_config from rpython.config.translationoption import SUPPORT__THREAD @@ -519,6 +520,52 @@ assert not err assert path.check(file=0) + def test_debug_start_stop_timestamp(self): + import sys + import time + from rpython.rlib.rtimer import read_timestamp + def entry_point(argv): + timestamp = int(argv[1]) + ts1 = debug_start("foo", timestamp=timestamp) + ts2 = read_timestamp() + ts3 = debug_stop("foo", timestamp=timestamp) + print ts1 + print ts2 + print ts3 + return 0 + t, cbuilder = self.compile(entry_point) + + def parse_out(out): + lines = out.strip().splitlines() + ts1, ts2, ts3 = lines + return int(ts1), int(ts2), int(ts3) + + # check with PYPYLOG :- + out, err = cbuilder.cmdexec("1", err=True, env={'PYPYLOG': ':-'}) + ts1, ts2, ts3 = parse_out(out) + assert ts3 > ts2 > ts1 + expected = ('[%x] {foo\n' % ts1 + + '[%x] foo}\n' % ts3) + assert err == expected + + # check with PYPYLOG profiling only + out, err = cbuilder.cmdexec("1", err=True, env={'PYPYLOG': '-'}) + ts1, ts2, ts3 = parse_out(out) + assert ts3 > ts2 > ts1 + expected = ('[%x] {foo\n' % ts1 + + '[%x] foo}\n' % ts3) + assert err == expected + + # check with PYPYLOG undefined + out, err = cbuilder.cmdexec("1", err=True, env={}) + ts1, ts2, ts3 = parse_out(out) + assert ts3 > ts2 > ts1 + + # check with PYPYLOG undefined and timestamp=False + out, err = cbuilder.cmdexec("0", err=True, env={}) + ts1, ts2, ts3 = parse_out(out) + assert ts1 == ts3 == 42; + def test_debug_print_start_stop_nonconst(self): def entry_point(argv): debug_start(argv[1]) From pypy.commits at gmail.com Wed Apr 18 09:28:26 2018 From: pypy.commits at gmail.com (antocuni) Date: Wed, 18 Apr 2018 06:28:26 -0700 (PDT) Subject: [pypy-commit] pypy default: fix llinterp debug_{start, stop} on 32bit Message-ID: <5ad747fa.d15d1c0a.a81c0.2034@mx.google.com> Author: Antonio Cuni Branch: Changeset: r94379:fbffeb3dc383 Date: 2018-04-18 13:27 +0000 http://bitbucket.org/pypy/pypy/changeset/fbffeb3dc383/ Log: fix llinterp debug_{start,stop} on 32bit diff --git a/rpython/rlib/debug.py b/rpython/rlib/debug.py --- a/rpython/rlib/debug.py +++ b/rpython/rlib/debug.py @@ -3,7 +3,7 @@ from rpython.rtyper.extregistry import ExtRegistryEntry from rpython.rlib.objectmodel import we_are_translated, always_inline -from rpython.rlib.rarithmetic import is_valid_int +from rpython.rlib.rarithmetic import is_valid_int, r_longlong from rpython.rtyper.extfunc import register_external from rpython.rtyper.lltypesystem import lltype from rpython.rtyper.lltypesystem import rffi @@ -100,8 +100,8 @@ _log.debug_start(category) if timestamp: - return c - return -42 # random undefined value + return r_longlong(c) + return r_longlong(-42) # random undefined value def _debug_stop(category, timestamp): c = int(time.clock() * 100) @@ -111,8 +111,8 @@ _log.debug_stop(category) if timestamp: - return c - return -42 # random undefined value + return r_longlong(c) + return r_longlong(-42) # random undefined value class Entry(ExtRegistryEntry): _about_ = _debug_start, _debug_stop From pypy.commits at gmail.com Thu Apr 19 04:51:03 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 19 Apr 2018 01:51:03 -0700 (PDT) Subject: [pypy-commit] pypy default: fix llinterp debug_{start, stop} on 32bit - one more time Message-ID: <5ad85877.954a1c0a.88e82.a032@mx.google.com> Author: Matti Picus Branch: Changeset: r94380:51718fe91a11 Date: 2018-04-19 11:20 +0300 http://bitbucket.org/pypy/pypy/changeset/51718fe91a11/ Log: fix llinterp debug_{start,stop} on 32bit - one more time diff --git a/rpython/rlib/debug.py b/rpython/rlib/debug.py --- a/rpython/rlib/debug.py +++ b/rpython/rlib/debug.py @@ -133,7 +133,7 @@ opname = fn.__name__[1:] # remove the '_' return hop.genop(opname, vlist, resulttype=TIMESTAMP_type) else: - return hop.inputconst(TIMESTAMP_type, 0) + return hop.inputconst(TIMESTAMP_type, r_longlong(0)) def have_debug_prints(): From pypy.commits at gmail.com Thu Apr 19 04:51:05 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 19 Apr 2018 01:51:05 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: merge default into branch Message-ID: <5ad85879.06581c0a.6b03a.b19b@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r94381:127336ebfcfe Date: 2018-04-19 11:20 +0300 http://bitbucket.org/pypy/pypy/changeset/127336ebfcfe/ Log: merge default into branch diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst --- a/pypy/doc/gc_info.rst +++ b/pypy/doc/gc_info.rst @@ -169,6 +169,12 @@ The total time spent inside minor collections since the last hook call. See below for more information on the unit. +``duration_min`` + The duration of the fastest minor collection since the last hook call. + +``duration_max`` + The duration of the slowest minor collection since the last hook call. + ``total_memory_used`` The amount of memory used at the end of the minor collection, in bytes. This include the memory used in arenas (for GC-managed memory) and @@ -180,7 +186,7 @@ The attributes for ``GcCollectStepStats`` are: -``count``, ``duration`` +``count``, ``duration``, ``duration_min``, ``duration_max`` See above. ``oldstate``, ``newstate`` diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -14,3 +14,7 @@ .. branch: gc-hooks Introduce GC hooks, as documented in doc/gc_info.rst + +.. branch: gc-hook-better-timestamp + +Improve GC hooks diff --git a/pypy/module/__pypy__/interp_debug.py b/pypy/module/__pypy__/interp_debug.py --- a/pypy/module/__pypy__/interp_debug.py +++ b/pypy/module/__pypy__/interp_debug.py @@ -3,9 +3,12 @@ from rpython.rlib import rtimer @jit.dont_look_inside - at unwrap_spec(category='text') -def debug_start(space, category): - debug.debug_start(category) + at unwrap_spec(category='text', timestamp=bool) +def debug_start(space, category, timestamp=False): + res = debug.debug_start(category, timestamp=timestamp) + if timestamp: + return space.newint(res) + return space.w_None @jit.dont_look_inside def debug_print(space, args_w): @@ -13,10 +16,12 @@ debug.debug_print(' '.join(parts)) @jit.dont_look_inside - at unwrap_spec(category='text') -def debug_stop(space, category): - debug.debug_stop(category) - + at unwrap_spec(category='text', timestamp=bool) +def debug_stop(space, category, timestamp=False): + res = debug.debug_stop(category, timestamp=timestamp) + if timestamp: + return space.newint(res) + return space.w_None @unwrap_spec(category='text') def debug_print_once(space, category, args_w): diff --git a/pypy/module/__pypy__/test/test_debug.py b/pypy/module/__pypy__/test/test_debug.py --- a/pypy/module/__pypy__/test/test_debug.py +++ b/pypy/module/__pypy__/test/test_debug.py @@ -59,3 +59,15 @@ from __pypy__ import debug_get_timestamp_unit unit = debug_get_timestamp_unit() assert unit in ('tsc', 'ns', 'QueryPerformanceCounter') + + def test_debug_start_stop_timestamp(self): + import time + from __pypy__ import debug_start, debug_stop, debug_read_timestamp + assert debug_start('foo') is None + assert debug_stop('foo') is None + ts1 = debug_start('foo', timestamp=True) + t = time.time() + while time.time() - t < 0.02: + pass + ts2 = debug_stop('foo', timestamp=True) + assert ts2 > ts1 diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py --- a/pypy/module/gc/hook.py +++ b/pypy/module/gc/hook.py @@ -1,7 +1,7 @@ from rpython.memory.gc.hook import GcHooks from rpython.memory.gc import incminimark from rpython.rlib.nonconst import NonConstant -from rpython.rlib.rarithmetic import r_uint, r_longlong +from rpython.rlib.rarithmetic import r_uint, r_longlong, longlongmax from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.typedef import TypeDef, interp_attrproperty, GetSetProperty @@ -35,6 +35,8 @@ action = self.w_hooks.gc_minor action.count += 1 action.duration += duration + action.duration_min = min(action.duration_min, duration) + action.duration_max = max(action.duration_max, duration) action.total_memory_used = total_memory_used action.pinned_objects = pinned_objects action.fire() @@ -43,6 +45,8 @@ action = self.w_hooks.gc_collect_step action.count += 1 action.duration += duration + action.duration_min = min(action.duration_min, duration) + action.duration_max = max(action.duration_max, duration) action.oldstate = oldstate action.newstate = newstate action.fire() @@ -112,18 +116,19 @@ class GcMinorHookAction(AsyncAction): - count = 0 - duration = r_longlong(0) total_memory_used = 0 pinned_objects = 0 def __init__(self, space): AsyncAction.__init__(self, space) self.w_callable = space.w_None + self.reset() def reset(self): self.count = 0 self.duration = r_longlong(0) + self.duration_min = r_longlong(longlongmax) + self.duration_max = r_longlong(0) def fix_annotation(self): # the annotation of the class and its attributes must be completed @@ -132,6 +137,8 @@ if NonConstant(False): self.count = NonConstant(-42) self.duration = NonConstant(r_longlong(-42)) + self.duration_min = NonConstant(r_longlong(-42)) + self.duration_max = NonConstant(r_longlong(-42)) self.total_memory_used = NonConstant(r_uint(42)) self.pinned_objects = NonConstant(-42) self.fire() @@ -140,6 +147,8 @@ w_stats = W_GcMinorStats( self.count, self.duration, + self.duration_min, + self.duration_max, self.total_memory_used, self.pinned_objects) self.reset() @@ -147,18 +156,19 @@ class GcCollectStepHookAction(AsyncAction): - count = 0 - duration = r_longlong(0) oldstate = 0 newstate = 0 def __init__(self, space): AsyncAction.__init__(self, space) self.w_callable = space.w_None + self.reset() def reset(self): self.count = 0 self.duration = r_longlong(0) + self.duration_min = r_longlong(longlongmax) + self.duration_max = r_longlong(0) def fix_annotation(self): # the annotation of the class and its attributes must be completed @@ -167,6 +177,8 @@ if NonConstant(False): self.count = NonConstant(-42) self.duration = NonConstant(r_longlong(-42)) + self.duration_min = NonConstant(r_longlong(-42)) + self.duration_max = NonConstant(r_longlong(-42)) self.oldstate = NonConstant(-42) self.newstate = NonConstant(-42) self.fire() @@ -175,6 +187,8 @@ w_stats = W_GcCollectStepStats( self.count, self.duration, + self.duration_min, + self.duration_max, self.oldstate, self.newstate) self.reset() @@ -182,7 +196,6 @@ class GcCollectHookAction(AsyncAction): - count = 0 num_major_collects = 0 arenas_count_before = 0 arenas_count_after = 0 @@ -193,6 +206,7 @@ def __init__(self, space): AsyncAction.__init__(self, space) self.w_callable = space.w_None + self.reset() def reset(self): self.count = 0 @@ -225,18 +239,24 @@ class W_GcMinorStats(W_Root): - def __init__(self, count, duration, total_memory_used, pinned_objects): + def __init__(self, count, duration, duration_min, duration_max, + total_memory_used, pinned_objects): self.count = count self.duration = duration + self.duration_min = duration_min + self.duration_max = duration_max self.total_memory_used = total_memory_used self.pinned_objects = pinned_objects class W_GcCollectStepStats(W_Root): - def __init__(self, count, duration, oldstate, newstate): + def __init__(self, count, duration, duration_min, duration_max, + oldstate, newstate): self.count = count self.duration = duration + self.duration_min = duration_min + self.duration_max = duration_max self.oldstate = oldstate self.newstate = newstate @@ -286,6 +306,8 @@ **wrap_many_ints(W_GcMinorStats, ( "count", "duration", + "duration_min", + "duration_max", "total_memory_used", "pinned_objects")) ) @@ -300,6 +322,8 @@ **wrap_many_ints(W_GcCollectStepStats, ( "count", "duration", + "duration_min", + "duration_max", "oldstate", "newstate")) ) diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py --- a/pypy/module/gc/test/test_hook.py +++ b/pypy/module/gc/test/test_hook.py @@ -134,18 +134,20 @@ self.steps = [] def on_gc_minor(self, stats): - self.minors.append((stats.count, stats.duration)) + self.minors.append((stats.count, stats.duration, + stats.duration_min, stats.duration_max)) def on_gc_collect_step(self, stats): - self.steps.append((stats.count, stats.duration)) + self.steps.append((stats.count, stats.duration, + stats.duration_min, stats.duration_max)) on_gc_collect = None myhooks = MyHooks() gc.hooks.set(myhooks) self.fire_many() - assert myhooks.minors == [(2, 12)] - assert myhooks.steps == [(3, 42)] + assert myhooks.minors == [(2, 12, 5, 7)] + assert myhooks.steps == [(3, 42, 5, 22)] def test_clear_queue(self): import gc diff --git a/rpython/rlib/debug.py b/rpython/rlib/debug.py --- a/rpython/rlib/debug.py +++ b/rpython/rlib/debug.py @@ -2,8 +2,8 @@ import time from rpython.rtyper.extregistry import ExtRegistryEntry -from rpython.rlib.objectmodel import we_are_translated -from rpython.rlib.rarithmetic import is_valid_int +from rpython.rlib.objectmodel import we_are_translated, always_inline +from rpython.rlib.rarithmetic import is_valid_int, r_longlong from rpython.rtyper.extfunc import register_external from rpython.rtyper.lltypesystem import lltype from rpython.rtyper.lltypesystem import rffi @@ -74,34 +74,66 @@ _start_colors_2 = "" _stop_colors = "" -def debug_start(category): + at always_inline +def debug_start(category, timestamp=False): + """ + Start a PYPYLOG section. + + By default, the return value is undefined. If timestamp is True, always + return the current timestamp, even if PYPYLOG is not set. + """ + return _debug_start(category, timestamp) + + at always_inline +def debug_stop(category, timestamp=False): + """ + Stop a PYPYLOG section. See debug_start for docs about timestamp + """ + return _debug_stop(category, timestamp) + + +def _debug_start(category, timestamp): c = int(time.clock() * 100) print >> sys.stderr, '%s[%x] {%s%s' % (_start_colors_1, c, category, _stop_colors) if _log is not None: _log.debug_start(category) -def debug_stop(category): + if timestamp: + return r_longlong(c) + return r_longlong(-42) # random undefined value + +def _debug_stop(category, timestamp): c = int(time.clock() * 100) print >> sys.stderr, '%s[%x] %s}%s' % (_start_colors_2, c, category, _stop_colors) if _log is not None: _log.debug_stop(category) + if timestamp: + return r_longlong(c) + return r_longlong(-42) # random undefined value + class Entry(ExtRegistryEntry): - _about_ = debug_start, debug_stop + _about_ = _debug_start, _debug_stop - def compute_result_annotation(self, s_category): - return None + def compute_result_annotation(self, s_category, s_timestamp): + from rpython.rlib.rtimer import s_TIMESTAMP + return s_TIMESTAMP def specialize_call(self, hop): from rpython.rtyper.lltypesystem.rstr import string_repr + from rpython.rlib.rtimer import TIMESTAMP_type fn = self.instance - vlist = hop.inputargs(string_repr) + _, r_timestamp = hop.args_r + vlist = hop.inputargs(string_repr, r_timestamp) hop.exception_cannot_occur() t = hop.rtyper.annotator.translator if t.config.translation.log: - hop.genop(fn.__name__, vlist) + opname = fn.__name__[1:] # remove the '_' + return hop.genop(opname, vlist, resulttype=TIMESTAMP_type) + else: + return hop.inputconst(TIMESTAMP_type, r_longlong(0)) def have_debug_prints(): diff --git a/rpython/rlib/rtimer.py b/rpython/rlib/rtimer.py --- a/rpython/rlib/rtimer.py +++ b/rpython/rlib/rtimer.py @@ -7,6 +7,15 @@ _is_64_bit = r_uint.BITS > 32 +from rpython.annotator.model import SomeInteger +if _is_64_bit: + s_TIMESTAMP = SomeInteger() + TIMESTAMP_type = lltype.Signed +else: + s_TIMESTAMP = SomeInteger(knowntype=r_longlong) + TIMESTAMP_type = rffi.LONGLONG + + # unit of values returned by read_timestamp. Should be in sync with the ones # defined in translator/c/debug_print.h UNIT_TSC = 0 @@ -32,19 +41,11 @@ _about_ = read_timestamp def compute_result_annotation(self): - from rpython.annotator.model import SomeInteger - if _is_64_bit: - return SomeInteger() - else: - return SomeInteger(knowntype=r_longlong) + return s_TIMESTAMP def specialize_call(self, hop): hop.exception_cannot_occur() - if _is_64_bit: - resulttype = lltype.Signed - else: - resulttype = rffi.LONGLONG - return hop.genop("ll_read_timestamp", [], resulttype=resulttype) + return hop.genop("ll_read_timestamp", [], resulttype=TIMESTAMP_type) class ReadTimestampEntry(ExtRegistryEntry): diff --git a/rpython/rlib/test/test_debug.py b/rpython/rlib/test/test_debug.py --- a/rpython/rlib/test/test_debug.py +++ b/rpython/rlib/test/test_debug.py @@ -120,6 +120,26 @@ assert dlog == [("mycat", [('debug_print', 'foo', 2, 'bar', 3)])] +def test_debug_start_stop_timestamp(): + import time + def f(timestamp): + ts_a = debug_start('foo', timestamp=timestamp) + # simulate some CPU time + t = time.time() + while time.time()-t < 0.02: + pass + ts_b = debug_stop('foo', timestamp=timestamp) + return ts_b - ts_a + + assert f(False) == 0 + assert f(True) > 0 + # + res = interpret(f, [False]) + assert res == 0 + res = interpret(f, [True]) + assert res > 0 + + def test_debug_print_traceback(): from rpython.translator.c.test.test_genc import compile from rpython.rtyper.lltypesystem import lltype diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py --- a/rpython/rtyper/lltypesystem/opimpl.py +++ b/rpython/rtyper/lltypesystem/opimpl.py @@ -586,11 +586,11 @@ def op_debug_print(*args): debug.debug_print(*map(_normalize, args)) -def op_debug_start(category): - debug.debug_start(_normalize(category)) +def op_debug_start(category, timestamp): + return debug.debug_start(_normalize(category), timestamp) -def op_debug_stop(category): - debug.debug_stop(_normalize(category)) +def op_debug_stop(category, timestamp): + return debug.debug_stop(_normalize(category), timestamp) def op_debug_offset(): return debug.debug_offset() diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py --- a/rpython/translator/c/funcgen.py +++ b/rpython/translator/c/funcgen.py @@ -813,20 +813,27 @@ "if (PYPY_HAVE_DEBUG_PRINTS) { fprintf(PYPY_DEBUG_FILE, %s); %s}" % (', '.join(argv), free_line)) - def _op_debug(self, opname, arg): - if isinstance(arg, Constant): - string_literal = c_string_constant(''.join(arg.value.chars)) - return "%s(%s);" % (opname, string_literal) + def _op_debug(self, macro, op): + v_cat, v_timestamp = op.args + if isinstance(v_cat, Constant): + string_literal = c_string_constant(''.join(v_cat.value.chars)) + return "%s = %s(%s, %s);" % (self.expr(op.result), + macro, + string_literal, + self.expr(v_timestamp)) else: - x = "%s(RPyString_AsCharP(%s));\n" % (opname, self.expr(arg)) + x = "%s = %s(RPyString_AsCharP(%s), %s);\n" % (self.expr(op.result), + macro, + self.expr(v_cat), + self.expr(v_timestamp)) x += "RPyString_FreeCache();" return x def OP_DEBUG_START(self, op): - return self._op_debug('PYPY_DEBUG_START', op.args[0]) + return self._op_debug('PYPY_DEBUG_START', op) def OP_DEBUG_STOP(self, op): - return self._op_debug('PYPY_DEBUG_STOP', op.args[0]) + return self._op_debug('PYPY_DEBUG_STOP', op) def OP_HAVE_DEBUG_PRINTS_FOR(self, op): arg = op.args[0] diff --git a/rpython/translator/c/src/debug_print.c b/rpython/translator/c/src/debug_print.c --- a/rpython/translator/c/src/debug_print.c +++ b/rpython/translator/c/src/debug_print.c @@ -199,8 +199,8 @@ #define PYPY_LONG_LONG_PRINTF_FORMAT "ll" #endif -static void display_startstop(const char *prefix, const char *postfix, - const char *category, const char *colors) +static long long display_startstop(const char *prefix, const char *postfix, + const char *category, const char *colors) { long long timestamp; READ_TIMESTAMP(timestamp); @@ -208,10 +208,12 @@ colors, timestamp, prefix, category, postfix, debug_stop_colors); + return timestamp; } -void pypy_debug_start(const char *category) +long long pypy_debug_start(const char *category, long timestamp) { + long long result = 42; pypy_debug_ensure_opened(); /* Enter a nesting level. Nested debug_prints are disabled by default because the following left shift introduces a 0 in the last bit. @@ -224,22 +226,30 @@ if (!debug_prefix || !startswithoneof(category, debug_prefix)) { /* wrong section name, or no PYPYLOG at all, skip it */ - return; + if (timestamp) + READ_TIMESTAMP(result); + return result; } /* else make this subsection active */ pypy_have_debug_prints |= 1; } - display_startstop("{", "", category, debug_start_colors_1); + return display_startstop("{", "", category, debug_start_colors_1); } -void pypy_debug_stop(const char *category) +long long pypy_debug_stop(const char *category, long timestamp) { + long long result = 42; if (debug_profile | (pypy_have_debug_prints & 1)) { - display_startstop("", "}", category, debug_start_colors_2); + result = display_startstop("", "}", category, debug_start_colors_2); fflush(pypy_debug_file); } + else if (timestamp) + { + READ_TIMESTAMP(result); + } pypy_have_debug_prints >>= 1; + return result; } long pypy_have_debug_prints_for(const char *category_prefix) diff --git a/rpython/translator/c/src/debug_print.h b/rpython/translator/c/src/debug_print.h --- a/rpython/translator/c/src/debug_print.h +++ b/rpython/translator/c/src/debug_print.h @@ -31,8 +31,8 @@ #define PYPY_HAVE_DEBUG_PRINTS (pypy_have_debug_prints & 1 ? \ (pypy_debug_ensure_opened(), 1) : 0) #define PYPY_DEBUG_FILE pypy_debug_file -#define PYPY_DEBUG_START(cat) pypy_debug_start(cat) -#define PYPY_DEBUG_STOP(cat) pypy_debug_stop(cat) +#define PYPY_DEBUG_START(cat, ts) pypy_debug_start(cat, ts) +#define PYPY_DEBUG_STOP(cat, ts) pypy_debug_stop(cat, ts) #define OP_DEBUG_OFFSET(res) res = pypy_debug_offset() #define OP_DEBUG_FORKED(ofs, _) pypy_debug_forked(ofs) #define OP_HAVE_DEBUG_PRINTS(r) r = (pypy_have_debug_prints & 1) @@ -42,8 +42,8 @@ /* prototypes (internal use only) */ RPY_EXTERN void pypy_debug_ensure_opened(void); -RPY_EXTERN void pypy_debug_start(const char *category); -RPY_EXTERN void pypy_debug_stop(const char *category); +RPY_EXTERN long long pypy_debug_start(const char *category, long timestamp); +RPY_EXTERN long long pypy_debug_stop(const char *category, long timestamp); RPY_EXTERN long pypy_debug_offset(void); RPY_EXTERN void pypy_debug_forked(long original_offset); RPY_EXTERN long pypy_have_debug_prints_for(const char *category_prefix); diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py --- a/rpython/translator/c/test/test_standalone.py +++ b/rpython/translator/c/test/test_standalone.py @@ -1,5 +1,6 @@ import py import sys, os, re +import textwrap from rpython.config.translationoption import get_combined_translation_config from rpython.config.translationoption import SUPPORT__THREAD @@ -519,6 +520,52 @@ assert not err assert path.check(file=0) + def test_debug_start_stop_timestamp(self): + import sys + import time + from rpython.rlib.rtimer import read_timestamp + def entry_point(argv): + timestamp = int(argv[1]) + ts1 = debug_start("foo", timestamp=timestamp) + ts2 = read_timestamp() + ts3 = debug_stop("foo", timestamp=timestamp) + print ts1 + print ts2 + print ts3 + return 0 + t, cbuilder = self.compile(entry_point) + + def parse_out(out): + lines = out.strip().splitlines() + ts1, ts2, ts3 = lines + return int(ts1), int(ts2), int(ts3) + + # check with PYPYLOG :- + out, err = cbuilder.cmdexec("1", err=True, env={'PYPYLOG': ':-'}) + ts1, ts2, ts3 = parse_out(out) + assert ts3 > ts2 > ts1 + expected = ('[%x] {foo\n' % ts1 + + '[%x] foo}\n' % ts3) + assert err == expected + + # check with PYPYLOG profiling only + out, err = cbuilder.cmdexec("1", err=True, env={'PYPYLOG': '-'}) + ts1, ts2, ts3 = parse_out(out) + assert ts3 > ts2 > ts1 + expected = ('[%x] {foo\n' % ts1 + + '[%x] foo}\n' % ts3) + assert err == expected + + # check with PYPYLOG undefined + out, err = cbuilder.cmdexec("1", err=True, env={}) + ts1, ts2, ts3 = parse_out(out) + assert ts3 > ts2 > ts1 + + # check with PYPYLOG undefined and timestamp=False + out, err = cbuilder.cmdexec("0", err=True, env={}) + ts1, ts2, ts3 = parse_out(out) + assert ts1 == ts3 == 42; + def test_debug_print_start_stop_nonconst(self): def entry_point(argv): debug_start(argv[1]) From pypy.commits at gmail.com Thu Apr 19 06:49:40 2018 From: pypy.commits at gmail.com (cfbolz) Date: Thu, 19 Apr 2018 03:49:40 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: fix different error messages (in several cases this brings us closer to CPython again) Message-ID: <5ad87444.cb8e1c0a.18cde.d49e@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: py3.5 Changeset: r94382:eccd76eec042 Date: 2018-04-19 12:48 +0200 http://bitbucket.org/pypy/pypy/changeset/eccd76eec042/ Log: fix different error messages (in several cases this brings us closer to CPython again) diff --git a/lib-python/3/test/test_exceptions.py b/lib-python/3/test/test_exceptions.py --- a/lib-python/3/test/test_exceptions.py +++ b/lib-python/3/test/test_exceptions.py @@ -164,10 +164,10 @@ is_pypy = check_impl_detail(pypy=True) check('def fact(x):\n\treturn x!\n', 2, 10) - check('1 +\n', 1, 4 - is_pypy) - check('def spam():\n print(1)\n print(2)', 3, 0 if is_pypy else 10) - check('Python = "Python" +', 1, 20 - is_pypy) - check('Python = "\u1e54\xfd\u0163\u0125\xf2\xf1" +', 1, 20 - is_pypy) + check('1 +\n', 1, 4) + check('def spam():\n print(1)\n print(2)', 3, 2 if is_pypy else 10) + check('Python = "Python" +', 1, 20) + check('Python = "\u1e54\xfd\u0163\u0125\xf2\xf1" +', 1, 20) @cpython_only def testSettingException(self): diff --git a/lib-python/3/test/test_fstring.py b/lib-python/3/test/test_fstring.py --- a/lib-python/3/test/test_fstring.py +++ b/lib-python/3/test/test_fstring.py @@ -319,7 +319,7 @@ ["f'{3)+(4}'", ]) - self.assertAllRaise(SyntaxError, 'EOL while scanning string literal', + self.assertAllRaise(SyntaxError, r'end of line \(EOL\) while scanning string literal', ["f'{\n}'", ]) @@ -741,7 +741,7 @@ self.assertEqual('{d[0]}'.format(d=d), 'integer') def test_invalid_expressions(self): - self.assertAllRaise(SyntaxError, 'invalid syntax', + self.assertAllRaise(SyntaxError, "closing parenthesis '.' does not match opening parenthesis '.'", [r"f'{a[4)}'", r"f'{a(4]}'", ]) From pypy.commits at gmail.com Thu Apr 19 12:57:20 2018 From: pypy.commits at gmail.com (rlamy) Date: Thu, 19 Apr 2018 09:57:20 -0700 (PDT) Subject: [pypy-commit] pypy py3tests: Fix issues with appleveldefs in submodules Message-ID: <5ad8ca70.87981c0a.fc177.4550@mx.google.com> Author: Ronan Lamy Branch: py3tests Changeset: r94383:81ecef10320d Date: 2018-04-19 17:53 +0100 http://bitbucket.org/pypy/pypy/changeset/81ecef10320d/ Log: Fix issues with appleveldefs in submodules diff --git a/pypy/interpreter/mixedmodule.py b/pypy/interpreter/mixedmodule.py --- a/pypy/interpreter/mixedmodule.py +++ b/pypy/interpreter/mixedmodule.py @@ -30,17 +30,21 @@ @not_rpython def install(self): - """install this module, and it's submodules into + """Install this module, and its submodules into space.builtin_modules""" Module.install(self) if hasattr(self, "submodules"): space = self.space - name = space.text_w(self.w_name) + pkgname = space.text_w(self.w_name) for sub_name, module_cls in self.submodules.iteritems(): if module_cls.submodule_name is None: module_cls.submodule_name = sub_name - module_name = space.newtext("%s.%s" % (name, sub_name)) - m = module_cls(space, module_name) + else: + assert module_cls.submodule_name == sub_name + name = "%s.%s" % (pkgname, sub_name) + module_cls.applevel_name = name + w_name = space.newtext(name) + m = module_cls(space, w_name) m.install() self.submodules_w.append(m) @@ -175,8 +179,6 @@ cls.loaders = loaders = {} pkgroot = cls.__module__.rsplit('.', 1)[0] appname = cls.get_applevel_name() - if cls.submodule_name is not None: - appname += '.%s' % (cls.submodule_name,) for name, spec in cls.interpleveldefs.items(): loaders[name] = getinterpevalloader(pkgroot, spec) for name, spec in cls.appleveldefs.items(): diff --git a/pypy/interpreter/test/test_mixedmodule.py b/pypy/interpreter/test/test_mixedmodule.py --- a/pypy/interpreter/test/test_mixedmodule.py +++ b/pypy/interpreter/test/test_mixedmodule.py @@ -34,7 +34,10 @@ m.install() assert space.builtin_modules["test_module"] is m - assert isinstance(space.builtin_modules["test_module.sub"], SubModule) + submod = space.builtin_modules["test_module.sub"] + assert isinstance(submod, SubModule) + assert submod.get_applevel_name() == "test_module.sub" + class AppTestMixedModule(object): pytestmark = pytest.mark.skipif("config.option.runappdirect") diff --git a/pypy/module/__pypy__/app_signal.py b/pypy/module/__pypy__/app_signal.py --- a/pypy/module/__pypy__/app_signal.py +++ b/pypy/module/__pypy__/app_signal.py @@ -1,4 +1,4 @@ -from . import thread +from .thread import _signals_enter, _signals_exit # ^^ relative import of __pypy__.thread. Note that some tests depend on # this (test_enable_signals in test_signal.py) to work properly, # otherwise they get caught in some deadlock waiting for the import @@ -13,7 +13,7 @@ that is within a "with signals_enabled:". This other thread should be ready to handle unexpected exceptions that the signal handler might raise --- notably KeyboardInterrupt.''' - __enter__ = thread._signals_enter - __exit__ = thread._signals_exit + __enter__ = _signals_enter + __exit__ = _signals_exit signals_enabled = SignalsEnabled() From pypy.commits at gmail.com Thu Apr 19 15:38:24 2018 From: pypy.commits at gmail.com (rlamy) Date: Thu, 19 Apr 2018 12:38:24 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: Fix incompatibility between branch fix-sre-problems and 77d216c5b248 Message-ID: <5ad8f030.1c69fb81.be925.9301@mx.google.com> Author: Ronan Lamy Branch: py3.6 Changeset: r94385:020c87e1474f Date: 2018-04-19 20:37 +0100 http://bitbucket.org/pypy/pypy/changeset/020c87e1474f/ Log: Fix incompatibility between branch fix-sre-problems and 77d216c5b248 diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py --- a/pypy/module/_sre/interp_sre.py +++ b/pypy/module/_sre/interp_sre.py @@ -144,11 +144,11 @@ # indexgroup nor groupindex: they are derivated from the pattern. return space.newbool( self.flags == other.flags and - self.code == other.code and + self.code.pattern == other.code.pattern and space.eq_w(self.w_pattern, other.w_pattern)) def descr_hash(self, space): - code = ''.join([chr(c) for c in self.code]) + code = ''.join([chr(c) for c in self.code.pattern]) return space.newint(compute_hash( (self.flags, code, space.hash_w(self.w_pattern)))) From pypy.commits at gmail.com Thu Apr 19 16:53:40 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 19 Apr 2018 13:53:40 -0700 (PDT) Subject: [pypy-commit] pypy default: generalize for python2, 3 translated and untranslated, remove Sub.__name__ check Message-ID: <5ad901d4.1c69fb81.539d9.8b7f@mx.google.com> Author: Matti Picus Branch: Changeset: r94386:29dd04feb879 Date: 2018-04-19 20:18 +0300 http://bitbucket.org/pypy/pypy/changeset/29dd04feb879/ Log: generalize for python2,3 translated and untranslated, remove Sub.__name__ check diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py --- a/pypy/module/cpyext/test/test_arraymodule.py +++ b/pypy/module/cpyext/test/test_arraymodule.py @@ -194,7 +194,4 @@ self.attrib = True import gc module.subclass_with_attribute(Sub, "addattrib", "attrib", gc.collect) - if self.runappdirect: - assert Sub.__module__ == 'pypy.module.cpyext.test.test_arraymodule' - assert str(Sub) == "" - + assert Sub.__module__ == __name__ From pypy.commits at gmail.com Thu Apr 19 16:53:43 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 19 Apr 2018 13:53:43 -0700 (PDT) Subject: [pypy-commit] pypy default: simplify test Message-ID: <5ad901d7.c55c1c0a.ac2bc.0043@mx.google.com> Author: Matti Picus Branch: Changeset: r94387:ad79cc0ce9a8 Date: 2018-04-19 23:37 +0300 http://bitbucket.org/pypy/pypy/changeset/ad79cc0ce9a8/ Log: simplify test diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py --- a/pypy/module/cpyext/test/test_unicodeobject.py +++ b/pypy/module/cpyext/test/test_unicodeobject.py @@ -26,11 +26,8 @@ if(PyUnicode_GetSize(s) != 11) { result = -PyUnicode_GetSize(s); } -#ifdef PYPY_VERSION - // Slightly silly test that tp_basicsize is reasonable. - if(s->ob_type->tp_basicsize != sizeof(void*)*7) + if(s->ob_type->tp_basicsize != sizeof(PyUnicodeObject)) result = s->ob_type->tp_basicsize; -#endif // PYPY_VERSION Py_DECREF(s); return PyLong_FromLong(result); """), From pypy.commits at gmail.com Thu Apr 19 16:53:44 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 19 Apr 2018 13:53:44 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: merge default into branch Message-ID: <5ad901d8.1c69fb81.5b1f2.8305@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r94388:a5488a8c8485 Date: 2018-04-19 23:41 +0300 http://bitbucket.org/pypy/pypy/changeset/a5488a8c8485/ Log: merge default into branch diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py --- a/pypy/module/cpyext/test/test_arraymodule.py +++ b/pypy/module/cpyext/test/test_arraymodule.py @@ -187,7 +187,4 @@ self.attrib = True import gc module.subclass_with_attribute(Sub, "addattrib", "attrib", gc.collect) - if self.runappdirect: - assert Sub.__module__ == 'pypy.module.cpyext.test.test_arraymodule' - assert str(Sub) == "" - + assert Sub.__module__ == __name__ diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py --- a/pypy/module/cpyext/test/test_unicodeobject.py +++ b/pypy/module/cpyext/test/test_unicodeobject.py @@ -27,11 +27,8 @@ if(PyUnicode_GetSize(s) != 11) { result = -PyUnicode_GetSize(s); } -#ifdef PYPY_VERSION - // Slightly silly test that tp_basicsize is reasonable. - if(s->ob_type->tp_basicsize != sizeof(void*)*12) + if(s->ob_type->tp_basicsize != sizeof(PyUnicodeObject)) result = s->ob_type->tp_basicsize; -#endif // PYPY_VERSION Py_DECREF(s); return PyLong_FromLong(result); """), From pypy.commits at gmail.com Thu Apr 19 16:53:46 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 19 Apr 2018 13:53:46 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: attribute name changed Message-ID: <5ad901da.1c69fb81.3babb.d5a9@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r94389:a0be09f88351 Date: 2018-04-19 23:41 +0300 http://bitbucket.org/pypy/pypy/changeset/a0be09f88351/ Log: attribute name changed diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py --- a/pypy/module/cpyext/cdatetime.py +++ b/pypy/module/cpyext/cdatetime.py @@ -149,7 +149,7 @@ # No tzinfo return py_datetime = rffi.cast(PyDateTime_Time, py_obj) - w_tzinfo = space.getattr(w_obj, space.newtext('tzinfo')) + w_tzinfo = space.getattr(w_obj, space.newtext('_tzinfo')) if space.is_none(w_tzinfo): py_datetime.c_hastzinfo = cts.cast('unsigned char', 0) py_datetime.c_tzinfo = lltype.nullptr(PyObject.TO) From pypy.commits at gmail.com Thu Apr 19 22:33:41 2018 From: pypy.commits at gmail.com (rlamy) Date: Thu, 19 Apr 2018 19:33:41 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: Skip import of SSLSession in ssl (to unbreak pip) Message-ID: <5ad95185.1c69fb81.44794.bbd5@mx.google.com> Author: Ronan Lamy Branch: py3.6 Changeset: r94390:d3ea363d17b6 Date: 2018-04-20 03:32 +0100 http://bitbucket.org/pypy/pypy/changeset/d3ea363d17b6/ Log: Skip import of SSLSession in ssl (to unbreak pip) diff --git a/lib-python/3/ssl.py b/lib-python/3/ssl.py --- a/lib-python/3/ssl.py +++ b/lib-python/3/ssl.py @@ -100,7 +100,7 @@ import _ssl # if we can't import it, let the error propagate from _ssl import OPENSSL_VERSION_NUMBER, OPENSSL_VERSION_INFO, OPENSSL_VERSION -from _ssl import _SSLContext, MemoryBIO, SSLSession +from _ssl import _SSLContext, MemoryBIO#, SSLSession # XXX: PyPy hack from _ssl import ( SSLError, SSLZeroReturnError, SSLWantReadError, SSLWantWriteError, SSLSyscallError, SSLEOFError, From pypy.commits at gmail.com Fri Apr 20 05:39:38 2018 From: pypy.commits at gmail.com (arigo) Date: Fri, 20 Apr 2018 02:39:38 -0700 (PDT) Subject: [pypy-commit] pypy.org extradoc: Fix bogus source link to pypy2-v5.10.1 Message-ID: <5ad9b55a.1c69fb81.e2e28.ff04@mx.google.com> Author: Armin Rigo Branch: extradoc Changeset: r922:11b8477b9161 Date: 2018-04-20 11:39 +0200 http://bitbucket.org/pypy/pypy.org/changeset/11b8477b9161/ Log: Fix bogus source link to pypy2-v5.10.1 diff --git a/download.html b/download.html --- a/download.html +++ b/download.html @@ -269,7 +269,7 @@

    Alternatively, get one of the following smaller packages for the source at the same revision as the above binaries:

    • diff --git a/source/download.txt b/source/download.txt --- a/source/download.txt +++ b/source/download.txt @@ -309,10 +309,10 @@ Alternatively, get one of the following smaller packages for the source at the same revision as the above binaries: - * `pypy2-v5.10.1-src.tar.bz2`__ (sources, PyPy 2 only) + * `pypy2-v5.10.0-src.tar.bz2`__ (sources, PyPy 2 only) * `pypy3-v5.10.1-src.tar.bz2`__ (sources, PyPy 3 only) - .. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.10.1-src.tar.bz2 + .. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.10.0-src.tar.bz2 .. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.10.1-src.tar.bz2 From pypy.commits at gmail.com Fri Apr 20 07:26:12 2018 From: pypy.commits at gmail.com (mattip) Date: Fri, 20 Apr 2018 04:26:12 -0700 (PDT) Subject: [pypy-commit] pypy default: restart whatsnew-head, release notice Message-ID: <5ad9ce54.1c69fb81.5ce62.49fe@mx.google.com> Author: Matti Picus Branch: Changeset: r94391:8941ee7fcd8f Date: 2018-04-20 14:13 +0300 http://bitbucket.org/pypy/pypy/changeset/8941ee7fcd8f/ Log: restart whatsnew-head, release notice diff --git a/pypy/doc/release-v6.0.0.rst b/pypy/doc/release-v6.0.0.rst --- a/pypy/doc/release-v6.0.0.rst +++ b/pypy/doc/release-v6.0.0.rst @@ -18,6 +18,8 @@ getting started writing code. We have improved our parser to emit more friendly `syntax errors`_, making PyPy not only faster but more friendly. +The GC now has `hooks`_ to gain more insights into its performance + The Windows PyPy3.5 release is still considered beta-quality. There are open issues with unicode handling especially around system calls and c-extensions. @@ -53,6 +55,7 @@ .. _`blog post`: https://morepypy.blogspot.it/2017/10/cape-of-good-hope-for-pypy-hello-from.html .. _pygobject: https://lazka.github.io/posts/2018-04_pypy-pygobject/index.html .. _`syntax errors`: https://morepypy.blogspot.com/2018/04/improving-syntaxerror-in-pypy.html +.. _`hooks`: gc_info.html#gc-hooks What is PyPy? ============= @@ -101,8 +104,9 @@ * Added missing attributes to C-API ``instancemethod`` on pypy3 * Store error state in thread-local storage for C-API. * Fix JIT bugs exposed in the sre module -* Improve speed of Python parser, improve ParseError messages slightly +* Improve speed of Python parser, improve ParseError messages and SyntaxError * Handle JIT hooks more efficiently +* Fix a rare GC bug exposed by intensive use of cpyext `Buffer` s We also refactored many parts of the JIT bridge optimizations, as well as cpyext internals, and together with new contributors fixed issues, added new diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -3,18 +3,7 @@ ========================== .. this is a revision shortly after release-pypy-6.0.0 -.. startrev: f22145c34985 +.. startrev: ad79cc0ce9a8 -.. branch: issue2752 -Fix a rare GC bug that was introduced more than one year ago, but was -not diagnosed before issue #2752. - -.. branch: gc-hooks - -Introduce GC hooks, as documented in doc/gc_info.rst - -.. branch: gc-hook-better-timestamp - -Improve GC hooks diff --git a/pypy/doc/whatsnew-pypy2-6.0.0.rst b/pypy/doc/whatsnew-pypy2-6.0.0.rst --- a/pypy/doc/whatsnew-pypy2-6.0.0.rst +++ b/pypy/doc/whatsnew-pypy2-6.0.0.rst @@ -109,3 +109,16 @@ Improve line offsets that are reported by SyntaxError. Improve error messages for a few situations, including mismatched parenthesis. + +.. branch: issue2752 + +Fix a rare GC bug that was introduced more than one year ago, but was +not diagnosed before issue #2752. + +.. branch: gc-hooks + +Introduce GC hooks, as documented in doc/gc_info.rst + +.. branch: gc-hook-better-timestamp + +Improve GC hooksd From pypy.commits at gmail.com Fri Apr 20 07:26:17 2018 From: pypy.commits at gmail.com (mattip) Date: Fri, 20 Apr 2018 04:26:17 -0700 (PDT) Subject: [pypy-commit] pypy release-pypy2.7-6.x: merge default into release Message-ID: <5ad9ce59.530b1c0a.aa7d2.a1d7@mx.google.com> Author: Matti Picus Branch: release-pypy2.7-6.x Changeset: r94392:0f99a0bed290 Date: 2018-04-20 14:20 +0300 http://bitbucket.org/pypy/pypy/changeset/0f99a0bed290/ Log: merge default into release diff too long, truncating to 2000 out of 4130 lines diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst --- a/pypy/doc/gc_info.rst +++ b/pypy/doc/gc_info.rst @@ -121,6 +121,166 @@ alive by GC objects, but not accounted in the GC +GC Hooks +-------- + +GC hooks are user-defined functions which are called whenever a specific GC +event occur, and can be used to monitor GC activity and pauses. You can +install the hooks by setting the following attributes: + +``gc.hook.on_gc_minor`` + Called whenever a minor collection occurs. It corresponds to + ``gc-minor`` sections inside ``PYPYLOG``. + +``gc.hook.on_gc_collect_step`` + Called whenever an incremental step of a major collection occurs. It + corresponds to ``gc-collect-step`` sections inside ``PYPYLOG``. + +``gc.hook.on_gc_collect`` + Called after the last incremental step, when a major collection is fully + done. It corresponds to ``gc-collect-done`` sections inside ``PYPYLOG``. + +To uninstall a hook, simply set the corresponding attribute to ``None``. To +install all hooks at once, you can call ``gc.hooks.set(obj)``, which will look +for methods ``on_gc_*`` on ``obj``. To uninstall all the hooks at once, you +can call ``gc.hooks.reset()``. + +The functions called by the hooks receive a single ``stats`` argument, which +contains various statistics about the event. + +Note that PyPy cannot call the hooks immediately after a GC event, but it has +to wait until it reaches a point in which the interpreter is in a known state +and calling user-defined code is harmless. It might happen that multiple +events occur before the hook is invoked: in this case, you can inspect the +value ``stats.count`` to know how many times the event occured since the last +time the hook was called. Similarly, ``stats.duration`` contains the +**total** time spent by the GC for this specific event since the last time the +hook was called. + +On the other hand, all the other fields of the ``stats`` object are relative +only to the **last** event of the series. + +The attributes for ``GcMinorStats`` are: + +``count`` + The number of minor collections occured since the last hook call. + +``duration`` + The total time spent inside minor collections since the last hook + call. See below for more information on the unit. + +``duration_min`` + The duration of the fastest minor collection since the last hook call. + +``duration_max`` + The duration of the slowest minor collection since the last hook call. + + ``total_memory_used`` + The amount of memory used at the end of the minor collection, in + bytes. This include the memory used in arenas (for GC-managed memory) and + raw-malloced memory (e.g., the content of numpy arrays). + +``pinned_objects`` + the number of pinned objects. + + +The attributes for ``GcCollectStepStats`` are: + +``count``, ``duration``, ``duration_min``, ``duration_max`` + See above. + +``oldstate``, ``newstate`` + Integers which indicate the state of the GC before and after the step. + +The value of ``oldstate`` and ``newstate`` is one of these constants, defined +inside ``gc.GcCollectStepStats``: ``STATE_SCANNING``, ``STATE_MARKING``, +``STATE_SWEEPING``, ``STATE_FINALIZING``. It is possible to get a string +representation of it by indexing the ``GC_STATS`` tuple. + + +The attributes for ``GcCollectStats`` are: + +``count`` + See above. + +``num_major_collects`` + The total number of major collections which have been done since the + start. Contrarily to ``count``, this is an always-growing counter and it's + not reset between invocations. + +``arenas_count_before``, ``arenas_count_after`` + Number of arenas used before and after the major collection. + +``arenas_bytes`` + Total number of bytes used by GC-managed objects. + +``rawmalloc_bytes_before``, ``rawmalloc_bytes_after`` + Total number of bytes used by raw-malloced objects, before and after the + major collection. + +Note that ``GcCollectStats`` has **not** got a ``duration`` field. This is +because all the GC work is done inside ``gc-collect-step``: +``gc-collect-done`` is used only to give additional stats, but doesn't do any +actual work. + +A note about the ``duration`` field: depending on the architecture and +operating system, PyPy uses different ways to read timestamps, so ``duration`` +is expressed in varying units. It is possible to know which by calling +``__pypy__.debug_get_timestamp_unit()``, which can be one of the following +values: + +``tsc`` + The default on ``x86`` machines: timestamps are expressed in CPU ticks, as + read by the `Time Stamp Counter`_. + +``ns`` + Timestamps are expressed in nanoseconds. + +``QueryPerformanceCounter`` + On Windows, in case for some reason ``tsc`` is not available: timestamps + are read using the win API ``QueryPerformanceCounter()``. + + +Unfortunately, there does not seem to be a reliable standard way for +converting ``tsc`` ticks into nanoseconds, although in practice on modern CPUs +it is enough to divide the ticks by the maximum nominal frequency of the CPU. +For this reason, PyPy gives the raw value, and leaves the job of doing the +conversion to external libraries. + +Here is an example of GC hooks in use:: + + import sys + import gc + + class MyHooks(object): + done = False + + def on_gc_minor(self, stats): + print 'gc-minor: count = %02d, duration = %d' % (stats.count, + stats.duration) + + def on_gc_collect_step(self, stats): + old = gc.GcCollectStepStats.GC_STATES[stats.oldstate] + new = gc.GcCollectStepStats.GC_STATES[stats.newstate] + print 'gc-collect-step: %s --> %s' % (old, new) + print ' count = %02d, duration = %d' % (stats.count, + stats.duration) + + def on_gc_collect(self, stats): + print 'gc-collect-done: count = %02d' % stats.count + self.done = True + + hooks = MyHooks() + gc.hooks.set(hooks) + + # simulate some GC activity + lst = [] + while not hooks.done: + lst = [lst, 1, 2, 3] + + +.. _`Time Stamp Counter`: https://en.wikipedia.org/wiki/Time_Stamp_Counter + .. _minimark-environment-variables: Environment variables diff --git a/pypy/doc/release-v6.0.0.rst b/pypy/doc/release-v6.0.0.rst --- a/pypy/doc/release-v6.0.0.rst +++ b/pypy/doc/release-v6.0.0.rst @@ -18,6 +18,8 @@ getting started writing code. We have improved our parser to emit more friendly `syntax errors`_, making PyPy not only faster but more friendly. +The GC now has `hooks`_ to gain more insights into its performance + The Windows PyPy3.5 release is still considered beta-quality. There are open issues with unicode handling especially around system calls and c-extensions. @@ -53,6 +55,7 @@ .. _`blog post`: https://morepypy.blogspot.it/2017/10/cape-of-good-hope-for-pypy-hello-from.html .. _pygobject: https://lazka.github.io/posts/2018-04_pypy-pygobject/index.html .. _`syntax errors`: https://morepypy.blogspot.com/2018/04/improving-syntaxerror-in-pypy.html +.. _`hooks`: gc_info.html#gc-hooks What is PyPy? ============= @@ -101,8 +104,9 @@ * Added missing attributes to C-API ``instancemethod`` on pypy3 * Store error state in thread-local storage for C-API. * Fix JIT bugs exposed in the sre module -* Improve speed of Python parser, improve ParseError messages slightly +* Improve speed of Python parser, improve ParseError messages and SyntaxError * Handle JIT hooks more efficiently +* Fix a rare GC bug exposed by intensive use of cpyext `Buffer` s We also refactored many parts of the JIT bridge optimizations, as well as cpyext internals, and together with new contributors fixed issues, added new diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -3,6 +3,7 @@ ========================== .. this is a revision shortly after release-pypy-6.0.0 -.. startrev: f22145c34985 +.. startrev: ad79cc0ce9a8 + diff --git a/pypy/doc/whatsnew-pypy2-6.0.0.rst b/pypy/doc/whatsnew-pypy2-6.0.0.rst --- a/pypy/doc/whatsnew-pypy2-6.0.0.rst +++ b/pypy/doc/whatsnew-pypy2-6.0.0.rst @@ -109,3 +109,16 @@ Improve line offsets that are reported by SyntaxError. Improve error messages for a few situations, including mismatched parenthesis. + +.. branch: issue2752 + +Fix a rare GC bug that was introduced more than one year ago, but was +not diagnosed before issue #2752. + +.. branch: gc-hooks + +Introduce GC hooks, as documented in doc/gc_info.rst + +.. branch: gc-hook-better-timestamp + +Improve GC hooksd diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -215,6 +215,7 @@ usage = SUPPRESS_USAGE take_options = True + space = None def opt_parser(self, config): parser = to_optparse(config, useoptions=["objspace.*"], @@ -364,15 +365,21 @@ from pypy.module.pypyjit.hooks import pypy_hooks return PyPyJitPolicy(pypy_hooks) + def get_gchooks(self): + from pypy.module.gc.hook import LowLevelGcHooks + if self.space is None: + raise Exception("get_gchooks must be called afeter get_entry_point") + return self.space.fromcache(LowLevelGcHooks) + def get_entry_point(self, config): - space = make_objspace(config) + self.space = make_objspace(config) # manually imports app_main.py filename = os.path.join(pypydir, 'interpreter', 'app_main.py') app = gateway.applevel(open(filename).read(), 'app_main.py', 'app_main') app.hidden_applevel = False - w_dict = app.getwdict(space) - entry_point, _ = create_entry_point(space, w_dict) + w_dict = app.getwdict(self.space) + entry_point, _ = create_entry_point(self.space, w_dict) return entry_point, None, PyPyAnnotatorPolicy() @@ -381,7 +388,7 @@ 'jitpolicy', 'get_entry_point', 'get_additional_config_options']: ns[name] = getattr(self, name) - + ns['get_gchooks'] = self.get_gchooks PyPyTarget().interface(globals()) diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -404,7 +404,7 @@ self._periodic_actions = [] self._nonperiodic_actions = [] self.has_bytecode_counter = False - self.fired_actions = None + self._fired_actions_reset() # the default value is not 100, unlike CPython 2.7, but a much # larger value, because we use a technique that not only allows # but actually *forces* another thread to run whenever the counter @@ -416,13 +416,28 @@ """Request for the action to be run before the next opcode.""" if not action._fired: action._fired = True - if self.fired_actions is None: - self.fired_actions = [] - self.fired_actions.append(action) + self._fired_actions_append(action) # set the ticker to -1 in order to force action_dispatcher() # to run at the next possible bytecode self.reset_ticker(-1) + def _fired_actions_reset(self): + # linked list of actions. We cannot use a normal RPython list because + # we want AsyncAction.fire() to be marked as @rgc.collect: this way, + # we can call it from e.g. GcHooks or cpyext's dealloc_trigger. + self._fired_actions_first = None + self._fired_actions_last = None + + @rgc.no_collect + def _fired_actions_append(self, action): + assert action._next is None + if self._fired_actions_first is None: + self._fired_actions_first = action + self._fired_actions_last = action + else: + self._fired_actions_last._next = action + self._fired_actions_last = action + @not_rpython def register_periodic_action(self, action, use_bytecode_counter): """ @@ -467,9 +482,9 @@ action.perform(ec, frame) # nonperiodic actions - list = self.fired_actions - if list is not None: - self.fired_actions = None + action = self._fired_actions_first + if action: + self._fired_actions_reset() # NB. in case there are several actions, we reset each # 'action._fired' to false only when we're about to call # 'action.perform()'. This means that if @@ -477,9 +492,10 @@ # the corresponding perform(), the fire() has no # effect---which is the effect we want, because # perform() will be called anyway. - for action in list: + while action is not None: action._fired = False action.perform(ec, frame) + action._next, action = None, action._next self.action_dispatcher = action_dispatcher @@ -512,10 +528,12 @@ to occur between two opcodes, not at a completely random time. """ _fired = False + _next = None def __init__(self, space): self.space = space + @rgc.no_collect def fire(self): """Request for the action to be run before the next opcode. The action must have been registered at space initalization time.""" diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py --- a/pypy/interpreter/test/test_executioncontext.py +++ b/pypy/interpreter/test/test_executioncontext.py @@ -37,6 +37,37 @@ pass assert i == 9 + def test_action_queue(self): + events = [] + + class Action1(executioncontext.AsyncAction): + def perform(self, ec, frame): + events.append('one') + + class Action2(executioncontext.AsyncAction): + def perform(self, ec, frame): + events.append('two') + + space = self.space + a1 = Action1(space) + a2 = Action2(space) + a1.fire() + a2.fire() + space.appexec([], """(): + n = 5 + return n + 2 + """) + assert events == ['one', 'two'] + # + events[:] = [] + a1.fire() + space.appexec([], """(): + n = 5 + return n + 2 + """) + assert events == ['one'] + + def test_periodic_action(self): from pypy.interpreter.executioncontext import ActionFlag diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -82,6 +82,8 @@ 'debug_stop' : 'interp_debug.debug_stop', 'debug_print_once' : 'interp_debug.debug_print_once', 'debug_flush' : 'interp_debug.debug_flush', + 'debug_read_timestamp' : 'interp_debug.debug_read_timestamp', + 'debug_get_timestamp_unit' : 'interp_debug.debug_get_timestamp_unit', 'builtinify' : 'interp_magic.builtinify', 'hidden_applevel' : 'interp_magic.hidden_applevel', 'get_hidden_tb' : 'interp_magic.get_hidden_tb', diff --git a/pypy/module/__pypy__/interp_debug.py b/pypy/module/__pypy__/interp_debug.py --- a/pypy/module/__pypy__/interp_debug.py +++ b/pypy/module/__pypy__/interp_debug.py @@ -1,11 +1,14 @@ from pypy.interpreter.gateway import unwrap_spec from rpython.rlib import debug, jit - +from rpython.rlib import rtimer @jit.dont_look_inside - at unwrap_spec(category='text') -def debug_start(space, category): - debug.debug_start(category) + at unwrap_spec(category='text', timestamp=bool) +def debug_start(space, category, timestamp=False): + res = debug.debug_start(category, timestamp=timestamp) + if timestamp: + return space.newint(res) + return space.w_None @jit.dont_look_inside def debug_print(space, args_w): @@ -13,10 +16,12 @@ debug.debug_print(' '.join(parts)) @jit.dont_look_inside - at unwrap_spec(category='text') -def debug_stop(space, category): - debug.debug_stop(category) - + at unwrap_spec(category='text', timestamp=bool) +def debug_stop(space, category, timestamp=False): + res = debug.debug_stop(category, timestamp=timestamp) + if timestamp: + return space.newint(res) + return space.w_None @unwrap_spec(category='text') def debug_print_once(space, category, args_w): @@ -28,3 +33,18 @@ @jit.dont_look_inside def debug_flush(space): debug.debug_flush() + +def debug_read_timestamp(space): + return space.newint(rtimer.read_timestamp()) + +def debug_get_timestamp_unit(space): + unit = rtimer.get_timestamp_unit() + if unit == rtimer.UNIT_TSC: + unit_str = 'tsc' + elif unit == rtimer.UNIT_NS: + unit_str = 'ns' + elif unit == rtimer.UNIT_QUERY_PERFORMANCE_COUNTER: + unit_str = 'QueryPerformanceCounter' + else: + unit_str = 'UNKNOWN(%d)' % unit + return space.newtext(unit_str) diff --git a/pypy/module/__pypy__/test/test_debug.py b/pypy/module/__pypy__/test/test_debug.py --- a/pypy/module/__pypy__/test/test_debug.py +++ b/pypy/module/__pypy__/test/test_debug.py @@ -48,3 +48,26 @@ from __pypy__ import debug_flush debug_flush() # assert did not crash + + def test_debug_read_timestamp(self): + from __pypy__ import debug_read_timestamp + a = debug_read_timestamp() + b = debug_read_timestamp() + assert b > a + + def test_debug_get_timestamp_unit(self): + from __pypy__ import debug_get_timestamp_unit + unit = debug_get_timestamp_unit() + assert unit in ('tsc', 'ns', 'QueryPerformanceCounter') + + def test_debug_start_stop_timestamp(self): + import time + from __pypy__ import debug_start, debug_stop, debug_read_timestamp + assert debug_start('foo') is None + assert debug_stop('foo') is None + ts1 = debug_start('foo', timestamp=True) + t = time.time() + while time.time() - t < 0.02: + pass + ts2 = debug_stop('foo', timestamp=True) + assert ts2 > ts1 diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py --- a/pypy/module/_cffi_backend/ctypearray.py +++ b/pypy/module/_cffi_backend/ctypearray.py @@ -70,7 +70,15 @@ length = wchar_helper.unicode_size_as_char32(u) return (w_value, length + 1) else: - explicitlength = space.getindex_w(w_value, space.w_OverflowError) + try: + explicitlength = space.getindex_w(w_value, + space.w_OverflowError) + except OperationError as e: + if e.match(space, space.w_TypeError): + raise oefmt(space.w_TypeError, + "expected new array length or list/tuple/str, " + "not %T", w_value) + raise if explicitlength < 0: raise oefmt(space.w_ValueError, "negative array length") return (space.w_None, explicitlength) diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -278,6 +278,9 @@ assert repr(q).startswith("" - + assert Sub.__module__ == __name__ diff --git a/pypy/module/cpyext/test/test_bufferobject.py b/pypy/module/cpyext/test/test_bufferobject.py --- a/pypy/module/cpyext/test/test_bufferobject.py +++ b/pypy/module/cpyext/test/test_bufferobject.py @@ -4,6 +4,7 @@ from pypy.module.cpyext.api import PyObject class AppTestBufferObject(AppTestCpythonExtensionBase): + def test_FromMemory(self): module = self.import_extension('foo', [ ("get_FromMemory", "METH_NOARGS", @@ -62,3 +63,61 @@ a = array.array('c', 'text') b = buffer(a) assert module.roundtrip(b) == 'text' + + + def test_issue2752(self): + iterations = 10 + if self.runappdirect: + iterations = 2000 + module = self.import_extension('foo', [ + ("test_mod", 'METH_VARARGS', + """ + PyObject *obj; + Py_buffer bp; + if (!PyArg_ParseTuple(args, "O", &obj)) + return NULL; + + if (PyObject_GetBuffer(obj, &bp, PyBUF_SIMPLE) == -1) + return NULL; + + if (((unsigned char*)bp.buf)[0] != '0') { + void * buf = (void*)bp.buf; + unsigned char val[4]; + char * s = PyString_AsString(obj); + memcpy(val, bp.buf, 4); + PyBuffer_Release(&bp); + if (PyObject_GetBuffer(obj, &bp, PyBUF_SIMPLE) == -1) + return NULL; + PyErr_Format(PyExc_ValueError, + "mismatch: %p [%x %x %x %x...] now %p [%x %x %x %x...] as str '%s'", + buf, val[0], val[1], val[2], val[3], + (void *)bp.buf, + ((unsigned char*)bp.buf)[0], + ((unsigned char*)bp.buf)[1], + ((unsigned char*)bp.buf)[2], + ((unsigned char*)bp.buf)[3], + s); + PyBuffer_Release(&bp); + return NULL; + } + + PyBuffer_Release(&bp); + Py_RETURN_NONE; + """), + ]) + bufsize = 4096 + def getdata(bufsize): + data = b'01234567' + for x in range(18): + data += data + if len(data) >= bufsize: + break + return data + for j in range(iterations): + block = getdata(bufsize) + assert block[:8] == '01234567' + try: + module.test_mod(block) + except ValueError as e: + print("%s at it=%d" % (e, j)) + assert False diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py --- a/pypy/module/cpyext/test/test_unicodeobject.py +++ b/pypy/module/cpyext/test/test_unicodeobject.py @@ -26,11 +26,8 @@ if(PyUnicode_GetSize(s) != 11) { result = -PyUnicode_GetSize(s); } -#ifdef PYPY_VERSION - // Slightly silly test that tp_basicsize is reasonable. - if(s->ob_type->tp_basicsize != sizeof(void*)*7) + if(s->ob_type->tp_basicsize != sizeof(PyUnicodeObject)) result = s->ob_type->tp_basicsize; -#endif // PYPY_VERSION Py_DECREF(s); return PyLong_FromLong(result); """), diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py --- a/pypy/module/gc/__init__.py +++ b/pypy/module/gc/__init__.py @@ -34,5 +34,7 @@ 'get_typeids_z': 'referents.get_typeids_z', 'get_typeids_list': 'referents.get_typeids_list', 'GcRef': 'referents.W_GcRef', + 'hooks': 'space.fromcache(hook.W_AppLevelHooks)', + 'GcCollectStepStats': 'hook.W_GcCollectStepStats', }) MixedModule.__init__(self, space, w_name) diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py new file mode 100644 --- /dev/null +++ b/pypy/module/gc/hook.py @@ -0,0 +1,341 @@ +from rpython.memory.gc.hook import GcHooks +from rpython.memory.gc import incminimark +from rpython.rlib.nonconst import NonConstant +from rpython.rlib.rarithmetic import r_uint, r_longlong, longlongmax +from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault +from pypy.interpreter.baseobjspace import W_Root +from pypy.interpreter.typedef import TypeDef, interp_attrproperty, GetSetProperty +from pypy.interpreter.executioncontext import AsyncAction + +class LowLevelGcHooks(GcHooks): + """ + These are the low-level hooks which are called directly from the GC. + + They can't do much, because the base class marks the methods as + @rgc.no_collect. + + This is expected to be a singleton, created by space.fromcache, and it is + integrated with the translation by targetpypystandalone.get_gchooks + """ + + def __init__(self, space): + self.space = space + self.w_hooks = space.fromcache(W_AppLevelHooks) + + def is_gc_minor_enabled(self): + return self.w_hooks.gc_minor_enabled + + def is_gc_collect_step_enabled(self): + return self.w_hooks.gc_collect_step_enabled + + def is_gc_collect_enabled(self): + return self.w_hooks.gc_collect_enabled + + def on_gc_minor(self, duration, total_memory_used, pinned_objects): + action = self.w_hooks.gc_minor + action.count += 1 + action.duration += duration + action.duration_min = min(action.duration_min, duration) + action.duration_max = max(action.duration_max, duration) + action.total_memory_used = total_memory_used + action.pinned_objects = pinned_objects + action.fire() + + def on_gc_collect_step(self, duration, oldstate, newstate): + action = self.w_hooks.gc_collect_step + action.count += 1 + action.duration += duration + action.duration_min = min(action.duration_min, duration) + action.duration_max = max(action.duration_max, duration) + action.oldstate = oldstate + action.newstate = newstate + action.fire() + + def on_gc_collect(self, num_major_collects, + arenas_count_before, arenas_count_after, + arenas_bytes, rawmalloc_bytes_before, + rawmalloc_bytes_after): + action = self.w_hooks.gc_collect + action.count += 1 + action.num_major_collects = num_major_collects + action.arenas_count_before = arenas_count_before + action.arenas_count_after = arenas_count_after + action.arenas_bytes = arenas_bytes + action.rawmalloc_bytes_before = rawmalloc_bytes_before + action.rawmalloc_bytes_after = rawmalloc_bytes_after + action.fire() + + +class W_AppLevelHooks(W_Root): + + def __init__(self, space): + self.space = space + self.gc_minor_enabled = False + self.gc_collect_step_enabled = False + self.gc_collect_enabled = False + self.gc_minor = GcMinorHookAction(space) + self.gc_collect_step = GcCollectStepHookAction(space) + self.gc_collect = GcCollectHookAction(space) + + def descr_get_on_gc_minor(self, space): + return self.gc_minor.w_callable + + def descr_set_on_gc_minor(self, space, w_obj): + self.gc_minor_enabled = not space.is_none(w_obj) + self.gc_minor.w_callable = w_obj + self.gc_minor.fix_annotation() + + def descr_get_on_gc_collect_step(self, space): + return self.gc_collect_step.w_callable + + def descr_set_on_gc_collect_step(self, space, w_obj): + self.gc_collect_step_enabled = not space.is_none(w_obj) + self.gc_collect_step.w_callable = w_obj + self.gc_collect_step.fix_annotation() + + def descr_get_on_gc_collect(self, space): + return self.gc_collect.w_callable + + def descr_set_on_gc_collect(self, space, w_obj): + self.gc_collect_enabled = not space.is_none(w_obj) + self.gc_collect.w_callable = w_obj + self.gc_collect.fix_annotation() + + def descr_set(self, space, w_obj): + w_a = space.getattr(w_obj, space.newtext('on_gc_minor')) + w_b = space.getattr(w_obj, space.newtext('on_gc_collect_step')) + w_c = space.getattr(w_obj, space.newtext('on_gc_collect')) + self.descr_set_on_gc_minor(space, w_a) + self.descr_set_on_gc_collect_step(space, w_b) + self.descr_set_on_gc_collect(space, w_c) + + def descr_reset(self, space): + self.descr_set_on_gc_minor(space, space.w_None) + self.descr_set_on_gc_collect_step(space, space.w_None) + self.descr_set_on_gc_collect(space, space.w_None) + + +class GcMinorHookAction(AsyncAction): + total_memory_used = 0 + pinned_objects = 0 + + def __init__(self, space): + AsyncAction.__init__(self, space) + self.w_callable = space.w_None + self.reset() + + def reset(self): + self.count = 0 + self.duration = r_longlong(0) + self.duration_min = r_longlong(longlongmax) + self.duration_max = r_longlong(0) + + def fix_annotation(self): + # the annotation of the class and its attributes must be completed + # BEFORE we do the gc transform; this makes sure that everything is + # annotated with the correct types + if NonConstant(False): + self.count = NonConstant(-42) + self.duration = NonConstant(r_longlong(-42)) + self.duration_min = NonConstant(r_longlong(-42)) + self.duration_max = NonConstant(r_longlong(-42)) + self.total_memory_used = NonConstant(r_uint(42)) + self.pinned_objects = NonConstant(-42) + self.fire() + + def perform(self, ec, frame): + w_stats = W_GcMinorStats( + self.count, + self.duration, + self.duration_min, + self.duration_max, + self.total_memory_used, + self.pinned_objects) + self.reset() + self.space.call_function(self.w_callable, w_stats) + + +class GcCollectStepHookAction(AsyncAction): + oldstate = 0 + newstate = 0 + + def __init__(self, space): + AsyncAction.__init__(self, space) + self.w_callable = space.w_None + self.reset() + + def reset(self): + self.count = 0 + self.duration = r_longlong(0) + self.duration_min = r_longlong(longlongmax) + self.duration_max = r_longlong(0) + + def fix_annotation(self): + # the annotation of the class and its attributes must be completed + # BEFORE we do the gc transform; this makes sure that everything is + # annotated with the correct types + if NonConstant(False): + self.count = NonConstant(-42) + self.duration = NonConstant(r_longlong(-42)) + self.duration_min = NonConstant(r_longlong(-42)) + self.duration_max = NonConstant(r_longlong(-42)) + self.oldstate = NonConstant(-42) + self.newstate = NonConstant(-42) + self.fire() + + def perform(self, ec, frame): + w_stats = W_GcCollectStepStats( + self.count, + self.duration, + self.duration_min, + self.duration_max, + self.oldstate, + self.newstate) + self.reset() + self.space.call_function(self.w_callable, w_stats) + + +class GcCollectHookAction(AsyncAction): + num_major_collects = 0 + arenas_count_before = 0 + arenas_count_after = 0 + arenas_bytes = 0 + rawmalloc_bytes_before = 0 + rawmalloc_bytes_after = 0 + + def __init__(self, space): + AsyncAction.__init__(self, space) + self.w_callable = space.w_None + self.reset() + + def reset(self): + self.count = 0 + + def fix_annotation(self): + # the annotation of the class and its attributes must be completed + # BEFORE we do the gc transform; this makes sure that everything is + # annotated with the correct types + if NonConstant(False): + self.count = NonConstant(-42) + self.num_major_collects = NonConstant(-42) + self.arenas_count_before = NonConstant(-42) + self.arenas_count_after = NonConstant(-42) + self.arenas_bytes = NonConstant(r_uint(42)) + self.rawmalloc_bytes_before = NonConstant(r_uint(42)) + self.rawmalloc_bytes_after = NonConstant(r_uint(42)) + self.fire() + + def perform(self, ec, frame): + w_stats = W_GcCollectStats(self.count, + self.num_major_collects, + self.arenas_count_before, + self.arenas_count_after, + self.arenas_bytes, + self.rawmalloc_bytes_before, + self.rawmalloc_bytes_after) + self.reset() + self.space.call_function(self.w_callable, w_stats) + + +class W_GcMinorStats(W_Root): + + def __init__(self, count, duration, duration_min, duration_max, + total_memory_used, pinned_objects): + self.count = count + self.duration = duration + self.duration_min = duration_min + self.duration_max = duration_max + self.total_memory_used = total_memory_used + self.pinned_objects = pinned_objects + + +class W_GcCollectStepStats(W_Root): + + def __init__(self, count, duration, duration_min, duration_max, + oldstate, newstate): + self.count = count + self.duration = duration + self.duration_min = duration_min + self.duration_max = duration_max + self.oldstate = oldstate + self.newstate = newstate + + +class W_GcCollectStats(W_Root): + def __init__(self, count, num_major_collects, + arenas_count_before, arenas_count_after, + arenas_bytes, rawmalloc_bytes_before, + rawmalloc_bytes_after): + self.count = count + self.num_major_collects = num_major_collects + self.arenas_count_before = arenas_count_before + self.arenas_count_after = arenas_count_after + self.arenas_bytes = arenas_bytes + self.rawmalloc_bytes_before = rawmalloc_bytes_before + self.rawmalloc_bytes_after = rawmalloc_bytes_after + + +# just a shortcut to make the typedefs shorter +def wrap_many_ints(cls, names): + d = {} + for name in names: + d[name] = interp_attrproperty(name, cls=cls, wrapfn="newint") + return d + + +W_AppLevelHooks.typedef = TypeDef( + "GcHooks", + on_gc_minor = GetSetProperty( + W_AppLevelHooks.descr_get_on_gc_minor, + W_AppLevelHooks.descr_set_on_gc_minor), + + on_gc_collect_step = GetSetProperty( + W_AppLevelHooks.descr_get_on_gc_collect_step, + W_AppLevelHooks.descr_set_on_gc_collect_step), + + on_gc_collect = GetSetProperty( + W_AppLevelHooks.descr_get_on_gc_collect, + W_AppLevelHooks.descr_set_on_gc_collect), + + set = interp2app(W_AppLevelHooks.descr_set), + reset = interp2app(W_AppLevelHooks.descr_reset), + ) + +W_GcMinorStats.typedef = TypeDef( + "GcMinorStats", + **wrap_many_ints(W_GcMinorStats, ( + "count", + "duration", + "duration_min", + "duration_max", + "total_memory_used", + "pinned_objects")) + ) + +W_GcCollectStepStats.typedef = TypeDef( + "GcCollectStepStats", + STATE_SCANNING = incminimark.STATE_SCANNING, + STATE_MARKING = incminimark.STATE_MARKING, + STATE_SWEEPING = incminimark.STATE_SWEEPING, + STATE_FINALIZING = incminimark.STATE_FINALIZING, + GC_STATES = tuple(incminimark.GC_STATES), + **wrap_many_ints(W_GcCollectStepStats, ( + "count", + "duration", + "duration_min", + "duration_max", + "oldstate", + "newstate")) + ) + +W_GcCollectStats.typedef = TypeDef( + "GcCollectStats", + **wrap_many_ints(W_GcCollectStats, ( + "count", + "num_major_collects", + "arenas_count_before", + "arenas_count_after", + "arenas_bytes", + "rawmalloc_bytes_before", + "rawmalloc_bytes_after")) + ) diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py new file mode 100644 --- /dev/null +++ b/pypy/module/gc/test/test_hook.py @@ -0,0 +1,178 @@ +import pytest +from rpython.rlib.rarithmetic import r_uint +from pypy.module.gc.hook import LowLevelGcHooks +from pypy.interpreter.baseobjspace import ObjSpace +from pypy.interpreter.gateway import interp2app, unwrap_spec + +class AppTestGcHooks(object): + + def setup_class(cls): + if cls.runappdirect: + pytest.skip("these tests cannot work with -A") + space = cls.space + gchooks = space.fromcache(LowLevelGcHooks) + + @unwrap_spec(ObjSpace, int, r_uint, int) + def fire_gc_minor(space, duration, total_memory_used, pinned_objects): + gchooks.fire_gc_minor(duration, total_memory_used, pinned_objects) + + @unwrap_spec(ObjSpace, int, int, int) + def fire_gc_collect_step(space, duration, oldstate, newstate): + gchooks.fire_gc_collect_step(duration, oldstate, newstate) + + @unwrap_spec(ObjSpace, int, int, int, r_uint, r_uint, r_uint) + def fire_gc_collect(space, a, b, c, d, e, f): + gchooks.fire_gc_collect(a, b, c, d, e, f) + + @unwrap_spec(ObjSpace) + def fire_many(space): + gchooks.fire_gc_minor(5, 0, 0) + gchooks.fire_gc_minor(7, 0, 0) + gchooks.fire_gc_collect_step(5, 0, 0) + gchooks.fire_gc_collect_step(15, 0, 0) + gchooks.fire_gc_collect_step(22, 0, 0) + gchooks.fire_gc_collect(1, 2, 3, 4, 5, 6) + + cls.w_fire_gc_minor = space.wrap(interp2app(fire_gc_minor)) + cls.w_fire_gc_collect_step = space.wrap(interp2app(fire_gc_collect_step)) + cls.w_fire_gc_collect = space.wrap(interp2app(fire_gc_collect)) + cls.w_fire_many = space.wrap(interp2app(fire_many)) + + def test_default(self): + import gc + assert gc.hooks.on_gc_minor is None + assert gc.hooks.on_gc_collect_step is None + assert gc.hooks.on_gc_collect is None + + def test_on_gc_minor(self): + import gc + lst = [] + def on_gc_minor(stats): + lst.append((stats.count, + stats.duration, + stats.total_memory_used, + stats.pinned_objects)) + gc.hooks.on_gc_minor = on_gc_minor + self.fire_gc_minor(10, 20, 30) + self.fire_gc_minor(40, 50, 60) + assert lst == [ + (1, 10, 20, 30), + (1, 40, 50, 60), + ] + # + gc.hooks.on_gc_minor = None + self.fire_gc_minor(70, 80, 90) # won't fire because the hooks is disabled + assert lst == [ + (1, 10, 20, 30), + (1, 40, 50, 60), + ] + + def test_on_gc_collect_step(self): + import gc + lst = [] + def on_gc_collect_step(stats): + lst.append((stats.count, + stats.duration, + stats.oldstate, + stats.newstate)) + gc.hooks.on_gc_collect_step = on_gc_collect_step + self.fire_gc_collect_step(10, 20, 30) + self.fire_gc_collect_step(40, 50, 60) + assert lst == [ + (1, 10, 20, 30), + (1, 40, 50, 60), + ] + # + gc.hooks.on_gc_collect_step = None + self.fire_gc_collect_step(70, 80, 90) # won't fire + assert lst == [ + (1, 10, 20, 30), + (1, 40, 50, 60), + ] + + def test_on_gc_collect(self): + import gc + lst = [] + def on_gc_collect(stats): + lst.append((stats.count, + stats.num_major_collects, + stats.arenas_count_before, + stats.arenas_count_after, + stats.arenas_bytes, + stats.rawmalloc_bytes_before, + stats.rawmalloc_bytes_after)) + gc.hooks.on_gc_collect = on_gc_collect + self.fire_gc_collect(1, 2, 3, 4, 5, 6) + self.fire_gc_collect(7, 8, 9, 10, 11, 12) + assert lst == [ + (1, 1, 2, 3, 4, 5, 6), + (1, 7, 8, 9, 10, 11, 12), + ] + # + gc.hooks.on_gc_collect = None + self.fire_gc_collect(42, 42, 42, 42, 42, 42) # won't fire + assert lst == [ + (1, 1, 2, 3, 4, 5, 6), + (1, 7, 8, 9, 10, 11, 12), + ] + + def test_consts(self): + import gc + S = gc.GcCollectStepStats + assert S.STATE_SCANNING == 0 + assert S.STATE_MARKING == 1 + assert S.STATE_SWEEPING == 2 + assert S.STATE_FINALIZING == 3 + assert S.GC_STATES == ('SCANNING', 'MARKING', 'SWEEPING', 'FINALIZING') + + def test_cumulative(self): + import gc + class MyHooks(object): + + def __init__(self): + self.minors = [] + self.steps = [] + + def on_gc_minor(self, stats): + self.minors.append((stats.count, stats.duration, + stats.duration_min, stats.duration_max)) + + def on_gc_collect_step(self, stats): + self.steps.append((stats.count, stats.duration, + stats.duration_min, stats.duration_max)) + + on_gc_collect = None + + myhooks = MyHooks() + gc.hooks.set(myhooks) + self.fire_many() + assert myhooks.minors == [(2, 12, 5, 7)] + assert myhooks.steps == [(3, 42, 5, 22)] + + def test_clear_queue(self): + import gc + class MyHooks(object): + + def __init__(self): + self.lst = [] + + def on_gc_minor(self, stats): + self.lst.append('minor') + + def on_gc_collect_step(self, stats): + self.lst.append('step') + + def on_gc_collect(self, stats): + self.lst.append('collect') + + myhooks = MyHooks() + gc.hooks.set(myhooks) + self.fire_many() + assert myhooks.lst == ['minor', 'step', 'collect'] + myhooks.lst[:] = [] + self.fire_gc_minor(0, 0, 0) + assert myhooks.lst == ['minor'] + gc.hooks.reset() + assert gc.hooks.on_gc_minor is None + assert gc.hooks.on_gc_collect_step is None + assert gc.hooks.on_gc_collect is None diff --git a/pypy/module/gc/test/test_ztranslation.py b/pypy/module/gc/test/test_ztranslation.py --- a/pypy/module/gc/test/test_ztranslation.py +++ b/pypy/module/gc/test/test_ztranslation.py @@ -1,4 +1,9 @@ from pypy.objspace.fake.checkmodule import checkmodule def test_checkmodule(): - checkmodule('gc') + # we need to ignore GcCollectStepStats, else checkmodule fails. I think + # this happens because W_GcCollectStepStats.__init__ is only called from + # GcCollectStepHookAction.perform() and the fake objspace doesn't know + # about those: so, perform() is never annotated and the annotator thinks + # W_GcCollectStepStats has no attributes + checkmodule('gc', ignore=['GcCollectStepStats']) diff --git a/pypy/objspace/fake/checkmodule.py b/pypy/objspace/fake/checkmodule.py --- a/pypy/objspace/fake/checkmodule.py +++ b/pypy/objspace/fake/checkmodule.py @@ -4,6 +4,7 @@ def checkmodule(*modnames, **kwds): translate_startup = kwds.pop('translate_startup', True) + ignore = set(kwds.pop('ignore', ())) assert not kwds config = get_pypy_config(translating=True) space = FakeObjSpace(config) @@ -17,6 +18,8 @@ module.init(space) modules.append(module) for name in module.loaders: + if name in ignore: + continue seeobj_w.append(module._load_lazily(space, name)) if hasattr(module, 'submodules'): for cls in module.submodules.itervalues(): diff --git a/pypy/tool/pytest/genreportdata.py b/pypy/tool/pytest/genreportdata.py deleted file mode 100755 --- a/pypy/tool/pytest/genreportdata.py +++ /dev/null @@ -1,30 +0,0 @@ -#! /usr/bin/env python -import py -import sys - -mydir = py.path.local(__file__).dirpath().realpath() -from pypy.tool.pytest import htmlreport -from pypy.tool.pytest import confpath - -if __name__ == '__main__': - if len(sys.argv) > 1: - testresultdir = py.path.local(sys.argv[1]) - assert testresultdir.check(dir=1) - else: - testresultdir = confpath.testresultdir - assert testresultdir.check(dir=1) - try: - resultwc = py.path.svnwc(testresultdir) - print "updating", resultwc - resultwc.update() - except (KeyboardInterrupt, RuntimeError): - raise - except Exception as e: #py.process.ExecutionFailed,e: - print >> sys.stderr, "Warning: ",e #Subversion update failed" - - print "traversing", mydir - rep = htmlreport.HtmlReport(testresultdir) - rep.parselatest() - - print "making html files" - rep.makeindex(testresultdir.join('index.html')) diff --git a/pypy/tool/pytest/htmlreport.py b/pypy/tool/pytest/htmlreport.py deleted file mode 100644 --- a/pypy/tool/pytest/htmlreport.py +++ /dev/null @@ -1,239 +0,0 @@ -#! /usr/bin/env python - -""" -the html test reporter - -""" -import sys, os, re -import pprint -import py -from pypy.tool.pytest import result -from pypy.tool.pytest.overview import ResultCache - -# -# various interesting path objects -# - -html = py.xml.html -NBSP = py.xml.raw(" ") - -class HtmlReport(object): - def __init__(self, resultdir): - self.resultcache = ResultCache(resultdir) - - def parselatest(self): - self.resultcache.parselatest() - - # - # rendering - # - - def render_latest_table(self, results): - table = html.table( - [html.th(x, align='left') - for x in ("failure", "filename", "revision", - "user", "platform", "elapsed", - "options", "last error line" - )], - ) - r = results[:] - def f(x, y): - xnum = x.isok() and 1 or (x.istimeout() and 2 or 3) - ynum = y.isok() and 1 or (y.istimeout() and 2 or 3) - res = -cmp(xnum, ynum) - if res == 0: - return cmp(x['execution-time'], y['execution-time']) - return res - r.sort(f) - for result in r: - table.append(self.render_result_row(result)) - return table - - def render_result_row(self, result): - dp = py.path.local(result['fspath']) - - options = " ".join([x for x in result.get('options', []) if x!= 'core']) - if not options: - options = NBSP - - failureratio = 100 * (1.0 - result.ratio_of_passed()) - self.data[result.testname] = failureratio - return html.tr( - html.td("%.2f%%" % failureratio, - style = "background-color: %s" % (getresultcolor(result),)), - html.td(self.render_test_references(result)), - html.td(result['pypy-revision']), - html.td(result['userhost'][:15]), - html.td(result['platform']), - html.td("%.2fs" % result['execution-time']), - html.td(options), - html.td(result.repr_short_error() or NBSP) - ) - - def getrelpath(self, p): - return p.relto(self.indexpath.dirpath()) - - def render_test_references(self, result): - dest = self.make_single_test_result(result) - #XXX: ask hg for differences between test and vendor branch - modified = result.ismodifiedtest() and " [mod]" or "" - return html.div(html.a(result.path.purebasename + modified, - href=self.getrelpath(dest)), - style="background-color: transparent") - - def make_single_test_result(self, result): - cache = self.indexpath.dirpath('.cache', result['userhost'][:15]) - cache.ensure(dir=1) - dest = cache.join(result.path.basename).new(ext='.html') - doc = ViewResult(result) - doc.writetopath(dest) - return dest - - def getcorelists(self): - def iscore(result): - return 'core' in result.get('options', []) - coretests = [] - noncoretests = [] - for name in self.resultcache.getnames(): - result = self.resultcache.getlatestrelevant(name) - if iscore(result): - coretests.append(result) - else: - noncoretests.append(result) - return coretests, noncoretests - - # generate html files - # - def makeindex(self, indexpath, detail="PyPy - latest"): - self.indexpath = indexpath - self.data = {} - doc = Document(title='pypy test results') - body = doc.body - coretests, noncoretests = self.getcorelists() - body.append(html.h2("%s compliance test results - " - "core tests" % detail)) - - body.append(self.render_test_summary('core', coretests)) - body.append(self.render_latest_table(coretests)) - body.append( - html.h2("%s compliance test results - non-core tests" % detail)) - body.append(self.render_test_summary('noncore', noncoretests)) - body.append(self.render_latest_table(noncoretests)) - doc.writetopath(indexpath) - datapath = indexpath.dirpath().join('data') - d = datapath.open('w') - print >>d, "data = ", - pprint.pprint(self.data, stream=d) - d.close() - self.data = None - - def render_test_summary(self, tag, tests): - ok = len([x for x in tests if x.isok()]) - err = len([x for x in tests if x.iserror()]) - to = len([x for x in tests if x.istimeout()]) - numtests = ok + err + to - assert numtests == len(tests) - assert numtests - - t = html.table() - sum100 = numtests / 100.0 - def row(*args): - return html.tr(*[html.td(arg) for arg in args]) - - sum_passed = sum([x.ratio_of_passed() for x in tests]) - compliancy = sum_passed/sum100 - self.data['%s-compliancy' % tag] = compliancy - t.append(row(html.b("tests compliancy"), - html.b("%.2f%%" % (compliancy,)))) - - passed = ok/sum100 - self.data['%s-passed' % tag] = passed - t.append(row("testmodules passed completely", "%.2f%%" % passed)) - failed = err/sum100 - self.data['%s-failed' % tag] = failed - t.append(row("testmodules (partially) failed", "%.2f%%" % failed)) - timedout = to/sum100 - self.data['%s-timedout' % tag] = timedout - t.append(row("testmodules timeout", "%.2f%%" % timedout)) - return t - -class Document(object): - def __init__(self, title=None): - self.body = html.body() - self.head = html.head() - self.doc = html.html(self.head, self.body) - if title is not None: - self.head.append( - html.meta(name="title", content=title)) - self.head.append( - html.link(rel="Stylesheet", type="text/css", href="/pypy/default.css")) - - def writetopath(self, p): - assert p.ext == '.html' - self.head.append( - html.meta(name="Content-Type", content="text/html;charset=UTF-8") - ) - s = self.doc.unicode().encode('utf-8') - p.write(s) - -def getresultcolor(result): - if result.isok(): - color = "#00ee00" - elif result.iserror(): - color = "#ee0000" - elif result.istimeout: - color = "#0000ee" - else: - color = "#444444" - return color - -class ViewResult(Document): - def __init__(self, result): - title = "%s testresult" % (result.path.purebasename,) - super(ViewResult, self).__init__(title=title) - color = getresultcolor(result) - self.body.append(html.h2(title, - style="background-color: %s" % color)) - self.body.append(self.render_meta_info(result)) - - for name in ('reportdiff', 'stdout', 'stderr'): - try: - text = result.getnamedtext(name) - except KeyError: - continue - self.body.append(html.h3(name)) - self.body.append(html.pre(text)) - - def render_meta_info(self, result): - t = html.table() - items = result.items() - items.sort() - for name, value in items: - if name.lower() == name: - t.append(html.tr( - html.td(name), html.td(value))) - return t - -class TestOfHtmlReportClass: - def setup_class(cls): - py.test.skip('needs move to own test file') - cls.testresultdir = confpath.testresultdir - cls.rep = rep = HtmlReport() - rep.parse_all(cls.testresultdir) - - def test_pickling(self): - # test pickling of report - tempdir = py.test.ensuretemp('reportpickle') - picklepath = tempdir.join('report.pickle') - picklepath.dump(self.rep) - x = picklepath.load() - assert len(x.results) == len(self.rep.results) - - def test_render_latest(self): - t = self.rep.render_latest_table(self.rep.results) - assert unicode(t) - -mydir = py.path.local(__file__).dirpath() - -def getpicklepath(): - return mydir.join('.htmlreport.pickle') diff --git a/pypy/tool/pytest/overview.py b/pypy/tool/pytest/overview.py deleted file mode 100644 --- a/pypy/tool/pytest/overview.py +++ /dev/null @@ -1,56 +0,0 @@ -from pypy.tool.pytest import result -import sys - -class ResultCache: - def __init__(self, resultdir): - self.resultdir = resultdir - self.name2result = {} - - def parselatest(self): - def filefilter(p): - return p.check(fnmatch='test_*.txt', file=1) - def rec(p): - return p.check(dotfile=0) - for x in self.resultdir.visit(filefilter, rec): - self.parse_one(x) - - def parse_one(self, resultpath): - try: - res = result.ResultFromMime(resultpath) - ver = res['testreport-version'] - if ver != "1.1" and ver != "1.1.1": - raise TypeError - except TypeError: # xxx - print >>sys.stderr, "could not parse %s" % resultpath - return - name = res.testname - print name - self.name2result.setdefault(name, []).append(res) - return res - - def getnames(self): - return self.name2result.keys() - - def getlatest(self, name, timeout=0, error=0, ok=0): - l = [] - resultlist = self.name2result[name] - maxrev = 0 - maxresult = None - for res in resultlist: - resrev = res['pypy-revision'] - if resrev == 'unknown': - continue - if resrev <= maxrev: - continue - if timeout or error or ok: - if not (timeout and res.istimeout() or - error and res.iserror() or - ok and res.isok()): - continue - maxrev = resrev - maxresult = res - return maxresult - - def getlatestrelevant(self, name): - # get the latest revision that did not time out. - return self.getlatest(name, error=1, ok=1) or self.getlatest(name) diff --git a/pypy/tool/pytest/result.py b/pypy/tool/pytest/result.py deleted file mode 100644 --- a/pypy/tool/pytest/result.py +++ /dev/null @@ -1,215 +0,0 @@ -import sys -import py -import re - -class Result(object): - def __init__(self, init=True): - self._headers = {} - self._blocks = {} - self._blocknames = [] - if init: - stdinit(self) - - def __setitem__(self, name, value): - self._headers[name.lower()] = value - - def __getitem__(self, name): - return self._headers[name.lower()] - - def get(self, name, default): - return self._headers.get(name, default) - - def __delitem__(self, name): - del self._headers[name.lower()] - - def items(self): - return self._headers.items() - - def addnamedtext(self, name, text): - assert isinstance(text, basestring) - assert isinstance(name, str) - self._blocknames.append(name) - self._blocks[name] = text - - def getnamedtext(self, name): - return self._blocks[name] - - def repr_short_error(self): - if not self.isok(): - if 'reportdiff' in self._blocks: - return "output comparison failed, see reportdiff" - else: - text = self.getnamedtext('stderr') - lines = text.strip().split('\n') - if lines: - return lines[-1] - - def repr_mimemessage(self): - from email.MIMEMultipart import MIMEMultipart - from email.MIMEText import MIMEText - - outer = MIMEMultipart() - items = self._headers.items() - items.sort() - reprs = {} - for name, value in items: - assert ':' not in name - chars = map(ord, name) - assert min(chars) >= 33 and max(chars) <= 126 - outer[name] = str(value) - if not isinstance(value, str): - typename = type(value).__name__ - assert typename in vars(py.std.__builtin__) - reprs[name] = typename - - outer['_reprs'] = repr(reprs) - - for name in self._blocknames: - text = self._blocks[name] - m = MIMEText(text) - m.add_header('Content-Disposition', 'attachment', filename=name) - outer.attach(m) - return outer - - def grep_nr(self,text,section='stdout'): - stdout = self._blocks[section] - find = re.search('%s(?P\d+)'%text,stdout) - if find: - return float(find.group('nr')) - return 0. - - def ratio_of_passed(self): - if self.isok(): - return 1. - elif self.istimeout(): - return 0. - else: - nr = self.grep_nr('Ran ') - if nr > 0: - return (nr - (self.grep_nr('errors=') + self.grep_nr('failures=')))/nr - else: - passed = self.grep_nr('TestFailed: ',section='stderr') - run = self.grep_nr('TestFailed: \d+/',section='stderr') - if run > 0: - return passed/run - else: - run = self.grep_nr('TestFailed: \d+ of ',section='stderr') - if run > 0 : - return (run-passed)/run - else: - return 0.0 - - def isok(self): - return self['outcome'].lower() == 'ok' - - def iserror(self): - return self['outcome'].lower()[:3] == 'err' or self['outcome'].lower() == 'fail' - - def istimeout(self): - return self['outcome'].lower() == 't/o' - -# XXX backward compatibility -def sanitize(msg, path): - if 'exit-status' in msg.keys(): - return msg - f = open(str(path), 'r') - msg = f.read() - f.close() - for broken in ('exit status', 'cpu model', 'cpu mhz'): - valid = broken.replace(' ','-') - invalid = msg.find(broken+':') - msg = (msg[:invalid] + valid + - msg[invalid+len(valid):]) - from email import message_from_string - msg = message_from_string(msg) - return msg - -def sanitize_reprs(reprs): - if 'exit status' in reprs: - reprs['exit-status'] = reprs.pop('exit status') - -class ResultFromMime(Result): - def __init__(self, path): - super(ResultFromMime, self).__init__(init=False) - f = open(str(path), 'r') - from email import message_from_file - msg = message_from_file(f) - f.close() - msg = sanitize(msg, path) - # XXX security wise evil (keep in mind once we accept reporsts - # from anonymous - #print msg['_reprs'] - self._reprs = eval(msg['_reprs']) - del msg['_reprs'] - sanitize_reprs(self._reprs) - for name, value in msg.items(): - if name in self._reprs: - value = eval(value) # XXX security - self._headers[name] = value - self.fspath = self['fspath'] - if self['platform'] == 'win32' and '\\' in self.fspath: - self.testname = self.fspath.split('\\')[-1] - else: - self.testname = self.fspath.split('/')[-1] - #if sys.platform != 'win32' and '\\' in self.fspath: - # self.fspath = py.path.local(self['fspath'].replace('\\' - self.path = path - - payload = msg.get_payload() - if payload: - for submsg in payload: - assert submsg.get_content_type() == 'text/plain' - fn = submsg.get_filename() - assert fn - # XXX we need to deal better with encodings to - # begin with - content = submsg.get_payload() - for candidate in 'utf8', 'latin1': - try: - text = unicode(content, candidate) - except UnicodeDecodeError: - continue - else: - unicode(content, candidate) - self.addnamedtext(fn, text) - - def ismodifiedtest(self): - # XXX we need proper cross-platform paths! - return 'modified' in self.fspath - - def __repr__(self): - return '<%s (%s) %r rev=%s>' %(self.__class__.__name__, - self['outcome'], - self.fspath, - self['pypy-revision']) - -def stdinit(result): - import getpass - import socket - try: - username = getpass.getuser() - except: - username = 'unknown' - userhost = '%s@%s' % (username, socket.gethostname()) - result['testreport-version'] = "1.1.1" - result['userhost'] = userhost - result['platform'] = sys.platform - result['python-version-info'] = sys.version_info - info = try_getcpuinfo() - if info is not None: - result['cpu-model'] = info.get('model name', "unknown") - result['cpu-mhz'] = info.get('cpu mhz', 'unknown') -# -# -# -def try_getcpuinfo(): - if sys.platform.startswith('linux'): - cpuinfopath = py.path.local('/proc/cpuinfo') - if cpuinfopath.check(file=1): - d = {} - for line in cpuinfopath.readlines(): - if line.strip(): - name, value = line.split(':', 1) - name = name.strip().lower() - d[name] = value.strip() - return d diff --git a/pypy/tool/pytest/test/data/test___all__.txt b/pypy/tool/pytest/test/data/test___all__.txt deleted file mode 100644 --- a/pypy/tool/pytest/test/data/test___all__.txt +++ /dev/null @@ -1,94 +0,0 @@ -Content-Type: multipart/mixed; boundary="===============0790678169==" -MIME-Version: 1.0 -execution-time: 1445.14346004 -exit status: 1 -fspath: /Users/anderslehmann/pypy/lib-python/2.4.1/test/test___all__.py -options: ['core', '_sre'] -outcome: T/O -platform: darwin -pypy-revision: 16114 -python-version-info: (2, 4, 1, 'final', 0) -startdate: Wed Aug 17 23:51:59 2005 -testreport-version: 1.1 -timeout: 1369.0 -userhost: anderslehmann at anders-lehmanns-15-powerbook-g4.local -_reprs: {'execution-time': 'float', 'python-version-info': 'tuple', - 'options': 'list', 'timeout': 'float', 'pypy-revision': 'int', - 'exit status': 'int'} - ---===============0790678169== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stdout" - -test_all (__main__.AllTest) ... ERROR - -====================================================================== -ERROR: test_all (__main__.AllTest) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test___all__.py", line 163, in test_all - self.check_all("tty") - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test___all__.py", line 26, in check_all - "%s has no __all__ attribute" % modname) - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_support.py", line 208, in verify - raise TestFailed(reason) -TestFailed: tty has no __all__ attribute - ----------------------------------------------------------------------- - ---===============0790678169== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stderr" - -faking -Loading grammar /Users/anderslehmann/pypy/pypy/interpreter/pyparser/data/Grammar2.4 -faking -faking -faking -fake-wrapping interp file ', mode 'w' at 0x12068> -fake-wrapping interp file ', mode 'w' at 0x120b0> -fake-wrapping interp file ', mode 'r' at 0x12020> -faking -faking -faking -faking -==========================timedout========================== -Traceback (application-level): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test___all__.py", line 192 in - test_main() - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test___all__.py", line 189 in test_main - test_support.run_unittest(AllTest) - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_support.py", line 290 in run_unittest - run_suite(suite, testclass) - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_support.py", line 262 in run_suite - result = runner.run(suite) -Traceback (application-level): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/atexit.py", line 29 in _run_exitfuncs - print >> sys.stderr, "Error in atexit._run_exitfuncs:" -KeyboardInterrupt -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/pypy/tool/alarm.py", line 43, in ? - execfile(_main_with_alarm(finished)) - File "/Users/anderslehmann/pypy/pypy/bin/py.py", line 206, in ? - sys.exit(main_(sys.argv)) - File "/Users/anderslehmann/pypy/pypy/bin/py.py", line 115, in main_ - if not main.run_toplevel(space, doit, verbose=Options.verbose): - File "/Users/anderslehmann/pypy/pypy/interpreter/main.py", line 150, in run_toplevel - operationerr.print_application_traceback(space) - File "/Users/anderslehmann/pypy/pypy/interpreter/error.py", line 83, in print_application_traceback - self.print_app_tb_only(file) - File "/Users/anderslehmann/pypy/pypy/interpreter/error.py", line 104, in print_app_tb_only - l = linecache.getline(fname, lineno) - File "/Library/Frameworks/Python.framework/Versions/2.4/lib/python2.4/linecache.py", line 14, in getline - lines = getlines(filename) - File "/Library/Frameworks/Python.framework/Versions/2.4/lib/python2.4/linecache.py", line 40, in getlines - return updatecache(filename) - File "/Library/Frameworks/Python.framework/Versions/2.4/lib/python2.4/linecache.py", line 101, in updatecache - lines = fp.readlines() -KeyboardInterrupt - ---===============0790678169==-- \ No newline at end of file diff --git a/pypy/tool/pytest/test/data/test_compile.txt b/pypy/tool/pytest/test/data/test_compile.txt deleted file mode 100644 --- a/pypy/tool/pytest/test/data/test_compile.txt +++ /dev/null @@ -1,111 +0,0 @@ -Content-Type: multipart/mixed; boundary="===============2137793924==" -MIME-Version: 1.0 -execution-time: 34.8464071751 -exit status: 1 -fspath: /Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py -options: ['core', '_sre'] -outcome: ERR -platform: darwin -pypy-revision: 16114 -python-version-info: (2, 4, 1, 'final', 0) -startdate: Thu Aug 18 03:08:18 2005 -testreport-version: 1.1 -timeout: 1521.0 -userhost: anderslehmann at anders-lehmanns-15-powerbook-g4.local -_reprs: {'execution-time': 'float', 'python-version-info': 'tuple', - 'options': 'list', 'timeout': 'float', 'pypy-revision': 'int', - 'exit status': 'int'} - ---===============2137793924== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stdout" - -test_argument_handling (__main__.TestSpecifics) ... FAIL -test_argument_order (__main__.TestSpecifics) ... FAIL -test_complex_args (__main__.TestSpecifics) ... ok -test_debug_assignment (__main__.TestSpecifics) ... FAIL -test_duplicate_global_local (__main__.TestSpecifics) ... ok -test_exec_with_general_mapping_for_locals (__main__.TestSpecifics) ... ok -test_float_literals (__main__.TestSpecifics) ... ok -test_for_distinct_code_objects (__main__.TestSpecifics) ... ok -test_import (__main__.TestSpecifics) ... FAIL -test_indentation (__main__.TestSpecifics) ... ok -test_literals_with_leading_zeroes (__main__.TestSpecifics) ... ok -test_none_assignment (__main__.TestSpecifics) ... FAIL -test_sequence_unpacking_error (__main__.TestSpecifics) ... ok -test_syntax_error (__main__.TestSpecifics) ... ok -test_unary_minus (__main__.TestSpecifics) ... ok - -====================================================================== -FAIL: test_argument_handling (__main__.TestSpecifics) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 18, in test_argument_handling - self.assertRaises(SyntaxError, eval, 'lambda a,a:0') -AssertionError: SyntaxError not raised - -====================================================================== -FAIL: test_argument_order (__main__.TestSpecifics) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 127, in test_argument_order - self.fail("non-default args after default") -AssertionError: non-default args after default - -====================================================================== -FAIL: test_debug_assignment (__main__.TestSpecifics) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 10, in test_debug_assignment - self.assertRaises(SyntaxError, compile, '__debug__ = 1', '?', 'single') -AssertionError: SyntaxError not raised - -====================================================================== -FAIL: test_import (__main__.TestSpecifics) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 253, in test_import - self.assertRaises(SyntaxError, compile, stmt, 'tmp', 'exec') -AssertionError: SyntaxError not raised - -====================================================================== -FAIL: test_none_assignment (__main__.TestSpecifics) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 211, in test_none_assignment - self.assertRaises(SyntaxError, compile, stmt, 'tmp', 'single') -AssertionError: SyntaxError not raised - ----------------------------------------------------------------------- -Ran 15 tests in 14.363s - -FAILED (failures=5) - ---===============2137793924== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stderr" - -faking -Loading grammar /Users/anderslehmann/pypy/pypy/interpreter/pyparser/data/Grammar2.4 -faking -faking -faking -fake-wrapping interp file ', mode 'w' at 0x12068> -fake-wrapping interp file ', mode 'w' at 0x120b0> -fake-wrapping interp file ', mode 'r' at 0x12020> -Traceback (application-level): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 268 in - test_main() - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 265 in test_main - test_support.run_unittest(TestSpecifics) - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_support.py", line 290 in run_unittest - run_suite(suite, testclass) - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_support.py", line 274 in run_suite - raise TestFailed(msg) -TestFailed: errors occurred in __main__.TestSpecifics - ---===============2137793924==-- \ No newline at end of file diff --git a/pypy/tool/pytest/test/data/test_descr.txt b/pypy/tool/pytest/test/data/test_descr.txt deleted file mode 100644 --- a/pypy/tool/pytest/test/data/test_descr.txt +++ /dev/null From pypy.commits at gmail.com Fri Apr 20 07:26:18 2018 From: pypy.commits at gmail.com (mattip) Date: Fri, 20 Apr 2018 04:26:18 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: merge default into py3.5 Message-ID: <5ad9ce5a.1c69fb81.1d53.2555@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r94393:580e3e26cd32 Date: 2018-04-20 14:21 +0300 http://bitbucket.org/pypy/pypy/changeset/580e3e26cd32/ Log: merge default into py3.5 diff --git a/pypy/doc/release-v6.0.0.rst b/pypy/doc/release-v6.0.0.rst --- a/pypy/doc/release-v6.0.0.rst +++ b/pypy/doc/release-v6.0.0.rst @@ -18,6 +18,8 @@ getting started writing code. We have improved our parser to emit more friendly `syntax errors`_, making PyPy not only faster but more friendly. +The GC now has `hooks`_ to gain more insights into its performance + The Windows PyPy3.5 release is still considered beta-quality. There are open issues with unicode handling especially around system calls and c-extensions. @@ -53,6 +55,7 @@ .. _`blog post`: https://morepypy.blogspot.it/2017/10/cape-of-good-hope-for-pypy-hello-from.html .. _pygobject: https://lazka.github.io/posts/2018-04_pypy-pygobject/index.html .. _`syntax errors`: https://morepypy.blogspot.com/2018/04/improving-syntaxerror-in-pypy.html +.. _`hooks`: gc_info.html#gc-hooks What is PyPy? ============= @@ -101,8 +104,9 @@ * Added missing attributes to C-API ``instancemethod`` on pypy3 * Store error state in thread-local storage for C-API. * Fix JIT bugs exposed in the sre module -* Improve speed of Python parser, improve ParseError messages slightly +* Improve speed of Python parser, improve ParseError messages and SyntaxError * Handle JIT hooks more efficiently +* Fix a rare GC bug exposed by intensive use of cpyext `Buffer` s We also refactored many parts of the JIT bridge optimizations, as well as cpyext internals, and together with new contributors fixed issues, added new diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -3,18 +3,7 @@ ========================== .. this is a revision shortly after release-pypy-6.0.0 -.. startrev: f22145c34985 +.. startrev: ad79cc0ce9a8 -.. branch: issue2752 -Fix a rare GC bug that was introduced more than one year ago, but was -not diagnosed before issue #2752. - -.. branch: gc-hooks - -Introduce GC hooks, as documented in doc/gc_info.rst - -.. branch: gc-hook-better-timestamp - -Improve GC hooks diff --git a/pypy/doc/whatsnew-pypy2-6.0.0.rst b/pypy/doc/whatsnew-pypy2-6.0.0.rst --- a/pypy/doc/whatsnew-pypy2-6.0.0.rst +++ b/pypy/doc/whatsnew-pypy2-6.0.0.rst @@ -113,3 +113,16 @@ Improve line offsets that are reported by SyntaxError. Improve error messages for a few situations, including mismatched parenthesis. + +.. branch: issue2752 + +Fix a rare GC bug that was introduced more than one year ago, but was +not diagnosed before issue #2752. + +.. branch: gc-hooks + +Introduce GC hooks, as documented in doc/gc_info.rst + +.. branch: gc-hook-better-timestamp + +Improve GC hooksd From pypy.commits at gmail.com Fri Apr 20 07:26:21 2018 From: pypy.commits at gmail.com (mattip) Date: Fri, 20 Apr 2018 04:26:21 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: restart whatsnew-pypy3-head Message-ID: <5ad9ce5d.5c6f1c0a.a6854.8d90@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r94394:95916fa94a60 Date: 2018-04-20 14:23 +0300 http://bitbucket.org/pypy/pypy/changeset/95916fa94a60/ Log: restart whatsnew-pypy3-head diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst --- a/pypy/doc/whatsnew-pypy3-head.rst +++ b/pypy/doc/whatsnew-pypy3-head.rst @@ -3,5 +3,5 @@ ======================== .. this is the revision after release-pypy3.5-v6.0 -.. startrev: bf74662ee4fa +.. startrev: 580e3e26cd32 From pypy.commits at gmail.com Fri Apr 20 07:26:23 2018 From: pypy.commits at gmail.com (mattip) Date: Fri, 20 Apr 2018 04:26:23 -0700 (PDT) Subject: [pypy-commit] pypy release-pypy3.5-6.x: merge py3.5 into release Message-ID: <5ad9ce5f.1c69fb81.83843.7d0e@mx.google.com> Author: Matti Picus Branch: release-pypy3.5-6.x Changeset: r94395:aa0e948e1bc4 Date: 2018-04-20 14:25 +0300 http://bitbucket.org/pypy/pypy/changeset/aa0e948e1bc4/ Log: merge py3.5 into release diff too long, truncating to 2000 out of 4118 lines diff --git a/lib-python/3/test/test_exceptions.py b/lib-python/3/test/test_exceptions.py --- a/lib-python/3/test/test_exceptions.py +++ b/lib-python/3/test/test_exceptions.py @@ -164,10 +164,10 @@ is_pypy = check_impl_detail(pypy=True) check('def fact(x):\n\treturn x!\n', 2, 10) - check('1 +\n', 1, 4 - is_pypy) - check('def spam():\n print(1)\n print(2)', 3, 0 if is_pypy else 10) - check('Python = "Python" +', 1, 20 - is_pypy) - check('Python = "\u1e54\xfd\u0163\u0125\xf2\xf1" +', 1, 20 - is_pypy) + check('1 +\n', 1, 4) + check('def spam():\n print(1)\n print(2)', 3, 2 if is_pypy else 10) + check('Python = "Python" +', 1, 20) + check('Python = "\u1e54\xfd\u0163\u0125\xf2\xf1" +', 1, 20) @cpython_only def testSettingException(self): diff --git a/lib-python/3/test/test_fstring.py b/lib-python/3/test/test_fstring.py --- a/lib-python/3/test/test_fstring.py +++ b/lib-python/3/test/test_fstring.py @@ -319,7 +319,7 @@ ["f'{3)+(4}'", ]) - self.assertAllRaise(SyntaxError, 'EOL while scanning string literal', + self.assertAllRaise(SyntaxError, r'end of line \(EOL\) while scanning string literal', ["f'{\n}'", ]) @@ -741,7 +741,7 @@ self.assertEqual('{d[0]}'.format(d=d), 'integer') def test_invalid_expressions(self): - self.assertAllRaise(SyntaxError, 'invalid syntax', + self.assertAllRaise(SyntaxError, "closing parenthesis '.' does not match opening parenthesis '.'", [r"f'{a[4)}'", r"f'{a(4]}'", ]) diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst --- a/pypy/doc/gc_info.rst +++ b/pypy/doc/gc_info.rst @@ -121,6 +121,166 @@ alive by GC objects, but not accounted in the GC +GC Hooks +-------- + +GC hooks are user-defined functions which are called whenever a specific GC +event occur, and can be used to monitor GC activity and pauses. You can +install the hooks by setting the following attributes: + +``gc.hook.on_gc_minor`` + Called whenever a minor collection occurs. It corresponds to + ``gc-minor`` sections inside ``PYPYLOG``. + +``gc.hook.on_gc_collect_step`` + Called whenever an incremental step of a major collection occurs. It + corresponds to ``gc-collect-step`` sections inside ``PYPYLOG``. + +``gc.hook.on_gc_collect`` + Called after the last incremental step, when a major collection is fully + done. It corresponds to ``gc-collect-done`` sections inside ``PYPYLOG``. + +To uninstall a hook, simply set the corresponding attribute to ``None``. To +install all hooks at once, you can call ``gc.hooks.set(obj)``, which will look +for methods ``on_gc_*`` on ``obj``. To uninstall all the hooks at once, you +can call ``gc.hooks.reset()``. + +The functions called by the hooks receive a single ``stats`` argument, which +contains various statistics about the event. + +Note that PyPy cannot call the hooks immediately after a GC event, but it has +to wait until it reaches a point in which the interpreter is in a known state +and calling user-defined code is harmless. It might happen that multiple +events occur before the hook is invoked: in this case, you can inspect the +value ``stats.count`` to know how many times the event occured since the last +time the hook was called. Similarly, ``stats.duration`` contains the +**total** time spent by the GC for this specific event since the last time the +hook was called. + +On the other hand, all the other fields of the ``stats`` object are relative +only to the **last** event of the series. + +The attributes for ``GcMinorStats`` are: + +``count`` + The number of minor collections occured since the last hook call. + +``duration`` + The total time spent inside minor collections since the last hook + call. See below for more information on the unit. + +``duration_min`` + The duration of the fastest minor collection since the last hook call. + +``duration_max`` + The duration of the slowest minor collection since the last hook call. + + ``total_memory_used`` + The amount of memory used at the end of the minor collection, in + bytes. This include the memory used in arenas (for GC-managed memory) and + raw-malloced memory (e.g., the content of numpy arrays). + +``pinned_objects`` + the number of pinned objects. + + +The attributes for ``GcCollectStepStats`` are: + +``count``, ``duration``, ``duration_min``, ``duration_max`` + See above. + +``oldstate``, ``newstate`` + Integers which indicate the state of the GC before and after the step. + +The value of ``oldstate`` and ``newstate`` is one of these constants, defined +inside ``gc.GcCollectStepStats``: ``STATE_SCANNING``, ``STATE_MARKING``, +``STATE_SWEEPING``, ``STATE_FINALIZING``. It is possible to get a string +representation of it by indexing the ``GC_STATS`` tuple. + + +The attributes for ``GcCollectStats`` are: + +``count`` + See above. + +``num_major_collects`` + The total number of major collections which have been done since the + start. Contrarily to ``count``, this is an always-growing counter and it's + not reset between invocations. + +``arenas_count_before``, ``arenas_count_after`` + Number of arenas used before and after the major collection. + +``arenas_bytes`` + Total number of bytes used by GC-managed objects. + +``rawmalloc_bytes_before``, ``rawmalloc_bytes_after`` + Total number of bytes used by raw-malloced objects, before and after the + major collection. + +Note that ``GcCollectStats`` has **not** got a ``duration`` field. This is +because all the GC work is done inside ``gc-collect-step``: +``gc-collect-done`` is used only to give additional stats, but doesn't do any +actual work. + +A note about the ``duration`` field: depending on the architecture and +operating system, PyPy uses different ways to read timestamps, so ``duration`` +is expressed in varying units. It is possible to know which by calling +``__pypy__.debug_get_timestamp_unit()``, which can be one of the following +values: + +``tsc`` + The default on ``x86`` machines: timestamps are expressed in CPU ticks, as + read by the `Time Stamp Counter`_. + +``ns`` + Timestamps are expressed in nanoseconds. + +``QueryPerformanceCounter`` + On Windows, in case for some reason ``tsc`` is not available: timestamps + are read using the win API ``QueryPerformanceCounter()``. + + +Unfortunately, there does not seem to be a reliable standard way for +converting ``tsc`` ticks into nanoseconds, although in practice on modern CPUs +it is enough to divide the ticks by the maximum nominal frequency of the CPU. +For this reason, PyPy gives the raw value, and leaves the job of doing the +conversion to external libraries. + +Here is an example of GC hooks in use:: + + import sys + import gc + + class MyHooks(object): + done = False + + def on_gc_minor(self, stats): + print 'gc-minor: count = %02d, duration = %d' % (stats.count, + stats.duration) + + def on_gc_collect_step(self, stats): + old = gc.GcCollectStepStats.GC_STATES[stats.oldstate] + new = gc.GcCollectStepStats.GC_STATES[stats.newstate] + print 'gc-collect-step: %s --> %s' % (old, new) + print ' count = %02d, duration = %d' % (stats.count, + stats.duration) + + def on_gc_collect(self, stats): + print 'gc-collect-done: count = %02d' % stats.count + self.done = True + + hooks = MyHooks() + gc.hooks.set(hooks) + + # simulate some GC activity + lst = [] + while not hooks.done: + lst = [lst, 1, 2, 3] + + +.. _`Time Stamp Counter`: https://en.wikipedia.org/wiki/Time_Stamp_Counter + .. _minimark-environment-variables: Environment variables diff --git a/pypy/doc/release-v6.0.0.rst b/pypy/doc/release-v6.0.0.rst --- a/pypy/doc/release-v6.0.0.rst +++ b/pypy/doc/release-v6.0.0.rst @@ -18,6 +18,8 @@ getting started writing code. We have improved our parser to emit more friendly `syntax errors`_, making PyPy not only faster but more friendly. +The GC now has `hooks`_ to gain more insights into its performance + The Windows PyPy3.5 release is still considered beta-quality. There are open issues with unicode handling especially around system calls and c-extensions. @@ -53,6 +55,7 @@ .. _`blog post`: https://morepypy.blogspot.it/2017/10/cape-of-good-hope-for-pypy-hello-from.html .. _pygobject: https://lazka.github.io/posts/2018-04_pypy-pygobject/index.html .. _`syntax errors`: https://morepypy.blogspot.com/2018/04/improving-syntaxerror-in-pypy.html +.. _`hooks`: gc_info.html#gc-hooks What is PyPy? ============= @@ -101,8 +104,9 @@ * Added missing attributes to C-API ``instancemethod`` on pypy3 * Store error state in thread-local storage for C-API. * Fix JIT bugs exposed in the sre module -* Improve speed of Python parser, improve ParseError messages slightly +* Improve speed of Python parser, improve ParseError messages and SyntaxError * Handle JIT hooks more efficiently +* Fix a rare GC bug exposed by intensive use of cpyext `Buffer` s We also refactored many parts of the JIT bridge optimizations, as well as cpyext internals, and together with new contributors fixed issues, added new diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -3,6 +3,7 @@ ========================== .. this is a revision shortly after release-pypy-6.0.0 -.. startrev: f22145c34985 +.. startrev: ad79cc0ce9a8 + diff --git a/pypy/doc/whatsnew-pypy2-6.0.0.rst b/pypy/doc/whatsnew-pypy2-6.0.0.rst --- a/pypy/doc/whatsnew-pypy2-6.0.0.rst +++ b/pypy/doc/whatsnew-pypy2-6.0.0.rst @@ -113,3 +113,16 @@ Improve line offsets that are reported by SyntaxError. Improve error messages for a few situations, including mismatched parenthesis. + +.. branch: issue2752 + +Fix a rare GC bug that was introduced more than one year ago, but was +not diagnosed before issue #2752. + +.. branch: gc-hooks + +Introduce GC hooks, as documented in doc/gc_info.rst + +.. branch: gc-hook-better-timestamp + +Improve GC hooksd diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst --- a/pypy/doc/whatsnew-pypy3-head.rst +++ b/pypy/doc/whatsnew-pypy3-head.rst @@ -3,5 +3,5 @@ ======================== .. this is the revision after release-pypy3.5-v6.0 -.. startrev: bf74662ee4fa +.. startrev: 580e3e26cd32 diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -223,6 +223,7 @@ usage = SUPPRESS_USAGE take_options = True + space = None def opt_parser(self, config): parser = to_optparse(config, useoptions=["objspace.*"], @@ -382,15 +383,21 @@ from pypy.module.pypyjit.hooks import pypy_hooks return PyPyJitPolicy(pypy_hooks) + def get_gchooks(self): + from pypy.module.gc.hook import LowLevelGcHooks + if self.space is None: + raise Exception("get_gchooks must be called afeter get_entry_point") + return self.space.fromcache(LowLevelGcHooks) + def get_entry_point(self, config): - space = make_objspace(config) + self.space = make_objspace(config) # manually imports app_main.py filename = os.path.join(pypydir, 'interpreter', 'app_main.py') app = gateway.applevel(open(filename).read(), 'app_main.py', 'app_main') app.hidden_applevel = False - w_dict = app.getwdict(space) - entry_point, _ = create_entry_point(space, w_dict) + w_dict = app.getwdict(self.space) + entry_point, _ = create_entry_point(self.space, w_dict) return entry_point, None, PyPyAnnotatorPolicy() @@ -399,7 +406,7 @@ 'jitpolicy', 'get_entry_point', 'get_additional_config_options']: ns[name] = getattr(self, name) - + ns['get_gchooks'] = self.get_gchooks PyPyTarget().interface(globals()) diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -392,7 +392,7 @@ self._periodic_actions = [] self._nonperiodic_actions = [] self.has_bytecode_counter = False - self.fired_actions = None + self._fired_actions_reset() # the default value is not 100, unlike CPython 2.7, but a much # larger value, because we use a technique that not only allows # but actually *forces* another thread to run whenever the counter @@ -404,13 +404,28 @@ """Request for the action to be run before the next opcode.""" if not action._fired: action._fired = True - if self.fired_actions is None: - self.fired_actions = [] - self.fired_actions.append(action) + self._fired_actions_append(action) # set the ticker to -1 in order to force action_dispatcher() # to run at the next possible bytecode self.reset_ticker(-1) + def _fired_actions_reset(self): + # linked list of actions. We cannot use a normal RPython list because + # we want AsyncAction.fire() to be marked as @rgc.collect: this way, + # we can call it from e.g. GcHooks or cpyext's dealloc_trigger. + self._fired_actions_first = None + self._fired_actions_last = None + + @rgc.no_collect + def _fired_actions_append(self, action): + assert action._next is None + if self._fired_actions_first is None: + self._fired_actions_first = action + self._fired_actions_last = action + else: + self._fired_actions_last._next = action + self._fired_actions_last = action + @not_rpython def register_periodic_action(self, action, use_bytecode_counter): """ @@ -455,9 +470,9 @@ action.perform(ec, frame) # nonperiodic actions - list = self.fired_actions - if list is not None: - self.fired_actions = None + action = self._fired_actions_first + if action: + self._fired_actions_reset() # NB. in case there are several actions, we reset each # 'action._fired' to false only when we're about to call # 'action.perform()'. This means that if @@ -465,9 +480,10 @@ # the corresponding perform(), the fire() has no # effect---which is the effect we want, because # perform() will be called anyway. - for action in list: + while action is not None: action._fired = False action.perform(ec, frame) + action._next, action = None, action._next self.action_dispatcher = action_dispatcher @@ -500,10 +516,12 @@ to occur between two opcodes, not at a completely random time. """ _fired = False + _next = None def __init__(self, space): self.space = space + @rgc.no_collect def fire(self): """Request for the action to be run before the next opcode. The action must have been registered at space initalization time.""" diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py --- a/pypy/interpreter/test/test_executioncontext.py +++ b/pypy/interpreter/test/test_executioncontext.py @@ -36,6 +36,37 @@ pass assert i == 9 + def test_action_queue(self): + events = [] + + class Action1(executioncontext.AsyncAction): + def perform(self, ec, frame): + events.append('one') + + class Action2(executioncontext.AsyncAction): + def perform(self, ec, frame): + events.append('two') + + space = self.space + a1 = Action1(space) + a2 = Action2(space) + a1.fire() + a2.fire() + space.appexec([], """(): + n = 5 + return n + 2 + """) + assert events == ['one', 'two'] + # + events[:] = [] + a1.fire() + space.appexec([], """(): + n = 5 + return n + 2 + """) + assert events == ['one'] + + def test_periodic_action(self): from pypy.interpreter.executioncontext import ActionFlag diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -83,6 +83,8 @@ 'debug_stop' : 'interp_debug.debug_stop', 'debug_print_once' : 'interp_debug.debug_print_once', 'debug_flush' : 'interp_debug.debug_flush', + 'debug_read_timestamp' : 'interp_debug.debug_read_timestamp', + 'debug_get_timestamp_unit' : 'interp_debug.debug_get_timestamp_unit', 'builtinify' : 'interp_magic.builtinify', 'hidden_applevel' : 'interp_magic.hidden_applevel', 'lookup_special' : 'interp_magic.lookup_special', diff --git a/pypy/module/__pypy__/interp_debug.py b/pypy/module/__pypy__/interp_debug.py --- a/pypy/module/__pypy__/interp_debug.py +++ b/pypy/module/__pypy__/interp_debug.py @@ -1,11 +1,14 @@ from pypy.interpreter.gateway import unwrap_spec from rpython.rlib import debug, jit - +from rpython.rlib import rtimer @jit.dont_look_inside - at unwrap_spec(category='text') -def debug_start(space, category): - debug.debug_start(category) + at unwrap_spec(category='text', timestamp=bool) +def debug_start(space, category, timestamp=False): + res = debug.debug_start(category, timestamp=timestamp) + if timestamp: + return space.newint(res) + return space.w_None @jit.dont_look_inside def debug_print(space, args_w): @@ -13,10 +16,12 @@ debug.debug_print(' '.join(parts)) @jit.dont_look_inside - at unwrap_spec(category='text') -def debug_stop(space, category): - debug.debug_stop(category) - + at unwrap_spec(category='text', timestamp=bool) +def debug_stop(space, category, timestamp=False): + res = debug.debug_stop(category, timestamp=timestamp) + if timestamp: + return space.newint(res) + return space.w_None @unwrap_spec(category='text') def debug_print_once(space, category, args_w): @@ -28,3 +33,18 @@ @jit.dont_look_inside def debug_flush(space): debug.debug_flush() + +def debug_read_timestamp(space): + return space.newint(rtimer.read_timestamp()) + +def debug_get_timestamp_unit(space): + unit = rtimer.get_timestamp_unit() + if unit == rtimer.UNIT_TSC: + unit_str = 'tsc' + elif unit == rtimer.UNIT_NS: + unit_str = 'ns' + elif unit == rtimer.UNIT_QUERY_PERFORMANCE_COUNTER: + unit_str = 'QueryPerformanceCounter' + else: + unit_str = 'UNKNOWN(%d)' % unit + return space.newtext(unit_str) diff --git a/pypy/module/__pypy__/test/test_debug.py b/pypy/module/__pypy__/test/test_debug.py --- a/pypy/module/__pypy__/test/test_debug.py +++ b/pypy/module/__pypy__/test/test_debug.py @@ -48,3 +48,26 @@ from __pypy__ import debug_flush debug_flush() # assert did not crash + + def test_debug_read_timestamp(self): + from __pypy__ import debug_read_timestamp + a = debug_read_timestamp() + b = debug_read_timestamp() + assert b > a + + def test_debug_get_timestamp_unit(self): + from __pypy__ import debug_get_timestamp_unit + unit = debug_get_timestamp_unit() + assert unit in ('tsc', 'ns', 'QueryPerformanceCounter') + + def test_debug_start_stop_timestamp(self): + import time + from __pypy__ import debug_start, debug_stop, debug_read_timestamp + assert debug_start('foo') is None + assert debug_stop('foo') is None + ts1 = debug_start('foo', timestamp=True) + t = time.time() + while time.time() - t < 0.02: + pass + ts2 = debug_stop('foo', timestamp=True) + assert ts2 > ts1 diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py --- a/pypy/module/_cffi_backend/ctypearray.py +++ b/pypy/module/_cffi_backend/ctypearray.py @@ -70,7 +70,15 @@ length = wchar_helper.unicode_size_as_char32(u) return (w_value, length + 1) else: - explicitlength = space.getindex_w(w_value, space.w_OverflowError) + try: + explicitlength = space.getindex_w(w_value, + space.w_OverflowError) + except OperationError as e: + if e.match(space, space.w_TypeError): + raise oefmt(space.w_TypeError, + "expected new array length or list/tuple/str, " + "not %T", w_value) + raise if explicitlength < 0: raise oefmt(space.w_ValueError, "negative array length") return (space.w_None, explicitlength) diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -278,6 +278,9 @@ assert repr(q).startswith("" - + assert Sub.__module__ == __name__ diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py --- a/pypy/module/cpyext/test/test_unicodeobject.py +++ b/pypy/module/cpyext/test/test_unicodeobject.py @@ -27,11 +27,8 @@ if(PyUnicode_GetSize(s) != 11) { result = -PyUnicode_GetSize(s); } -#ifdef PYPY_VERSION - // Slightly silly test that tp_basicsize is reasonable. - if(s->ob_type->tp_basicsize != sizeof(void*)*12) + if(s->ob_type->tp_basicsize != sizeof(PyUnicodeObject)) result = s->ob_type->tp_basicsize; -#endif // PYPY_VERSION Py_DECREF(s); return PyLong_FromLong(result); """), diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py --- a/pypy/module/gc/__init__.py +++ b/pypy/module/gc/__init__.py @@ -34,5 +34,7 @@ 'get_typeids_z': 'referents.get_typeids_z', 'get_typeids_list': 'referents.get_typeids_list', 'GcRef': 'referents.W_GcRef', + 'hooks': 'space.fromcache(hook.W_AppLevelHooks)', + 'GcCollectStepStats': 'hook.W_GcCollectStepStats', }) MixedModule.__init__(self, space, w_name) diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py new file mode 100644 --- /dev/null +++ b/pypy/module/gc/hook.py @@ -0,0 +1,341 @@ +from rpython.memory.gc.hook import GcHooks +from rpython.memory.gc import incminimark +from rpython.rlib.nonconst import NonConstant +from rpython.rlib.rarithmetic import r_uint, r_longlong, longlongmax +from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault +from pypy.interpreter.baseobjspace import W_Root +from pypy.interpreter.typedef import TypeDef, interp_attrproperty, GetSetProperty +from pypy.interpreter.executioncontext import AsyncAction + +class LowLevelGcHooks(GcHooks): + """ + These are the low-level hooks which are called directly from the GC. + + They can't do much, because the base class marks the methods as + @rgc.no_collect. + + This is expected to be a singleton, created by space.fromcache, and it is + integrated with the translation by targetpypystandalone.get_gchooks + """ + + def __init__(self, space): + self.space = space + self.w_hooks = space.fromcache(W_AppLevelHooks) + + def is_gc_minor_enabled(self): + return self.w_hooks.gc_minor_enabled + + def is_gc_collect_step_enabled(self): + return self.w_hooks.gc_collect_step_enabled + + def is_gc_collect_enabled(self): + return self.w_hooks.gc_collect_enabled + + def on_gc_minor(self, duration, total_memory_used, pinned_objects): + action = self.w_hooks.gc_minor + action.count += 1 + action.duration += duration + action.duration_min = min(action.duration_min, duration) + action.duration_max = max(action.duration_max, duration) + action.total_memory_used = total_memory_used + action.pinned_objects = pinned_objects + action.fire() + + def on_gc_collect_step(self, duration, oldstate, newstate): + action = self.w_hooks.gc_collect_step + action.count += 1 + action.duration += duration + action.duration_min = min(action.duration_min, duration) + action.duration_max = max(action.duration_max, duration) + action.oldstate = oldstate + action.newstate = newstate + action.fire() + + def on_gc_collect(self, num_major_collects, + arenas_count_before, arenas_count_after, + arenas_bytes, rawmalloc_bytes_before, + rawmalloc_bytes_after): + action = self.w_hooks.gc_collect + action.count += 1 + action.num_major_collects = num_major_collects + action.arenas_count_before = arenas_count_before + action.arenas_count_after = arenas_count_after + action.arenas_bytes = arenas_bytes + action.rawmalloc_bytes_before = rawmalloc_bytes_before + action.rawmalloc_bytes_after = rawmalloc_bytes_after + action.fire() + + +class W_AppLevelHooks(W_Root): + + def __init__(self, space): + self.space = space + self.gc_minor_enabled = False + self.gc_collect_step_enabled = False + self.gc_collect_enabled = False + self.gc_minor = GcMinorHookAction(space) + self.gc_collect_step = GcCollectStepHookAction(space) + self.gc_collect = GcCollectHookAction(space) + + def descr_get_on_gc_minor(self, space): + return self.gc_minor.w_callable + + def descr_set_on_gc_minor(self, space, w_obj): + self.gc_minor_enabled = not space.is_none(w_obj) + self.gc_minor.w_callable = w_obj + self.gc_minor.fix_annotation() + + def descr_get_on_gc_collect_step(self, space): + return self.gc_collect_step.w_callable + + def descr_set_on_gc_collect_step(self, space, w_obj): + self.gc_collect_step_enabled = not space.is_none(w_obj) + self.gc_collect_step.w_callable = w_obj + self.gc_collect_step.fix_annotation() + + def descr_get_on_gc_collect(self, space): + return self.gc_collect.w_callable + + def descr_set_on_gc_collect(self, space, w_obj): + self.gc_collect_enabled = not space.is_none(w_obj) + self.gc_collect.w_callable = w_obj + self.gc_collect.fix_annotation() + + def descr_set(self, space, w_obj): + w_a = space.getattr(w_obj, space.newtext('on_gc_minor')) + w_b = space.getattr(w_obj, space.newtext('on_gc_collect_step')) + w_c = space.getattr(w_obj, space.newtext('on_gc_collect')) + self.descr_set_on_gc_minor(space, w_a) + self.descr_set_on_gc_collect_step(space, w_b) + self.descr_set_on_gc_collect(space, w_c) + + def descr_reset(self, space): + self.descr_set_on_gc_minor(space, space.w_None) + self.descr_set_on_gc_collect_step(space, space.w_None) + self.descr_set_on_gc_collect(space, space.w_None) + + +class GcMinorHookAction(AsyncAction): + total_memory_used = 0 + pinned_objects = 0 + + def __init__(self, space): + AsyncAction.__init__(self, space) + self.w_callable = space.w_None + self.reset() + + def reset(self): + self.count = 0 + self.duration = r_longlong(0) + self.duration_min = r_longlong(longlongmax) + self.duration_max = r_longlong(0) + + def fix_annotation(self): + # the annotation of the class and its attributes must be completed + # BEFORE we do the gc transform; this makes sure that everything is + # annotated with the correct types + if NonConstant(False): + self.count = NonConstant(-42) + self.duration = NonConstant(r_longlong(-42)) + self.duration_min = NonConstant(r_longlong(-42)) + self.duration_max = NonConstant(r_longlong(-42)) + self.total_memory_used = NonConstant(r_uint(42)) + self.pinned_objects = NonConstant(-42) + self.fire() + + def perform(self, ec, frame): + w_stats = W_GcMinorStats( + self.count, + self.duration, + self.duration_min, + self.duration_max, + self.total_memory_used, + self.pinned_objects) + self.reset() + self.space.call_function(self.w_callable, w_stats) + + +class GcCollectStepHookAction(AsyncAction): + oldstate = 0 + newstate = 0 + + def __init__(self, space): + AsyncAction.__init__(self, space) + self.w_callable = space.w_None + self.reset() + + def reset(self): + self.count = 0 + self.duration = r_longlong(0) + self.duration_min = r_longlong(longlongmax) + self.duration_max = r_longlong(0) + + def fix_annotation(self): + # the annotation of the class and its attributes must be completed + # BEFORE we do the gc transform; this makes sure that everything is + # annotated with the correct types + if NonConstant(False): + self.count = NonConstant(-42) + self.duration = NonConstant(r_longlong(-42)) + self.duration_min = NonConstant(r_longlong(-42)) + self.duration_max = NonConstant(r_longlong(-42)) + self.oldstate = NonConstant(-42) + self.newstate = NonConstant(-42) + self.fire() + + def perform(self, ec, frame): + w_stats = W_GcCollectStepStats( + self.count, + self.duration, + self.duration_min, + self.duration_max, + self.oldstate, + self.newstate) + self.reset() + self.space.call_function(self.w_callable, w_stats) + + +class GcCollectHookAction(AsyncAction): + num_major_collects = 0 + arenas_count_before = 0 + arenas_count_after = 0 + arenas_bytes = 0 + rawmalloc_bytes_before = 0 + rawmalloc_bytes_after = 0 + + def __init__(self, space): + AsyncAction.__init__(self, space) + self.w_callable = space.w_None + self.reset() + + def reset(self): + self.count = 0 + + def fix_annotation(self): + # the annotation of the class and its attributes must be completed + # BEFORE we do the gc transform; this makes sure that everything is + # annotated with the correct types + if NonConstant(False): + self.count = NonConstant(-42) + self.num_major_collects = NonConstant(-42) + self.arenas_count_before = NonConstant(-42) + self.arenas_count_after = NonConstant(-42) + self.arenas_bytes = NonConstant(r_uint(42)) + self.rawmalloc_bytes_before = NonConstant(r_uint(42)) + self.rawmalloc_bytes_after = NonConstant(r_uint(42)) + self.fire() + + def perform(self, ec, frame): + w_stats = W_GcCollectStats(self.count, + self.num_major_collects, + self.arenas_count_before, + self.arenas_count_after, + self.arenas_bytes, + self.rawmalloc_bytes_before, + self.rawmalloc_bytes_after) + self.reset() + self.space.call_function(self.w_callable, w_stats) + + +class W_GcMinorStats(W_Root): + + def __init__(self, count, duration, duration_min, duration_max, + total_memory_used, pinned_objects): + self.count = count + self.duration = duration + self.duration_min = duration_min + self.duration_max = duration_max + self.total_memory_used = total_memory_used + self.pinned_objects = pinned_objects + + +class W_GcCollectStepStats(W_Root): + + def __init__(self, count, duration, duration_min, duration_max, + oldstate, newstate): + self.count = count + self.duration = duration + self.duration_min = duration_min + self.duration_max = duration_max + self.oldstate = oldstate + self.newstate = newstate + + +class W_GcCollectStats(W_Root): + def __init__(self, count, num_major_collects, + arenas_count_before, arenas_count_after, + arenas_bytes, rawmalloc_bytes_before, + rawmalloc_bytes_after): + self.count = count + self.num_major_collects = num_major_collects + self.arenas_count_before = arenas_count_before + self.arenas_count_after = arenas_count_after + self.arenas_bytes = arenas_bytes + self.rawmalloc_bytes_before = rawmalloc_bytes_before + self.rawmalloc_bytes_after = rawmalloc_bytes_after + + +# just a shortcut to make the typedefs shorter +def wrap_many_ints(cls, names): + d = {} + for name in names: + d[name] = interp_attrproperty(name, cls=cls, wrapfn="newint") + return d + + +W_AppLevelHooks.typedef = TypeDef( + "GcHooks", + on_gc_minor = GetSetProperty( + W_AppLevelHooks.descr_get_on_gc_minor, + W_AppLevelHooks.descr_set_on_gc_minor), + + on_gc_collect_step = GetSetProperty( + W_AppLevelHooks.descr_get_on_gc_collect_step, + W_AppLevelHooks.descr_set_on_gc_collect_step), + + on_gc_collect = GetSetProperty( + W_AppLevelHooks.descr_get_on_gc_collect, + W_AppLevelHooks.descr_set_on_gc_collect), + + set = interp2app(W_AppLevelHooks.descr_set), + reset = interp2app(W_AppLevelHooks.descr_reset), + ) + +W_GcMinorStats.typedef = TypeDef( + "GcMinorStats", + **wrap_many_ints(W_GcMinorStats, ( + "count", + "duration", + "duration_min", + "duration_max", + "total_memory_used", + "pinned_objects")) + ) + +W_GcCollectStepStats.typedef = TypeDef( + "GcCollectStepStats", + STATE_SCANNING = incminimark.STATE_SCANNING, + STATE_MARKING = incminimark.STATE_MARKING, + STATE_SWEEPING = incminimark.STATE_SWEEPING, + STATE_FINALIZING = incminimark.STATE_FINALIZING, + GC_STATES = tuple(incminimark.GC_STATES), + **wrap_many_ints(W_GcCollectStepStats, ( + "count", + "duration", + "duration_min", + "duration_max", + "oldstate", + "newstate")) + ) + +W_GcCollectStats.typedef = TypeDef( + "GcCollectStats", + **wrap_many_ints(W_GcCollectStats, ( + "count", + "num_major_collects", + "arenas_count_before", + "arenas_count_after", + "arenas_bytes", + "rawmalloc_bytes_before", + "rawmalloc_bytes_after")) + ) diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py new file mode 100644 --- /dev/null +++ b/pypy/module/gc/test/test_hook.py @@ -0,0 +1,178 @@ +import pytest +from rpython.rlib.rarithmetic import r_uint +from pypy.module.gc.hook import LowLevelGcHooks +from pypy.interpreter.baseobjspace import ObjSpace +from pypy.interpreter.gateway import interp2app, unwrap_spec + +class AppTestGcHooks(object): + + def setup_class(cls): + if cls.runappdirect: + pytest.skip("these tests cannot work with -A") + space = cls.space + gchooks = space.fromcache(LowLevelGcHooks) + + @unwrap_spec(ObjSpace, int, r_uint, int) + def fire_gc_minor(space, duration, total_memory_used, pinned_objects): + gchooks.fire_gc_minor(duration, total_memory_used, pinned_objects) + + @unwrap_spec(ObjSpace, int, int, int) + def fire_gc_collect_step(space, duration, oldstate, newstate): + gchooks.fire_gc_collect_step(duration, oldstate, newstate) + + @unwrap_spec(ObjSpace, int, int, int, r_uint, r_uint, r_uint) + def fire_gc_collect(space, a, b, c, d, e, f): + gchooks.fire_gc_collect(a, b, c, d, e, f) + + @unwrap_spec(ObjSpace) + def fire_many(space): + gchooks.fire_gc_minor(5, 0, 0) + gchooks.fire_gc_minor(7, 0, 0) + gchooks.fire_gc_collect_step(5, 0, 0) + gchooks.fire_gc_collect_step(15, 0, 0) + gchooks.fire_gc_collect_step(22, 0, 0) + gchooks.fire_gc_collect(1, 2, 3, 4, 5, 6) + + cls.w_fire_gc_minor = space.wrap(interp2app(fire_gc_minor)) + cls.w_fire_gc_collect_step = space.wrap(interp2app(fire_gc_collect_step)) + cls.w_fire_gc_collect = space.wrap(interp2app(fire_gc_collect)) + cls.w_fire_many = space.wrap(interp2app(fire_many)) + + def test_default(self): + import gc + assert gc.hooks.on_gc_minor is None + assert gc.hooks.on_gc_collect_step is None + assert gc.hooks.on_gc_collect is None + + def test_on_gc_minor(self): + import gc + lst = [] + def on_gc_minor(stats): + lst.append((stats.count, + stats.duration, + stats.total_memory_used, + stats.pinned_objects)) + gc.hooks.on_gc_minor = on_gc_minor + self.fire_gc_minor(10, 20, 30) + self.fire_gc_minor(40, 50, 60) + assert lst == [ + (1, 10, 20, 30), + (1, 40, 50, 60), + ] + # + gc.hooks.on_gc_minor = None + self.fire_gc_minor(70, 80, 90) # won't fire because the hooks is disabled + assert lst == [ + (1, 10, 20, 30), + (1, 40, 50, 60), + ] + + def test_on_gc_collect_step(self): + import gc + lst = [] + def on_gc_collect_step(stats): + lst.append((stats.count, + stats.duration, + stats.oldstate, + stats.newstate)) + gc.hooks.on_gc_collect_step = on_gc_collect_step + self.fire_gc_collect_step(10, 20, 30) + self.fire_gc_collect_step(40, 50, 60) + assert lst == [ + (1, 10, 20, 30), + (1, 40, 50, 60), + ] + # + gc.hooks.on_gc_collect_step = None + self.fire_gc_collect_step(70, 80, 90) # won't fire + assert lst == [ + (1, 10, 20, 30), + (1, 40, 50, 60), + ] + + def test_on_gc_collect(self): + import gc + lst = [] + def on_gc_collect(stats): + lst.append((stats.count, + stats.num_major_collects, + stats.arenas_count_before, + stats.arenas_count_after, + stats.arenas_bytes, + stats.rawmalloc_bytes_before, + stats.rawmalloc_bytes_after)) + gc.hooks.on_gc_collect = on_gc_collect + self.fire_gc_collect(1, 2, 3, 4, 5, 6) + self.fire_gc_collect(7, 8, 9, 10, 11, 12) + assert lst == [ + (1, 1, 2, 3, 4, 5, 6), + (1, 7, 8, 9, 10, 11, 12), + ] + # + gc.hooks.on_gc_collect = None + self.fire_gc_collect(42, 42, 42, 42, 42, 42) # won't fire + assert lst == [ + (1, 1, 2, 3, 4, 5, 6), + (1, 7, 8, 9, 10, 11, 12), + ] + + def test_consts(self): + import gc + S = gc.GcCollectStepStats + assert S.STATE_SCANNING == 0 + assert S.STATE_MARKING == 1 + assert S.STATE_SWEEPING == 2 + assert S.STATE_FINALIZING == 3 + assert S.GC_STATES == ('SCANNING', 'MARKING', 'SWEEPING', 'FINALIZING') + + def test_cumulative(self): + import gc + class MyHooks(object): + + def __init__(self): + self.minors = [] + self.steps = [] + + def on_gc_minor(self, stats): + self.minors.append((stats.count, stats.duration, + stats.duration_min, stats.duration_max)) + + def on_gc_collect_step(self, stats): + self.steps.append((stats.count, stats.duration, + stats.duration_min, stats.duration_max)) + + on_gc_collect = None + + myhooks = MyHooks() + gc.hooks.set(myhooks) + self.fire_many() + assert myhooks.minors == [(2, 12, 5, 7)] + assert myhooks.steps == [(3, 42, 5, 22)] + + def test_clear_queue(self): + import gc + class MyHooks(object): + + def __init__(self): + self.lst = [] + + def on_gc_minor(self, stats): + self.lst.append('minor') + + def on_gc_collect_step(self, stats): + self.lst.append('step') + + def on_gc_collect(self, stats): + self.lst.append('collect') + + myhooks = MyHooks() + gc.hooks.set(myhooks) + self.fire_many() + assert myhooks.lst == ['minor', 'step', 'collect'] + myhooks.lst[:] = [] + self.fire_gc_minor(0, 0, 0) + assert myhooks.lst == ['minor'] + gc.hooks.reset() + assert gc.hooks.on_gc_minor is None + assert gc.hooks.on_gc_collect_step is None + assert gc.hooks.on_gc_collect is None diff --git a/pypy/module/gc/test/test_ztranslation.py b/pypy/module/gc/test/test_ztranslation.py --- a/pypy/module/gc/test/test_ztranslation.py +++ b/pypy/module/gc/test/test_ztranslation.py @@ -1,4 +1,9 @@ from pypy.objspace.fake.checkmodule import checkmodule def test_checkmodule(): - checkmodule('gc') + # we need to ignore GcCollectStepStats, else checkmodule fails. I think + # this happens because W_GcCollectStepStats.__init__ is only called from + # GcCollectStepHookAction.perform() and the fake objspace doesn't know + # about those: so, perform() is never annotated and the annotator thinks + # W_GcCollectStepStats has no attributes + checkmodule('gc', ignore=['GcCollectStepStats']) diff --git a/pypy/objspace/fake/checkmodule.py b/pypy/objspace/fake/checkmodule.py --- a/pypy/objspace/fake/checkmodule.py +++ b/pypy/objspace/fake/checkmodule.py @@ -4,6 +4,7 @@ def checkmodule(*modnames, **kwds): translate_startup = kwds.pop('translate_startup', True) + ignore = set(kwds.pop('ignore', ())) assert not kwds config = get_pypy_config(translating=True) space = FakeObjSpace(config) @@ -17,6 +18,8 @@ module.init(space) modules.append(module) for name in module.loaders: + if name in ignore: + continue seeobj_w.append(module._load_lazily(space, name)) if hasattr(module, 'submodules'): for cls in module.submodules.itervalues(): diff --git a/pypy/tool/pytest/genreportdata.py b/pypy/tool/pytest/genreportdata.py deleted file mode 100755 --- a/pypy/tool/pytest/genreportdata.py +++ /dev/null @@ -1,30 +0,0 @@ -#! /usr/bin/env python -import py -import sys - -mydir = py.path.local(__file__).dirpath().realpath() -from pypy.tool.pytest import htmlreport -from pypy.tool.pytest import confpath - -if __name__ == '__main__': - if len(sys.argv) > 1: - testresultdir = py.path.local(sys.argv[1]) - assert testresultdir.check(dir=1) - else: - testresultdir = confpath.testresultdir - assert testresultdir.check(dir=1) - try: - resultwc = py.path.svnwc(testresultdir) - print "updating", resultwc - resultwc.update() - except (KeyboardInterrupt, RuntimeError): - raise - except Exception as e: #py.process.ExecutionFailed,e: - print >> sys.stderr, "Warning: ",e #Subversion update failed" - - print "traversing", mydir - rep = htmlreport.HtmlReport(testresultdir) - rep.parselatest() - - print "making html files" - rep.makeindex(testresultdir.join('index.html')) diff --git a/pypy/tool/pytest/htmlreport.py b/pypy/tool/pytest/htmlreport.py deleted file mode 100644 --- a/pypy/tool/pytest/htmlreport.py +++ /dev/null @@ -1,239 +0,0 @@ -#! /usr/bin/env python - -""" -the html test reporter - -""" -import sys, os, re -import pprint -import py -from pypy.tool.pytest import result -from pypy.tool.pytest.overview import ResultCache - -# -# various interesting path objects -# - -html = py.xml.html -NBSP = py.xml.raw(" ") - -class HtmlReport(object): - def __init__(self, resultdir): - self.resultcache = ResultCache(resultdir) - - def parselatest(self): - self.resultcache.parselatest() - - # - # rendering - # - - def render_latest_table(self, results): - table = html.table( - [html.th(x, align='left') - for x in ("failure", "filename", "revision", - "user", "platform", "elapsed", - "options", "last error line" - )], - ) - r = results[:] - def f(x, y): - xnum = x.isok() and 1 or (x.istimeout() and 2 or 3) - ynum = y.isok() and 1 or (y.istimeout() and 2 or 3) - res = -cmp(xnum, ynum) - if res == 0: - return cmp(x['execution-time'], y['execution-time']) - return res - r.sort(f) - for result in r: - table.append(self.render_result_row(result)) - return table - - def render_result_row(self, result): - dp = py.path.local(result['fspath']) - - options = " ".join([x for x in result.get('options', []) if x!= 'core']) - if not options: - options = NBSP - - failureratio = 100 * (1.0 - result.ratio_of_passed()) - self.data[result.testname] = failureratio - return html.tr( - html.td("%.2f%%" % failureratio, - style = "background-color: %s" % (getresultcolor(result),)), - html.td(self.render_test_references(result)), - html.td(result['pypy-revision']), - html.td(result['userhost'][:15]), - html.td(result['platform']), - html.td("%.2fs" % result['execution-time']), - html.td(options), - html.td(result.repr_short_error() or NBSP) - ) - - def getrelpath(self, p): - return p.relto(self.indexpath.dirpath()) - - def render_test_references(self, result): - dest = self.make_single_test_result(result) - #XXX: ask hg for differences between test and vendor branch - modified = result.ismodifiedtest() and " [mod]" or "" - return html.div(html.a(result.path.purebasename + modified, - href=self.getrelpath(dest)), - style="background-color: transparent") - - def make_single_test_result(self, result): - cache = self.indexpath.dirpath('.cache', result['userhost'][:15]) - cache.ensure(dir=1) - dest = cache.join(result.path.basename).new(ext='.html') - doc = ViewResult(result) - doc.writetopath(dest) - return dest - - def getcorelists(self): - def iscore(result): - return 'core' in result.get('options', []) - coretests = [] - noncoretests = [] - for name in self.resultcache.getnames(): - result = self.resultcache.getlatestrelevant(name) - if iscore(result): - coretests.append(result) - else: - noncoretests.append(result) - return coretests, noncoretests - - # generate html files - # - def makeindex(self, indexpath, detail="PyPy - latest"): - self.indexpath = indexpath - self.data = {} - doc = Document(title='pypy test results') - body = doc.body - coretests, noncoretests = self.getcorelists() - body.append(html.h2("%s compliance test results - " - "core tests" % detail)) - - body.append(self.render_test_summary('core', coretests)) - body.append(self.render_latest_table(coretests)) - body.append( - html.h2("%s compliance test results - non-core tests" % detail)) - body.append(self.render_test_summary('noncore', noncoretests)) - body.append(self.render_latest_table(noncoretests)) - doc.writetopath(indexpath) - datapath = indexpath.dirpath().join('data') - d = datapath.open('w') - print >>d, "data = ", - pprint.pprint(self.data, stream=d) - d.close() - self.data = None - - def render_test_summary(self, tag, tests): - ok = len([x for x in tests if x.isok()]) - err = len([x for x in tests if x.iserror()]) - to = len([x for x in tests if x.istimeout()]) - numtests = ok + err + to - assert numtests == len(tests) - assert numtests - - t = html.table() - sum100 = numtests / 100.0 - def row(*args): - return html.tr(*[html.td(arg) for arg in args]) - - sum_passed = sum([x.ratio_of_passed() for x in tests]) - compliancy = sum_passed/sum100 - self.data['%s-compliancy' % tag] = compliancy - t.append(row(html.b("tests compliancy"), - html.b("%.2f%%" % (compliancy,)))) - - passed = ok/sum100 - self.data['%s-passed' % tag] = passed - t.append(row("testmodules passed completely", "%.2f%%" % passed)) - failed = err/sum100 - self.data['%s-failed' % tag] = failed - t.append(row("testmodules (partially) failed", "%.2f%%" % failed)) - timedout = to/sum100 - self.data['%s-timedout' % tag] = timedout - t.append(row("testmodules timeout", "%.2f%%" % timedout)) - return t - -class Document(object): - def __init__(self, title=None): - self.body = html.body() - self.head = html.head() - self.doc = html.html(self.head, self.body) - if title is not None: - self.head.append( - html.meta(name="title", content=title)) - self.head.append( - html.link(rel="Stylesheet", type="text/css", href="/pypy/default.css")) - - def writetopath(self, p): - assert p.ext == '.html' - self.head.append( - html.meta(name="Content-Type", content="text/html;charset=UTF-8") - ) - s = self.doc.unicode().encode('utf-8') - p.write(s) - -def getresultcolor(result): - if result.isok(): - color = "#00ee00" - elif result.iserror(): - color = "#ee0000" - elif result.istimeout: - color = "#0000ee" - else: - color = "#444444" - return color - -class ViewResult(Document): - def __init__(self, result): - title = "%s testresult" % (result.path.purebasename,) - super(ViewResult, self).__init__(title=title) - color = getresultcolor(result) - self.body.append(html.h2(title, - style="background-color: %s" % color)) - self.body.append(self.render_meta_info(result)) - - for name in ('reportdiff', 'stdout', 'stderr'): - try: - text = result.getnamedtext(name) - except KeyError: - continue - self.body.append(html.h3(name)) - self.body.append(html.pre(text)) - - def render_meta_info(self, result): - t = html.table() - items = result.items() - items.sort() - for name, value in items: - if name.lower() == name: - t.append(html.tr( - html.td(name), html.td(value))) - return t - -class TestOfHtmlReportClass: - def setup_class(cls): - py.test.skip('needs move to own test file') - cls.testresultdir = confpath.testresultdir - cls.rep = rep = HtmlReport() - rep.parse_all(cls.testresultdir) - - def test_pickling(self): - # test pickling of report - tempdir = py.test.ensuretemp('reportpickle') - picklepath = tempdir.join('report.pickle') - picklepath.dump(self.rep) - x = picklepath.load() - assert len(x.results) == len(self.rep.results) - - def test_render_latest(self): - t = self.rep.render_latest_table(self.rep.results) - assert unicode(t) - -mydir = py.path.local(__file__).dirpath() - -def getpicklepath(): - return mydir.join('.htmlreport.pickle') diff --git a/pypy/tool/pytest/overview.py b/pypy/tool/pytest/overview.py deleted file mode 100644 --- a/pypy/tool/pytest/overview.py +++ /dev/null @@ -1,56 +0,0 @@ -from pypy.tool.pytest import result -import sys - -class ResultCache: - def __init__(self, resultdir): - self.resultdir = resultdir - self.name2result = {} - - def parselatest(self): - def filefilter(p): - return p.check(fnmatch='test_*.txt', file=1) - def rec(p): - return p.check(dotfile=0) - for x in self.resultdir.visit(filefilter, rec): - self.parse_one(x) - - def parse_one(self, resultpath): - try: - res = result.ResultFromMime(resultpath) - ver = res['testreport-version'] - if ver != "1.1" and ver != "1.1.1": - raise TypeError - except TypeError: # xxx - print >>sys.stderr, "could not parse %s" % resultpath - return - name = res.testname - print name - self.name2result.setdefault(name, []).append(res) - return res - - def getnames(self): - return self.name2result.keys() - - def getlatest(self, name, timeout=0, error=0, ok=0): - l = [] - resultlist = self.name2result[name] - maxrev = 0 - maxresult = None - for res in resultlist: - resrev = res['pypy-revision'] - if resrev == 'unknown': - continue - if resrev <= maxrev: - continue - if timeout or error or ok: - if not (timeout and res.istimeout() or - error and res.iserror() or - ok and res.isok()): - continue - maxrev = resrev - maxresult = res - return maxresult - - def getlatestrelevant(self, name): - # get the latest revision that did not time out. - return self.getlatest(name, error=1, ok=1) or self.getlatest(name) diff --git a/pypy/tool/pytest/result.py b/pypy/tool/pytest/result.py deleted file mode 100644 --- a/pypy/tool/pytest/result.py +++ /dev/null @@ -1,215 +0,0 @@ -import sys -import py -import re - -class Result(object): - def __init__(self, init=True): - self._headers = {} - self._blocks = {} - self._blocknames = [] - if init: - stdinit(self) - - def __setitem__(self, name, value): - self._headers[name.lower()] = value - - def __getitem__(self, name): - return self._headers[name.lower()] - - def get(self, name, default): - return self._headers.get(name, default) - - def __delitem__(self, name): - del self._headers[name.lower()] - - def items(self): - return self._headers.items() - - def addnamedtext(self, name, text): - assert isinstance(text, basestring) - assert isinstance(name, str) - self._blocknames.append(name) - self._blocks[name] = text - - def getnamedtext(self, name): - return self._blocks[name] - - def repr_short_error(self): - if not self.isok(): - if 'reportdiff' in self._blocks: - return "output comparison failed, see reportdiff" - else: - text = self.getnamedtext('stderr') - lines = text.strip().split('\n') - if lines: - return lines[-1] - - def repr_mimemessage(self): - from email.MIMEMultipart import MIMEMultipart - from email.MIMEText import MIMEText - - outer = MIMEMultipart() - items = self._headers.items() - items.sort() - reprs = {} - for name, value in items: - assert ':' not in name - chars = map(ord, name) - assert min(chars) >= 33 and max(chars) <= 126 - outer[name] = str(value) - if not isinstance(value, str): - typename = type(value).__name__ - assert typename in vars(py.std.__builtin__) - reprs[name] = typename - - outer['_reprs'] = repr(reprs) - - for name in self._blocknames: - text = self._blocks[name] - m = MIMEText(text) - m.add_header('Content-Disposition', 'attachment', filename=name) - outer.attach(m) - return outer - - def grep_nr(self,text,section='stdout'): - stdout = self._blocks[section] - find = re.search('%s(?P\d+)'%text,stdout) - if find: - return float(find.group('nr')) - return 0. - - def ratio_of_passed(self): - if self.isok(): - return 1. - elif self.istimeout(): - return 0. - else: - nr = self.grep_nr('Ran ') - if nr > 0: - return (nr - (self.grep_nr('errors=') + self.grep_nr('failures=')))/nr - else: - passed = self.grep_nr('TestFailed: ',section='stderr') - run = self.grep_nr('TestFailed: \d+/',section='stderr') - if run > 0: - return passed/run - else: - run = self.grep_nr('TestFailed: \d+ of ',section='stderr') - if run > 0 : - return (run-passed)/run - else: - return 0.0 - - def isok(self): - return self['outcome'].lower() == 'ok' - - def iserror(self): - return self['outcome'].lower()[:3] == 'err' or self['outcome'].lower() == 'fail' - - def istimeout(self): - return self['outcome'].lower() == 't/o' - -# XXX backward compatibility -def sanitize(msg, path): - if 'exit-status' in msg.keys(): - return msg - f = open(str(path), 'r') - msg = f.read() - f.close() - for broken in ('exit status', 'cpu model', 'cpu mhz'): - valid = broken.replace(' ','-') - invalid = msg.find(broken+':') - msg = (msg[:invalid] + valid + - msg[invalid+len(valid):]) - from email import message_from_string - msg = message_from_string(msg) - return msg - -def sanitize_reprs(reprs): - if 'exit status' in reprs: - reprs['exit-status'] = reprs.pop('exit status') - -class ResultFromMime(Result): - def __init__(self, path): - super(ResultFromMime, self).__init__(init=False) - f = open(str(path), 'r') - from email import message_from_file - msg = message_from_file(f) - f.close() - msg = sanitize(msg, path) - # XXX security wise evil (keep in mind once we accept reporsts - # from anonymous - #print msg['_reprs'] - self._reprs = eval(msg['_reprs']) - del msg['_reprs'] - sanitize_reprs(self._reprs) - for name, value in msg.items(): - if name in self._reprs: - value = eval(value) # XXX security - self._headers[name] = value - self.fspath = self['fspath'] - if self['platform'] == 'win32' and '\\' in self.fspath: - self.testname = self.fspath.split('\\')[-1] - else: - self.testname = self.fspath.split('/')[-1] - #if sys.platform != 'win32' and '\\' in self.fspath: - # self.fspath = py.path.local(self['fspath'].replace('\\' - self.path = path - - payload = msg.get_payload() - if payload: - for submsg in payload: - assert submsg.get_content_type() == 'text/plain' - fn = submsg.get_filename() - assert fn - # XXX we need to deal better with encodings to - # begin with - content = submsg.get_payload() - for candidate in 'utf8', 'latin1': - try: - text = unicode(content, candidate) - except UnicodeDecodeError: - continue - else: - unicode(content, candidate) - self.addnamedtext(fn, text) - - def ismodifiedtest(self): - # XXX we need proper cross-platform paths! - return 'modified' in self.fspath - - def __repr__(self): - return '<%s (%s) %r rev=%s>' %(self.__class__.__name__, - self['outcome'], - self.fspath, - self['pypy-revision']) - -def stdinit(result): - import getpass - import socket - try: - username = getpass.getuser() - except: - username = 'unknown' - userhost = '%s@%s' % (username, socket.gethostname()) - result['testreport-version'] = "1.1.1" - result['userhost'] = userhost - result['platform'] = sys.platform - result['python-version-info'] = sys.version_info - info = try_getcpuinfo() - if info is not None: - result['cpu-model'] = info.get('model name', "unknown") - result['cpu-mhz'] = info.get('cpu mhz', 'unknown') -# -# -# -def try_getcpuinfo(): - if sys.platform.startswith('linux'): - cpuinfopath = py.path.local('/proc/cpuinfo') - if cpuinfopath.check(file=1): - d = {} - for line in cpuinfopath.readlines(): - if line.strip(): - name, value = line.split(':', 1) - name = name.strip().lower() - d[name] = value.strip() - return d diff --git a/pypy/tool/pytest/test/data/test___all__.txt b/pypy/tool/pytest/test/data/test___all__.txt deleted file mode 100644 --- a/pypy/tool/pytest/test/data/test___all__.txt +++ /dev/null @@ -1,94 +0,0 @@ -Content-Type: multipart/mixed; boundary="===============0790678169==" -MIME-Version: 1.0 -execution-time: 1445.14346004 -exit status: 1 -fspath: /Users/anderslehmann/pypy/lib-python/2.4.1/test/test___all__.py -options: ['core', '_sre'] -outcome: T/O -platform: darwin -pypy-revision: 16114 -python-version-info: (2, 4, 1, 'final', 0) -startdate: Wed Aug 17 23:51:59 2005 -testreport-version: 1.1 -timeout: 1369.0 -userhost: anderslehmann at anders-lehmanns-15-powerbook-g4.local -_reprs: {'execution-time': 'float', 'python-version-info': 'tuple', - 'options': 'list', 'timeout': 'float', 'pypy-revision': 'int', - 'exit status': 'int'} - ---===============0790678169== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stdout" - -test_all (__main__.AllTest) ... ERROR - -====================================================================== -ERROR: test_all (__main__.AllTest) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test___all__.py", line 163, in test_all - self.check_all("tty") - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test___all__.py", line 26, in check_all - "%s has no __all__ attribute" % modname) - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_support.py", line 208, in verify - raise TestFailed(reason) -TestFailed: tty has no __all__ attribute - ----------------------------------------------------------------------- - ---===============0790678169== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stderr" - -faking -Loading grammar /Users/anderslehmann/pypy/pypy/interpreter/pyparser/data/Grammar2.4 -faking -faking -faking -fake-wrapping interp file ', mode 'w' at 0x12068> -fake-wrapping interp file ', mode 'w' at 0x120b0> -fake-wrapping interp file ', mode 'r' at 0x12020> -faking -faking -faking -faking -==========================timedout========================== -Traceback (application-level): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test___all__.py", line 192 in - test_main() - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test___all__.py", line 189 in test_main - test_support.run_unittest(AllTest) - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_support.py", line 290 in run_unittest - run_suite(suite, testclass) - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_support.py", line 262 in run_suite - result = runner.run(suite) -Traceback (application-level): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/atexit.py", line 29 in _run_exitfuncs - print >> sys.stderr, "Error in atexit._run_exitfuncs:" -KeyboardInterrupt -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/pypy/tool/alarm.py", line 43, in ? - execfile(_main_with_alarm(finished)) - File "/Users/anderslehmann/pypy/pypy/bin/py.py", line 206, in ? - sys.exit(main_(sys.argv)) - File "/Users/anderslehmann/pypy/pypy/bin/py.py", line 115, in main_ - if not main.run_toplevel(space, doit, verbose=Options.verbose): - File "/Users/anderslehmann/pypy/pypy/interpreter/main.py", line 150, in run_toplevel - operationerr.print_application_traceback(space) - File "/Users/anderslehmann/pypy/pypy/interpreter/error.py", line 83, in print_application_traceback - self.print_app_tb_only(file) - File "/Users/anderslehmann/pypy/pypy/interpreter/error.py", line 104, in print_app_tb_only - l = linecache.getline(fname, lineno) - File "/Library/Frameworks/Python.framework/Versions/2.4/lib/python2.4/linecache.py", line 14, in getline - lines = getlines(filename) - File "/Library/Frameworks/Python.framework/Versions/2.4/lib/python2.4/linecache.py", line 40, in getlines - return updatecache(filename) - File "/Library/Frameworks/Python.framework/Versions/2.4/lib/python2.4/linecache.py", line 101, in updatecache - lines = fp.readlines() -KeyboardInterrupt - ---===============0790678169==-- \ No newline at end of file diff --git a/pypy/tool/pytest/test/data/test_compile.txt b/pypy/tool/pytest/test/data/test_compile.txt deleted file mode 100644 --- a/pypy/tool/pytest/test/data/test_compile.txt +++ /dev/null @@ -1,111 +0,0 @@ -Content-Type: multipart/mixed; boundary="===============2137793924==" -MIME-Version: 1.0 -execution-time: 34.8464071751 -exit status: 1 -fspath: /Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py -options: ['core', '_sre'] -outcome: ERR -platform: darwin -pypy-revision: 16114 -python-version-info: (2, 4, 1, 'final', 0) -startdate: Thu Aug 18 03:08:18 2005 -testreport-version: 1.1 -timeout: 1521.0 -userhost: anderslehmann at anders-lehmanns-15-powerbook-g4.local -_reprs: {'execution-time': 'float', 'python-version-info': 'tuple', - 'options': 'list', 'timeout': 'float', 'pypy-revision': 'int', - 'exit status': 'int'} - ---===============2137793924== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stdout" - -test_argument_handling (__main__.TestSpecifics) ... FAIL -test_argument_order (__main__.TestSpecifics) ... FAIL -test_complex_args (__main__.TestSpecifics) ... ok -test_debug_assignment (__main__.TestSpecifics) ... FAIL -test_duplicate_global_local (__main__.TestSpecifics) ... ok -test_exec_with_general_mapping_for_locals (__main__.TestSpecifics) ... ok -test_float_literals (__main__.TestSpecifics) ... ok -test_for_distinct_code_objects (__main__.TestSpecifics) ... ok -test_import (__main__.TestSpecifics) ... FAIL -test_indentation (__main__.TestSpecifics) ... ok -test_literals_with_leading_zeroes (__main__.TestSpecifics) ... ok -test_none_assignment (__main__.TestSpecifics) ... FAIL -test_sequence_unpacking_error (__main__.TestSpecifics) ... ok -test_syntax_error (__main__.TestSpecifics) ... ok -test_unary_minus (__main__.TestSpecifics) ... ok - -====================================================================== -FAIL: test_argument_handling (__main__.TestSpecifics) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 18, in test_argument_handling - self.assertRaises(SyntaxError, eval, 'lambda a,a:0') -AssertionError: SyntaxError not raised - -====================================================================== -FAIL: test_argument_order (__main__.TestSpecifics) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 127, in test_argument_order - self.fail("non-default args after default") -AssertionError: non-default args after default - -====================================================================== -FAIL: test_debug_assignment (__main__.TestSpecifics) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 10, in test_debug_assignment - self.assertRaises(SyntaxError, compile, '__debug__ = 1', '?', 'single') -AssertionError: SyntaxError not raised - -====================================================================== -FAIL: test_import (__main__.TestSpecifics) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 253, in test_import - self.assertRaises(SyntaxError, compile, stmt, 'tmp', 'exec') -AssertionError: SyntaxError not raised - -====================================================================== -FAIL: test_none_assignment (__main__.TestSpecifics) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 211, in test_none_assignment - self.assertRaises(SyntaxError, compile, stmt, 'tmp', 'single') -AssertionError: SyntaxError not raised - ----------------------------------------------------------------------- -Ran 15 tests in 14.363s - -FAILED (failures=5) - ---===============2137793924== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stderr" - -faking -Loading grammar /Users/anderslehmann/pypy/pypy/interpreter/pyparser/data/Grammar2.4 -faking -faking -faking -fake-wrapping interp file ', mode 'w' at 0x12068> -fake-wrapping interp file ', mode 'w' at 0x120b0> -fake-wrapping interp file ', mode 'r' at 0x12020> -Traceback (application-level): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 268 in - test_main() - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 265 in test_main - test_support.run_unittest(TestSpecifics) - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_support.py", line 290 in run_unittest - run_suite(suite, testclass) - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_support.py", line 274 in run_suite - raise TestFailed(msg) -TestFailed: errors occurred in __main__.TestSpecifics - ---===============2137793924==-- \ No newline at end of file diff --git a/pypy/tool/pytest/test/data/test_descr.txt b/pypy/tool/pytest/test/data/test_descr.txt deleted file mode 100644 --- a/pypy/tool/pytest/test/data/test_descr.txt +++ /dev/null @@ -1,233 +0,0 @@ -Content-Type: multipart/mixed; boundary="===============1265023865==" -MIME-Version: 1.0 -execution-time: 4098.8407588 -exit status: 1 -fspath: /Users/anderslehmann/pypy/lib-python/modified-2.4.1/test/test_descr.py -options: ['oldstyle', 'core'] -outcome: ERR -platform: darwin -pypy-revision: 16388 -python-version-info: (2, 4, 1, 'final', 0) -startdate: Wed Aug 24 16:54:12 2005 From pypy.commits at gmail.com Fri Apr 20 12:13:57 2018 From: pypy.commits at gmail.com (rlamy) Date: Fri, 20 Apr 2018 09:13:57 -0700 (PDT) Subject: [pypy-commit] pypy default: Test that tp_basicsize is correctly set (fails on py3.5) Message-ID: <5ada11c5.1c69fb81.a2536.9a2a@mx.google.com> Author: Ronan Lamy Branch: Changeset: r94396:2a9fc2de357a Date: 2018-04-20 17:12 +0100 http://bitbucket.org/pypy/pypy/changeset/2a9fc2de357a/ Log: Test that tp_basicsize is correctly set (fails on py3.5) diff --git a/pypy/module/cpyext/test/test_datetime.py b/pypy/module/cpyext/test/test_datetime.py --- a/pypy/module/cpyext/test/test_datetime.py +++ b/pypy/module/cpyext/test/test_datetime.py @@ -1,3 +1,5 @@ +import pytest + from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.cdatetime import * @@ -82,6 +84,14 @@ date = datetime.datetime.fromtimestamp(0) assert space.unwrap(space.str(w_date)) == str(date) + @pytest.mark.parametrize('name', ['Time', 'DateTime', 'Date', 'Delta']) + def test_basicsize(self, space, name): + datetime = _PyDateTime_Import(space) + py_size = getattr(datetime, "c_%sType" % name).c_tp_basicsize + c_size = rffi.sizeof(cts.gettype("PyDateTime_%s" % name)) + assert py_size == c_size + + class AppTestDatetime(AppTestCpythonExtensionBase): def test_CAPI(self): module = self.import_extension('foo', [ @@ -271,9 +281,9 @@ 6, 6, 6, 6, args, PyDateTimeAPI->TimeType); """), ("datetime_with_tzinfo", "METH_O", - """ + """ PyObject * obj; - int tzrefcnt = args->ob_refcnt; + int tzrefcnt = args->ob_refcnt; PyDateTime_IMPORT; obj = PyDateTimeAPI->DateTime_FromDateAndTime( 2000, 6, 6, 6, 6, 6, 6, args, @@ -291,7 +301,7 @@ return NULL; } return obj; - + """), ], prologue='#include "datetime.h"\n') from datetime import tzinfo, datetime, timedelta, time @@ -339,4 +349,4 @@ assert module.checks(o) == (True,) * 3 + (False,) * 7 # isinstance(datetime, date) o = tzinfo() assert module.checks(o) == (False,) * 8 + (True,) * 2 - + From pypy.commits at gmail.com Fri Apr 20 12:13:59 2018 From: pypy.commits at gmail.com (rlamy) Date: Fri, 20 Apr 2018 09:13:59 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: hg merge default Message-ID: <5ada11c7.1c69fb81.2c686.7ccc@mx.google.com> Author: Ronan Lamy Branch: py3.5 Changeset: r94397:b7da70a8fd2c Date: 2018-04-20 17:13 +0100 http://bitbucket.org/pypy/pypy/changeset/b7da70a8fd2c/ Log: hg merge default diff --git a/pypy/module/cpyext/test/test_datetime.py b/pypy/module/cpyext/test/test_datetime.py --- a/pypy/module/cpyext/test/test_datetime.py +++ b/pypy/module/cpyext/test/test_datetime.py @@ -1,3 +1,5 @@ +import pytest + from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.cdatetime import * @@ -82,6 +84,14 @@ date = datetime.datetime.fromtimestamp(0) assert space.unwrap(space.str(w_date)) == str(date) + @pytest.mark.parametrize('name', ['Time', 'DateTime', 'Date', 'Delta']) + def test_basicsize(self, space, name): + datetime = _PyDateTime_Import(space) + py_size = getattr(datetime, "c_%sType" % name).c_tp_basicsize + c_size = rffi.sizeof(cts.gettype("PyDateTime_%s" % name)) + assert py_size == c_size + + class AppTestDatetime(AppTestCpythonExtensionBase): def test_CAPI(self): module = self.import_extension('foo', [ @@ -271,9 +281,9 @@ 6, 6, 6, 6, args, PyDateTimeAPI->TimeType); """), ("datetime_with_tzinfo", "METH_O", - """ + """ PyObject * obj; - int tzrefcnt = args->ob_refcnt; + int tzrefcnt = args->ob_refcnt; PyDateTime_IMPORT; obj = PyDateTimeAPI->DateTime_FromDateAndTime( 2000, 6, 6, 6, 6, 6, 6, args, @@ -291,7 +301,7 @@ return NULL; } return obj; - + """), ], prologue='#include "datetime.h"\n') from datetime import tzinfo, datetime, timedelta, time @@ -339,4 +349,4 @@ assert module.checks(o) == (True,) * 3 + (False,) * 7 # isinstance(datetime, date) o = tzinfo() assert module.checks(o) == (False,) * 8 + (True,) * 2 - + From pypy.commits at gmail.com Fri Apr 20 18:34:16 2018 From: pypy.commits at gmail.com (rlamy) Date: Fri, 20 Apr 2018 15:34:16 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Brute-force tp_basicsize to the correct value on the datetime types Message-ID: <5ada6ae8.cdc41c0a.6c8a6.c193@mx.google.com> Author: Ronan Lamy Branch: py3.5 Changeset: r94398:7ac5e33a8260 Date: 2018-04-20 23:33 +0100 http://bitbucket.org/pypy/pypy/changeset/7ac5e33a8260/ Log: Brute-force tp_basicsize to the correct value on the datetime types diff --git a/pypy/module/__pypy__/interp_pypydatetime.py b/pypy/module/__pypy__/interp_pypydatetime.py --- a/pypy/module/__pypy__/interp_pypydatetime.py +++ b/pypy/module/__pypy__/interp_pypydatetime.py @@ -5,7 +5,7 @@ def create_class(name): class W_Class(W_Root): - 'builtin base clasee for datetime.%s to allow interop with cpyext' % name + 'builtin base class for datetime.%s to allow interop with cpyext' % name def descr_new__(space, w_type): return space.allocate_instance(W_Class, w_type) diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py --- a/pypy/module/cpyext/cdatetime.py +++ b/pypy/module/cpyext/cdatetime.py @@ -40,18 +40,26 @@ w_type = space.getattr(w_datetime, space.newtext("datetime")) datetimeAPI.c_DateTimeType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) + datetimeAPI.c_DateTimeType.c_tp_basicsize = rffi.sizeof( + cts.gettype('PyDateTime_DateTime')) w_type = space.getattr(w_datetime, space.newtext("time")) datetimeAPI.c_TimeType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) + datetimeAPI.c_TimeType.c_tp_basicsize = rffi.sizeof( + cts.gettype('PyDateTime_Time')) w_type = space.getattr(w_datetime, space.newtext("timedelta")) datetimeAPI.c_DeltaType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) + datetimeAPI.c_DeltaType.c_tp_basicsize = rffi.sizeof( + cts.gettype('PyDateTime_Delta')) w_type = space.getattr(w_datetime, space.newtext("tzinfo")) datetimeAPI.c_TZInfoType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) + datetimeAPI.c_TZInfoType.c_tp_basicsize = rffi.sizeof( + cts.gettype('PyDateTime_TZInfo')) datetimeAPI.c_Date_FromDate = llhelper( _PyDate_FromDate.api_func.functype, @@ -124,7 +132,7 @@ # app level datetime.date. If a c-extension class uses datetime.date for its # base class and defines a tp_dealloc, we will get this: # c_class->tp_dealloc == tp_dealloc_func - # c_class->tp_base == datetime.date, + # c_class->tp_base == datetime.date, # datetime.date->tp_dealloc = _PyPy_subtype_dealloc # datetime.date->tp_base = W_DateTime_Date # W_DateTime_Date->tp_dealloc = _PyPy_subtype_dealloc From pypy.commits at gmail.com Fri Apr 20 18:51:03 2018 From: pypy.commits at gmail.com (rlamy) Date: Fri, 20 Apr 2018 15:51:03 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: hg merge py3.5 Message-ID: <5ada6ed7.1c69fb81.7a83b.53c2@mx.google.com> Author: Ronan Lamy Branch: py3.6 Changeset: r94399:d1a46c216644 Date: 2018-04-20 23:47 +0100 http://bitbucket.org/pypy/pypy/changeset/d1a46c216644/ Log: hg merge py3.5 diff --git a/pypy/doc/release-v6.0.0.rst b/pypy/doc/release-v6.0.0.rst --- a/pypy/doc/release-v6.0.0.rst +++ b/pypy/doc/release-v6.0.0.rst @@ -18,6 +18,8 @@ getting started writing code. We have improved our parser to emit more friendly `syntax errors`_, making PyPy not only faster but more friendly. +The GC now has `hooks`_ to gain more insights into its performance + The Windows PyPy3.5 release is still considered beta-quality. There are open issues with unicode handling especially around system calls and c-extensions. @@ -53,6 +55,7 @@ .. _`blog post`: https://morepypy.blogspot.it/2017/10/cape-of-good-hope-for-pypy-hello-from.html .. _pygobject: https://lazka.github.io/posts/2018-04_pypy-pygobject/index.html .. _`syntax errors`: https://morepypy.blogspot.com/2018/04/improving-syntaxerror-in-pypy.html +.. _`hooks`: gc_info.html#gc-hooks What is PyPy? ============= @@ -101,8 +104,9 @@ * Added missing attributes to C-API ``instancemethod`` on pypy3 * Store error state in thread-local storage for C-API. * Fix JIT bugs exposed in the sre module -* Improve speed of Python parser, improve ParseError messages slightly +* Improve speed of Python parser, improve ParseError messages and SyntaxError * Handle JIT hooks more efficiently +* Fix a rare GC bug exposed by intensive use of cpyext `Buffer` s We also refactored many parts of the JIT bridge optimizations, as well as cpyext internals, and together with new contributors fixed issues, added new diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -3,18 +3,7 @@ ========================== .. this is a revision shortly after release-pypy-6.0.0 -.. startrev: f22145c34985 +.. startrev: ad79cc0ce9a8 -.. branch: issue2752 -Fix a rare GC bug that was introduced more than one year ago, but was -not diagnosed before issue #2752. - -.. branch: gc-hooks - -Introduce GC hooks, as documented in doc/gc_info.rst - -.. branch: gc-hook-better-timestamp - -Improve GC hooks diff --git a/pypy/doc/whatsnew-pypy2-6.0.0.rst b/pypy/doc/whatsnew-pypy2-6.0.0.rst --- a/pypy/doc/whatsnew-pypy2-6.0.0.rst +++ b/pypy/doc/whatsnew-pypy2-6.0.0.rst @@ -113,3 +113,16 @@ Improve line offsets that are reported by SyntaxError. Improve error messages for a few situations, including mismatched parenthesis. + +.. branch: issue2752 + +Fix a rare GC bug that was introduced more than one year ago, but was +not diagnosed before issue #2752. + +.. branch: gc-hooks + +Introduce GC hooks, as documented in doc/gc_info.rst + +.. branch: gc-hook-better-timestamp + +Improve GC hooksd diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst --- a/pypy/doc/whatsnew-pypy3-head.rst +++ b/pypy/doc/whatsnew-pypy3-head.rst @@ -3,5 +3,5 @@ ======================== .. this is the revision after release-pypy3.5-v6.0 -.. startrev: bf74662ee4fa +.. startrev: 580e3e26cd32 diff --git a/pypy/module/__pypy__/interp_pypydatetime.py b/pypy/module/__pypy__/interp_pypydatetime.py --- a/pypy/module/__pypy__/interp_pypydatetime.py +++ b/pypy/module/__pypy__/interp_pypydatetime.py @@ -5,7 +5,7 @@ def create_class(name): class W_Class(W_Root): - 'builtin base clasee for datetime.%s to allow interop with cpyext' % name + 'builtin base class for datetime.%s to allow interop with cpyext' % name def descr_new__(space, w_type): return space.allocate_instance(W_Class, w_type) diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py --- a/pypy/module/cpyext/cdatetime.py +++ b/pypy/module/cpyext/cdatetime.py @@ -40,18 +40,26 @@ w_type = space.getattr(w_datetime, space.newtext("datetime")) datetimeAPI.c_DateTimeType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) + datetimeAPI.c_DateTimeType.c_tp_basicsize = rffi.sizeof( + cts.gettype('PyDateTime_DateTime')) w_type = space.getattr(w_datetime, space.newtext("time")) datetimeAPI.c_TimeType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) + datetimeAPI.c_TimeType.c_tp_basicsize = rffi.sizeof( + cts.gettype('PyDateTime_Time')) w_type = space.getattr(w_datetime, space.newtext("timedelta")) datetimeAPI.c_DeltaType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) + datetimeAPI.c_DeltaType.c_tp_basicsize = rffi.sizeof( + cts.gettype('PyDateTime_Delta')) w_type = space.getattr(w_datetime, space.newtext("tzinfo")) datetimeAPI.c_TZInfoType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) + datetimeAPI.c_TZInfoType.c_tp_basicsize = rffi.sizeof( + cts.gettype('PyDateTime_TZInfo')) datetimeAPI.c_Date_FromDate = llhelper( _PyDate_FromDate.api_func.functype, @@ -124,7 +132,7 @@ # app level datetime.date. If a c-extension class uses datetime.date for its # base class and defines a tp_dealloc, we will get this: # c_class->tp_dealloc == tp_dealloc_func - # c_class->tp_base == datetime.date, + # c_class->tp_base == datetime.date, # datetime.date->tp_dealloc = _PyPy_subtype_dealloc # datetime.date->tp_base = W_DateTime_Date # W_DateTime_Date->tp_dealloc = _PyPy_subtype_dealloc @@ -149,7 +157,7 @@ # No tzinfo return py_datetime = rffi.cast(PyDateTime_Time, py_obj) - w_tzinfo = space.getattr(w_obj, space.newtext('tzinfo')) + w_tzinfo = space.getattr(w_obj, space.newtext('_tzinfo')) if space.is_none(w_tzinfo): py_datetime.c_hastzinfo = cts.cast('unsigned char', 0) py_datetime.c_tzinfo = lltype.nullptr(PyObject.TO) diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py --- a/pypy/module/cpyext/test/test_arraymodule.py +++ b/pypy/module/cpyext/test/test_arraymodule.py @@ -187,7 +187,4 @@ self.attrib = True import gc module.subclass_with_attribute(Sub, "addattrib", "attrib", gc.collect) - if self.runappdirect: - assert Sub.__module__ == 'pypy.module.cpyext.test.test_arraymodule' - assert str(Sub) == "" - + assert Sub.__module__ == __name__ diff --git a/pypy/module/cpyext/test/test_datetime.py b/pypy/module/cpyext/test/test_datetime.py --- a/pypy/module/cpyext/test/test_datetime.py +++ b/pypy/module/cpyext/test/test_datetime.py @@ -1,3 +1,5 @@ +import pytest + from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.cdatetime import * @@ -82,6 +84,14 @@ date = datetime.datetime.fromtimestamp(0) assert space.unwrap(space.str(w_date)) == str(date) + @pytest.mark.parametrize('name', ['Time', 'DateTime', 'Date', 'Delta']) + def test_basicsize(self, space, name): + datetime = _PyDateTime_Import(space) + py_size = getattr(datetime, "c_%sType" % name).c_tp_basicsize + c_size = rffi.sizeof(cts.gettype("PyDateTime_%s" % name)) + assert py_size == c_size + + class AppTestDatetime(AppTestCpythonExtensionBase): def test_CAPI(self): module = self.import_extension('foo', [ @@ -271,9 +281,9 @@ 6, 6, 6, 6, args, PyDateTimeAPI->TimeType); """), ("datetime_with_tzinfo", "METH_O", - """ + """ PyObject * obj; - int tzrefcnt = args->ob_refcnt; + int tzrefcnt = args->ob_refcnt; PyDateTime_IMPORT; obj = PyDateTimeAPI->DateTime_FromDateAndTime( 2000, 6, 6, 6, 6, 6, 6, args, @@ -291,7 +301,7 @@ return NULL; } return obj; - + """), ], prologue='#include "datetime.h"\n') from datetime import tzinfo, datetime, timedelta, time @@ -339,4 +349,4 @@ assert module.checks(o) == (True,) * 3 + (False,) * 7 # isinstance(datetime, date) o = tzinfo() assert module.checks(o) == (False,) * 8 + (True,) * 2 - + diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py --- a/pypy/module/cpyext/test/test_unicodeobject.py +++ b/pypy/module/cpyext/test/test_unicodeobject.py @@ -27,11 +27,8 @@ if(PyUnicode_GetSize(s) != 11) { result = -PyUnicode_GetSize(s); } -#ifdef PYPY_VERSION - // Slightly silly test that tp_basicsize is reasonable. - if(s->ob_type->tp_basicsize != sizeof(void*)*12) + if(s->ob_type->tp_basicsize != sizeof(PyUnicodeObject)) result = s->ob_type->tp_basicsize; -#endif // PYPY_VERSION Py_DECREF(s); return PyLong_FromLong(result); """), From pypy.commits at gmail.com Fri Apr 20 20:32:52 2018 From: pypy.commits at gmail.com (wlav) Date: Fri, 20 Apr 2018 17:32:52 -0700 (PDT) Subject: [pypy-commit] pypy cppyy-packaging: naming consistency w/ CPython/cppyy Message-ID: <5ada86b4.93e51c0a.9807b.4544@mx.google.com> Author: Wim Lavrijsen Branch: cppyy-packaging Changeset: r94400:0df7710aad8b Date: 2017-10-30 13:14 -0700 http://bitbucket.org/pypy/pypy/changeset/0df7710aad8b/ Log: naming consistency w/ CPython/cppyy diff --git a/pypy/module/_cppyy/__init__.py b/pypy/module/_cppyy/__init__.py --- a/pypy/module/_cppyy/__init__.py +++ b/pypy/module/_cppyy/__init__.py @@ -13,7 +13,7 @@ '_set_function_generator': 'interp_cppyy.set_function_generator', '_register_class' : 'interp_cppyy.register_class', '_get_nullptr' : 'interp_cppyy.get_nullptr', - 'CPPClassBase' : 'interp_cppyy.W_CPPClass', + 'CPPInstanceBase' : 'interp_cppyy.W_CPPInstance', 'addressof' : 'interp_cppyy.addressof', '_bind_object' : 'interp_cppyy._bind_object', 'bind_object' : 'interp_cppyy.bind_object', diff --git a/pypy/module/_cppyy/capi/loadable_capi.py b/pypy/module/_cppyy/capi/loadable_capi.py --- a/pypy/module/_cppyy/capi/loadable_capi.py +++ b/pypy/module/_cppyy/capi/loadable_capi.py @@ -607,7 +607,7 @@ """Return a python string taking into account \0""" from pypy.module._cppyy import interp_cppyy - cppstr = space.interp_w(interp_cppyy.W_CPPClass, w_self, can_be_None=False) + cppstr = space.interp_w(interp_cppyy.W_CPPInstance, w_self, can_be_None=False) return space.newtext(c_stdstring2charp(space, cppstr._rawobject)) # setup pythonizations for later use at run-time diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py --- a/pypy/module/_cppyy/converter.py +++ b/pypy/module/_cppyy/converter.py @@ -22,8 +22,8 @@ def get_rawobject(space, w_obj, can_be_None=True): - from pypy.module._cppyy.interp_cppyy import W_CPPClass - cppinstance = space.interp_w(W_CPPClass, w_obj, can_be_None=can_be_None) + from pypy.module._cppyy.interp_cppyy import W_CPPInstance + cppinstance = space.interp_w(W_CPPInstance, w_obj, can_be_None=can_be_None) if cppinstance: rawobject = cppinstance.get_rawobject() assert lltype.typeOf(rawobject) == capi.C_OBJECT @@ -31,15 +31,15 @@ return capi.C_NULL_OBJECT def set_rawobject(space, w_obj, address): - from pypy.module._cppyy.interp_cppyy import W_CPPClass - cppinstance = space.interp_w(W_CPPClass, w_obj, can_be_None=True) + from pypy.module._cppyy.interp_cppyy import W_CPPInstance + cppinstance = space.interp_w(W_CPPInstance, w_obj, can_be_None=True) if cppinstance: assert lltype.typeOf(cppinstance._rawobject) == capi.C_OBJECT cppinstance._rawobject = rffi.cast(capi.C_OBJECT, address) def get_rawobject_nonnull(space, w_obj): - from pypy.module._cppyy.interp_cppyy import W_CPPClass - cppinstance = space.interp_w(W_CPPClass, w_obj, can_be_None=True) + from pypy.module._cppyy.interp_cppyy import W_CPPInstance + cppinstance = space.interp_w(W_CPPInstance, w_obj, can_be_None=True) if cppinstance: cppinstance._nullcheck() rawobject = cppinstance.get_rawobject() @@ -502,8 +502,8 @@ self.clsdecl = clsdecl def _unwrap_object(self, space, w_obj): - from pypy.module._cppyy.interp_cppyy import W_CPPClass - if isinstance(w_obj, W_CPPClass): + from pypy.module._cppyy.interp_cppyy import W_CPPInstance + if isinstance(w_obj, W_CPPInstance): from pypy.module._cppyy.interp_cppyy import INSTANCE_FLAGS_IS_R_VALUE if w_obj.flags & INSTANCE_FLAGS_IS_R_VALUE: # reject moves as all are explicit @@ -534,8 +534,8 @@ class InstanceMoveConverter(InstanceRefConverter): def _unwrap_object(self, space, w_obj): # moving is same as by-ref, but have to check that move is allowed - from pypy.module._cppyy.interp_cppyy import W_CPPClass, INSTANCE_FLAGS_IS_R_VALUE - if isinstance(w_obj, W_CPPClass): + from pypy.module._cppyy.interp_cppyy import W_CPPInstance, INSTANCE_FLAGS_IS_R_VALUE + if isinstance(w_obj, W_CPPInstance): if w_obj.flags & INSTANCE_FLAGS_IS_R_VALUE: w_obj.flags &= ~INSTANCE_FLAGS_IS_R_VALUE return InstanceRefConverter._unwrap_object(self, space, w_obj) @@ -598,8 +598,8 @@ raise FastCallNotPossible def finalize_call(self, space, w_obj, call_local): - from pypy.module._cppyy.interp_cppyy import W_CPPClass - assert isinstance(w_obj, W_CPPClass) + from pypy.module._cppyy.interp_cppyy import W_CPPInstance + assert isinstance(w_obj, W_CPPInstance) r = rffi.cast(rffi.VOIDPP, call_local) w_obj._rawobject = rffi.cast(capi.C_OBJECT, r[0]) @@ -617,8 +617,8 @@ InstanceConverter.__init__(self, space, cppclass) def _unwrap_object(self, space, w_obj): - from pypy.module._cppyy.interp_cppyy import W_CPPClass - if isinstance(w_obj, W_CPPClass): + from pypy.module._cppyy.interp_cppyy import W_CPPInstance + if isinstance(w_obj, W_CPPInstance): arg = InstanceConverter._unwrap_object(self, space, w_obj) return capi.c_stdstring2stdstring(space, arg) else: diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py --- a/pypy/module/_cppyy/interp_cppyy.py +++ b/pypy/module/_cppyy/interp_cppyy.py @@ -174,7 +174,7 @@ @staticmethod def unpack_cppthis(space, w_cppinstance, declaring_scope): - cppinstance = space.interp_w(W_CPPClass, w_cppinstance, can_be_None=False) + cppinstance = space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=False) cppinstance._nullcheck() return cppinstance.get_cppthis(declaring_scope) @@ -611,7 +611,7 @@ self.scope.name) w_result = W_CPPOverload.call(self, w_cppinstance, args_w) newthis = rffi.cast(capi.C_OBJECT, self.space.uint_w(w_result)) - cppinstance = self.space.interp_w(W_CPPClass, w_cppinstance, can_be_None=True) + cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=True) if cppinstance is not None: cppinstance._rawobject = newthis memory_regulator.register(cppinstance) @@ -682,7 +682,7 @@ return offset def get(self, w_cppinstance, w_pycppclass): - cppinstance = self.space.interp_w(W_CPPClass, w_cppinstance, can_be_None=True) + cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=True) if not cppinstance: raise oefmt(self.space.w_AttributeError, "attribute access requires an instance") @@ -690,7 +690,7 @@ return self.converter.from_memory(self.space, w_cppinstance, w_pycppclass, offset) def set(self, w_cppinstance, w_value): - cppinstance = self.space.interp_w(W_CPPClass, w_cppinstance, can_be_None=True) + cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=True) if not cppinstance: raise oefmt(self.space.w_AttributeError, "attribute access requires an instance") @@ -1029,7 +1029,7 @@ W_CPPComplexClassDecl.typedef.acceptable_as_base_class = False -class W_CPPClass(W_Root): +class W_CPPInstance(W_Root): _attrs_ = ['space', 'clsdecl', '_rawobject', 'flags', 'finalizer_registered'] _immutable_fields_ = ['clsdecl'] @@ -1109,8 +1109,8 @@ # find a global overload in gbl, in __gnu_cxx (for iterators), or in the # scopes of the argument classes (TODO: implement that last option) try: - # TODO: expecting w_other to be an W_CPPClass is too limiting - other = self.space.interp_w(W_CPPClass, w_other, can_be_None=False) + # TODO: expecting w_other to be an W_CPPInstance is too limiting + other = self.space.interp_w(W_CPPInstance, w_other, can_be_None=False) for name in ["", "__gnu_cxx", "__1"]: nss = scope_byname(self.space, name) meth_idx = capi.c_get_global_operator( @@ -1132,7 +1132,7 @@ # fallback 2: direct pointer comparison (the class comparison is needed since # the first data member in a struct and the struct have the same address) - other = self.space.interp_w(W_CPPClass, w_other, can_be_None=False) # TODO: factor out + other = self.space.interp_w(W_CPPInstance, w_other, can_be_None=False) # TODO: factor out iseq = (self._rawobject == other._rawobject) and (self.clsdecl == other.clsdecl) return self.space.newbool(iseq) @@ -1176,19 +1176,19 @@ if self.flags & INSTANCE_FLAGS_PYTHON_OWNS: self.destruct() -W_CPPClass.typedef = TypeDef( - 'CPPClass', - __python_owns__ = GetSetProperty(W_CPPClass.fget_python_owns, W_CPPClass.fset_python_owns), - __init__ = interp2app(W_CPPClass.instance__init__), - __eq__ = interp2app(W_CPPClass.instance__eq__), - __ne__ = interp2app(W_CPPClass.instance__ne__), - __nonzero__ = interp2app(W_CPPClass.instance__nonzero__), - __len__ = interp2app(W_CPPClass.instance__len__), - __cmp__ = interp2app(W_CPPClass.instance__cmp__), - __repr__ = interp2app(W_CPPClass.instance__repr__), - __destruct__ = interp2app(W_CPPClass.destruct), +W_CPPInstance.typedef = TypeDef( + 'CPPInstance', + __python_owns__ = GetSetProperty(W_CPPInstance.fget_python_owns, W_CPPInstance.fset_python_owns), + __init__ = interp2app(W_CPPInstance.instance__init__), + __eq__ = interp2app(W_CPPInstance.instance__eq__), + __ne__ = interp2app(W_CPPInstance.instance__ne__), + __nonzero__ = interp2app(W_CPPInstance.instance__nonzero__), + __len__ = interp2app(W_CPPInstance.instance__len__), + __cmp__ = interp2app(W_CPPInstance.instance__cmp__), + __repr__ = interp2app(W_CPPInstance.instance__repr__), + __destruct__ = interp2app(W_CPPInstance.destruct), ) -W_CPPClass.typedef.acceptable_as_base_class = True +W_CPPInstance.typedef.acceptable_as_base_class = True class MemoryRegulator: @@ -1200,7 +1200,7 @@ # Note that for now, the associated test carries an m_padding to make # a difference in the addresses. def __init__(self): - self.objects = rweakref.RWeakValueDictionary(int, W_CPPClass) + self.objects = rweakref.RWeakValueDictionary(int, W_CPPInstance) def register(self, obj): if not obj._rawobject: @@ -1266,8 +1266,8 @@ return obj # fresh creation - w_cppinstance = space.allocate_instance(W_CPPClass, w_pycppclass) - cppinstance = space.interp_w(W_CPPClass, w_cppinstance, can_be_None=False) + w_cppinstance = space.allocate_instance(W_CPPInstance, w_pycppclass) + cppinstance = space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=False) cppinstance.__init__(space, clsdecl, rawobject, is_ref, python_owns) memory_regulator.register(cppinstance) return w_cppinstance @@ -1311,7 +1311,7 @@ def move(space, w_obj): """Casts the given instance into an C++-style rvalue.""" - obj = space.interp_w(W_CPPClass, w_obj, can_be_None=True) + obj = space.interp_w(W_CPPInstance, w_obj, can_be_None=True) if obj: obj.flags |= INSTANCE_FLAGS_IS_R_VALUE return w_obj diff --git a/pypy/module/_cppyy/pythonify.py b/pypy/module/_cppyy/pythonify.py --- a/pypy/module/_cppyy/pythonify.py +++ b/pypy/module/_cppyy/pythonify.py @@ -7,7 +7,7 @@ # Metaclasses are needed to store C++ static data members as properties. Since # the interp-level does not support metaclasses, they are created at app-level. # These are the metaclass base classes: -class CPPMetaScope(type): +class CPPScope(type): def __getattr__(self, name): try: return get_scoped_pycppitem(self, name) # will cache on self @@ -15,11 +15,11 @@ raise AttributeError("%s object has no attribute '%s' (details: %s)" % (self, name, str(e))) -class CPPMetaNamespace(CPPMetaScope): +class CPPMetaNamespace(CPPScope): def __dir__(self): return self.__cppdecl__.__dir__() -class CPPMetaClass(CPPMetaScope): +class CPPClass(CPPScope): pass # namespace base class (class base class defined in _init_pythonify) @@ -173,7 +173,7 @@ # get a list of base classes for class creation bases = [get_pycppclass(base) for base in decl.get_base_names()] if not bases: - bases = [CPPClass,] + bases = [CPPInstance,] else: # it's possible that the required class now has been built if one of # the base classes uses it in e.g. a function interface @@ -214,7 +214,7 @@ # create a metaclass to allow properties (for static data write access) metabases = [type(base) for base in bases] - metacpp = type(CPPMetaScope)(cl_name+'_meta', _drop_cycles(metabases), d_meta) + metacpp = type(CPPScope)(cl_name+'_meta', _drop_cycles(metabases), d_meta) # create the python-side C++ class pycls = metacpp(cl_name, _drop_cycles(bases), d_class) @@ -412,11 +412,11 @@ # at pypy-c startup, rather than on the "import _cppyy" statement import _cppyy - # root of all proxy classes: CPPClass in pythonify exists to combine the - # CPPMetaScope metaclass with the interp-level CPPClassBase - global CPPClass - class CPPClass(_cppyy.CPPClassBase): - __metaclass__ = CPPMetaScope + # root of all proxy classes: CPPInstance in pythonify exists to combine + # the CPPScope metaclass with the interp-level CPPInstanceBase + global CPPInstance + class CPPInstance(_cppyy.CPPInstanceBase): + __metaclass__ = CPPScope pass # class generator callback diff --git a/pypy/module/_cppyy/test/test_zjit.py b/pypy/module/_cppyy/test/test_zjit.py --- a/pypy/module/_cppyy/test/test_zjit.py +++ b/pypy/module/_cppyy/test/test_zjit.py @@ -282,7 +282,7 @@ inst = interp_cppyy._bind_object(space, FakeInt(0), cls, True) cls.get_overload("__init__").call(inst, [FakeInt(0)]) cppmethod = cls.get_overload(method_name) - assert isinstance(inst, interp_cppyy.W_CPPClass) + assert isinstance(inst, interp_cppyy.W_CPPInstance) i = 10 while i > 0: drv.jit_merge_point(inst=inst, cppmethod=cppmethod, i=i) From pypy.commits at gmail.com Fri Apr 20 20:32:55 2018 From: pypy.commits at gmail.com (wlav) Date: Fri, 20 Apr 2018 17:32:55 -0700 (PDT) Subject: [pypy-commit] pypy cppyy-packaging: fix C++ warnings Message-ID: <5ada86b7.06581c0a.fd34c.830e@mx.google.com> Author: Wim Lavrijsen Branch: cppyy-packaging Changeset: r94401:edb9132eda33 Date: 2017-10-30 13:17 -0700 http://bitbucket.org/pypy/pypy/changeset/edb9132eda33/ Log: fix C++ warnings diff --git a/pypy/module/_cppyy/test/advancedcpp.h b/pypy/module/_cppyy/test/advancedcpp.h --- a/pypy/module/_cppyy/test/advancedcpp.h +++ b/pypy/module/_cppyy/test/advancedcpp.h @@ -59,7 +59,7 @@ class a_class { // for esoteric inheritance testing public: a_class() { m_a = 1; m_da = 1.1; } - ~a_class() {} + virtual ~a_class() {} virtual int get_value() = 0; public: @@ -221,6 +221,7 @@ //=========================================================================== class some_abstract_class { // to test abstract class handling public: + virtual ~some_abstract_class() {} virtual void a_virtual_method() = 0; }; diff --git a/pypy/module/_cppyy/test/fragile.h b/pypy/module/_cppyy/test/fragile.h --- a/pypy/module/_cppyy/test/fragile.h +++ b/pypy/module/_cppyy/test/fragile.h @@ -30,6 +30,7 @@ void overload(int, no_such_class* p = 0) {} }; + static const int dummy_location = 0xdead; class E { @@ -105,6 +106,7 @@ class M { public: + virtual ~M() {} enum E1 { kOnce=42 }; enum E2 { kTwice=12 }; }; From pypy.commits at gmail.com Fri Apr 20 20:33:09 2018 From: pypy.commits at gmail.com (wlav) Date: Fri, 20 Apr 2018 17:33:09 -0700 (PDT) Subject: [pypy-commit] pypy cppyy-packaging: merge default into branch Message-ID: <5ada86c5.06051c0a.4ee73.4f0d@mx.google.com> Author: Wim Lavrijsen Branch: cppyy-packaging Changeset: r94402:b74ad9bd1274 Date: 2018-01-26 14:27 -0800 http://bitbucket.org/pypy/pypy/changeset/b74ad9bd1274/ Log: merge default into branch diff too long, truncating to 2000 out of 93932 lines diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -44,3 +44,10 @@ d72f9800a42b46a8056951b1da2426d2c2d8d502 release-pypy3.5-v5.9.0 03d614975835870da65ff0481e1edad68ebbcb8d release-pypy2.7-v5.9.0 84a2f3e6a7f88f2fe698e473998755b3bd1a12e2 release-pypy2.7-v5.9.0 +0e7ea4fe15e82d5124e805e2e4a37cae1a402d4b release-pypy2.7-v5.10.0 +a91df6163fb76df245091f741dbf6a23ddc72374 release-pypy3.5-v5.10.0 +a91df6163fb76df245091f741dbf6a23ddc72374 release-pypy3.5-v5.10.0 +0000000000000000000000000000000000000000 release-pypy3.5-v5.10.0 +0000000000000000000000000000000000000000 release-pypy3.5-v5.10.0 +09f9160b643e3f02ccb8c843b2fbb4e5cbf54082 release-pypy3.5-v5.10.0 +3f6eaa010fce78cc7973bdc1dfdb95970f08fed2 release-pypy3.5-v5.10.1 diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -30,7 +30,7 @@ DEALINGS IN THE SOFTWARE. -PyPy Copyright holders 2003-2017 +PyPy Copyright holders 2003-2018 ----------------------------------- Except when otherwise stated (look for LICENSE files or information at @@ -339,8 +339,10 @@ Stanisław Halik Julien Phalip Roman Podoliaka + Steve Papanik Eli Stevens Boglarka Vezer + gabrielg PavloKapyshin Tomer Chachamu Christopher Groskopf @@ -363,11 +365,13 @@ Konrad Delong Dinu Gherman pizi + Tomáš Pružina James Robert Armin Ronacher Diana Popa Mads Kiilerich Brett Cannon + Caleb Hattingh aliceinwire Zooko Wilcox-O Hearn James Lan @@ -388,6 +392,7 @@ Jason Madden Yaroslav Fedevych Even Wiik Thomassen + m at funkyhat.org Stefan Marr Heinrich-Heine University, Germany diff --git a/_pytest/terminal.py b/_pytest/terminal.py --- a/_pytest/terminal.py +++ b/_pytest/terminal.py @@ -366,11 +366,11 @@ EXIT_OK, EXIT_TESTSFAILED, EXIT_INTERRUPTED, EXIT_USAGEERROR, EXIT_NOTESTSCOLLECTED) if exitstatus in summary_exit_codes: - self.config.hook.pytest_terminal_summary(terminalreporter=self) self.summary_errors() self.summary_failures() self.summary_warnings() self.summary_passes() + self.config.hook.pytest_terminal_summary(terminalreporter=self) if exitstatus == EXIT_INTERRUPTED: self._report_keyboardinterrupt() del self._keyboardinterrupt_memo diff --git a/extra_tests/requirements.txt b/extra_tests/requirements.txt new file mode 100644 --- /dev/null +++ b/extra_tests/requirements.txt @@ -0,0 +1,3 @@ +pytest +hypothesis +vmprof diff --git a/extra_tests/test_bytes.py b/extra_tests/test_bytes.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_bytes.py @@ -0,0 +1,84 @@ +from hypothesis import strategies as st +from hypothesis import given, example + +st_bytestring = st.binary() | st.binary().map(bytearray) + + at given(st_bytestring, st_bytestring, st_bytestring) +def test_find(u, prefix, suffix): + s = prefix + u + suffix + assert 0 <= s.find(u) <= len(prefix) + assert s.find(u, len(prefix), len(s) - len(suffix)) == len(prefix) + + at given(st_bytestring, st_bytestring, st_bytestring) +def test_index(u, prefix, suffix): + s = prefix + u + suffix + assert 0 <= s.index(u) <= len(prefix) + assert s.index(u, len(prefix), len(s) - len(suffix)) == len(prefix) + + at given(st_bytestring, st_bytestring, st_bytestring) +def test_rfind(u, prefix, suffix): + s = prefix + u + suffix + assert s.rfind(u) >= len(prefix) + assert s.rfind(u, len(prefix), len(s) - len(suffix)) == len(prefix) + + at given(st_bytestring, st_bytestring, st_bytestring) +def test_rindex(u, prefix, suffix): + s = prefix + u + suffix + assert s.rindex(u) >= len(prefix) + assert s.rindex(u, len(prefix), len(s) - len(suffix)) == len(prefix) + +def adjust_indices(u, start, end): + if end < 0: + end = max(end + len(u), 0) + else: + end = min(end, len(u)) + if start < 0: + start = max(start + len(u), 0) + return start, end + + at given(st_bytestring, st_bytestring) +def test_startswith_basic(u, v): + assert u.startswith(v) is (u[:len(v)] == v) + + at example(b'x', b'', 1) + at example(b'x', b'', 2) + at given(st_bytestring, st_bytestring, st.integers()) +def test_startswith_start(u, v, start): + expected = u[start:].startswith(v) if v else (start <= len(u)) + assert u.startswith(v, start) is expected + + at example(b'x', b'', 1, 0) + at example(b'xx', b'', -1, 0) + at given(st_bytestring, st_bytestring, st.integers(), st.integers()) +def test_startswith_3(u, v, start, end): + if v: + expected = u[start:end].startswith(v) + else: # CPython leaks implementation details in this case + start0, end0 = adjust_indices(u, start, end) + expected = start0 <= len(u) and start0 <= end0 + assert u.startswith(v, start, end) is expected + + at given(st_bytestring, st_bytestring) +def test_endswith_basic(u, v): + if len(v) > len(u): + assert u.endswith(v) is False + else: + assert u.endswith(v) is (u[len(u) - len(v):] == v) + + at example(b'x', b'', 1) + at example(b'x', b'', 2) + at given(st_bytestring, st_bytestring, st.integers()) +def test_endswith_2(u, v, start): + expected = u[start:].endswith(v) if v else (start <= len(u)) + assert u.endswith(v, start) is expected + + at example(b'x', b'', 1, 0) + at example(b'xx', b'', -1, 0) + at given(st_bytestring, st_bytestring, st.integers(), st.integers()) +def test_endswith_3(u, v, start, end): + if v: + expected = u[start:end].endswith(v) + else: # CPython leaks implementation details in this case + start0, end0 = adjust_indices(u, start, end) + expected = start0 <= len(u) and start0 <= end0 + assert u.endswith(v, start, end) is expected diff --git a/extra_tests/test_json.py b/extra_tests/test_json.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_json.py @@ -0,0 +1,33 @@ +import pytest +import json +from hypothesis import given, strategies + +def is_(x, y): + return type(x) is type(y) and x == y + +def test_no_ensure_ascii(): + assert is_(json.dumps(u"\u1234", ensure_ascii=False), u'"\u1234"') + assert is_(json.dumps("\xc0", ensure_ascii=False), '"\xc0"') + with pytest.raises(UnicodeDecodeError) as excinfo: + json.dumps((u"\u1234", "\xc0"), ensure_ascii=False) + assert str(excinfo.value).startswith( + "'ascii' codec can't decode byte 0xc0 ") + with pytest.raises(UnicodeDecodeError) as excinfo: + json.dumps(("\xc0", u"\u1234"), ensure_ascii=False) + assert str(excinfo.value).startswith( + "'ascii' codec can't decode byte 0xc0 ") + +def test_issue2191(): + assert is_(json.dumps(u"xxx", ensure_ascii=False), u'"xxx"') + +jsondata = strategies.recursive( + strategies.none() | + strategies.booleans() | + strategies.floats(allow_nan=False) | + strategies.text(), + lambda children: strategies.lists(children) | + strategies.dictionaries(strategies.text(), children)) + + at given(jsondata) +def test_roundtrip(d): + assert json.loads(json.dumps(d)) == d diff --git a/extra_tests/test_textio.py b/extra_tests/test_textio.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_textio.py @@ -0,0 +1,48 @@ +from hypothesis import given, strategies as st + +from io import BytesIO, TextIOWrapper +import os + +def translate_newlines(text): + text = text.replace('\r\n', '\n') + text = text.replace('\r', '\n') + return text.replace('\n', os.linesep) + + at st.composite +def st_readline_universal( + draw, st_nlines=st.integers(min_value=0, max_value=10)): + n_lines = draw(st_nlines) + lines = draw(st.lists( + st.text(st.characters(blacklist_characters='\r\n')), + min_size=n_lines, max_size=n_lines)) + limits = [] + for line in lines: + limit = draw(st.integers(min_value=0, max_value=len(line) + 5)) + limits.append(limit) + limits.append(-1) + endings = draw(st.lists( + st.sampled_from(['\n', '\r', '\r\n']), + min_size=n_lines, max_size=n_lines)) + return ( + ''.join(line + ending for line, ending in zip(lines, endings)), + limits) + + at given(data=st_readline_universal(), + mode=st.sampled_from(['\r', '\n', '\r\n', '', None])) +def test_readline(data, mode): + txt, limits = data + textio = TextIOWrapper( + BytesIO(txt.encode('utf-8', 'surrogatepass')), + encoding='utf-8', errors='surrogatepass', newline=mode) + lines = [] + for limit in limits: + line = textio.readline(limit) + if limit >= 0: + assert len(line) <= limit + if line: + lines.append(line) + elif limit: + break + if mode is None: + txt = translate_newlines(txt) + assert txt.startswith(u''.join(lines)) diff --git a/extra_tests/test_unicode.py b/extra_tests/test_unicode.py --- a/extra_tests/test_unicode.py +++ b/extra_tests/test_unicode.py @@ -1,3 +1,4 @@ +import sys import pytest from hypothesis import strategies as st from hypothesis import given, settings, example @@ -32,3 +33,89 @@ @given(s=st.text()) def test_composition(s, norm1, norm2, norm3): assert normalize(norm2, normalize(norm1, s)) == normalize(norm3, s) + + at given(st.text(), st.text(), st.text()) +def test_find(u, prefix, suffix): + s = prefix + u + suffix + assert 0 <= s.find(u) <= len(prefix) + assert s.find(u, len(prefix), len(s) - len(suffix)) == len(prefix) + + at given(st.text(), st.text(), st.text()) +def test_index(u, prefix, suffix): + s = prefix + u + suffix + assert 0 <= s.index(u) <= len(prefix) + assert s.index(u, len(prefix), len(s) - len(suffix)) == len(prefix) + + at given(st.text(), st.text(), st.text()) +def test_rfind(u, prefix, suffix): + s = prefix + u + suffix + assert s.rfind(u) >= len(prefix) + assert s.rfind(u, len(prefix), len(s) - len(suffix)) == len(prefix) + + at given(st.text(), st.text(), st.text()) +def test_rindex(u, prefix, suffix): + s = prefix + u + suffix + assert s.rindex(u) >= len(prefix) + assert s.rindex(u, len(prefix), len(s) - len(suffix)) == len(prefix) + +def adjust_indices(u, start, end): + if end < 0: + end = max(end + len(u), 0) + else: + end = min(end, len(u)) + if start < 0: + start = max(start + len(u), 0) + return start, end + + at given(st.text(), st.text()) +def test_startswith_basic(u, v): + assert u.startswith(v) is (u[:len(v)] == v) + + at example(u'x', u'', 1) + at example(u'x', u'', 2) + at given(st.text(), st.text(), st.integers()) +def test_startswith_2(u, v, start): + if v or sys.version_info[0] == 2: + expected = u[start:].startswith(v) + else: # CPython leaks implementation details in this case + expected = start <= len(u) + assert u.startswith(v, start) is expected + + at example(u'x', u'', 1, 0) + at example(u'xx', u'', -1, 0) + at given(st.text(), st.text(), st.integers(), st.integers()) +def test_startswith_3(u, v, start, end): + if v or sys.version_info[0] == 2: + expected = u[start:end].startswith(v) + else: # CPython leaks implementation details in this case + start0, end0 = adjust_indices(u, start, end) + expected = start0 <= len(u) and start0 <= end0 + assert u.startswith(v, start, end) is expected + + at given(st.text(), st.text()) +def test_endswith_basic(u, v): + if len(v) > len(u): + assert u.endswith(v) is False + else: + assert u.endswith(v) is (u[len(u) - len(v):] == v) + + at example(u'x', u'', 1) + at example(u'x', u'', 2) + at given(st.text(), st.text(), st.integers()) +def test_endswith_2(u, v, start): + if v or sys.version_info[0] == 2: + expected = u[start:].endswith(v) + else: # CPython leaks implementation details in this case + expected = start <= len(u) + assert u.endswith(v, start) is expected + + at example(u'x', u'', 1, 0) + at example(u'xx', u'', -1, 0) + at given(st.text(), st.text(), st.integers(), st.integers()) +def test_endswith_3(u, v, start, end): + if v or sys.version_info[0] == 2: + expected = u[start:end].endswith(v) + else: # CPython leaks implementation details in this case + start0, end0 = adjust_indices(u, start, end) + expected = start0 <= len(u) and start0 <= end0 + assert u.endswith(v, start, end) is expected diff --git a/extra_tests/test_vmprof_greenlet.py b/extra_tests/test_vmprof_greenlet.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_vmprof_greenlet.py @@ -0,0 +1,28 @@ +import time +import pytest +import greenlet +vmprof = pytest.importorskip('vmprof') + +def count_samples(filename): + stats = vmprof.read_profile(filename) + return len(stats.profiles) + +def cpuburn(duration): + end = time.time() + duration + while time.time() < end: + pass + +def test_sampling_inside_callback(tmpdir): + # see also test_sampling_inside_callback inside + # pypy/module/_continuation/test/test_stacklet.py + # + G = greenlet.greenlet(cpuburn) + fname = tmpdir.join('log.vmprof') + with fname.open('w+b') as f: + vmprof.enable(f.fileno(), 1/250.0) + G.switch(0.1) + vmprof.disable() + + samples = count_samples(str(fname)) + # 0.1 seconds at 250Hz should be 25 samples + assert 23 < samples < 27 diff --git a/lib-python/2.7/ctypes/__init__.py b/lib-python/2.7/ctypes/__init__.py --- a/lib-python/2.7/ctypes/__init__.py +++ b/lib-python/2.7/ctypes/__init__.py @@ -360,14 +360,15 @@ self._FuncPtr = _FuncPtr if handle is None: - if flags & _FUNCFLAG_CDECL: - pypy_dll = _ffi.CDLL(name, mode) - else: - pypy_dll = _ffi.WinDLL(name, mode) - self.__pypy_dll__ = pypy_dll - handle = int(pypy_dll) - if _sys.maxint > 2 ** 32: - handle = int(handle) # long -> int + handle = 0 + if flags & _FUNCFLAG_CDECL: + pypy_dll = _ffi.CDLL(name, mode, handle) + else: + pypy_dll = _ffi.WinDLL(name, mode, handle) + self.__pypy_dll__ = pypy_dll + handle = int(pypy_dll) + if _sys.maxint > 2 ** 32: + handle = int(handle) # long -> int self._handle = handle def __repr__(self): diff --git a/lib-python/2.7/inspect.py b/lib-python/2.7/inspect.py --- a/lib-python/2.7/inspect.py +++ b/lib-python/2.7/inspect.py @@ -40,6 +40,10 @@ import linecache from operator import attrgetter from collections import namedtuple +try: + from cpyext import is_cpyext_function as _is_cpyext_function +except ImportError: + _is_cpyext_function = lambda obj: False # These constants are from Include/code.h. CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS = 0x1, 0x2, 0x4, 0x8 @@ -230,7 +234,7 @@ __doc__ documentation string __name__ original name of this function or method __self__ instance to which a method is bound, or None""" - return isinstance(object, types.BuiltinFunctionType) + return isinstance(object, types.BuiltinFunctionType) or _is_cpyext_function(object) def isroutine(object): """Return true if the object is any kind of function or method.""" diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py --- a/lib-python/2.7/subprocess.py +++ b/lib-python/2.7/subprocess.py @@ -1296,7 +1296,7 @@ 'copyfile' in caller.f_globals): dest_dir = sys.pypy_resolvedirof(target_executable) src_dir = sys.pypy_resolvedirof(sys.executable) - for libname in ['libpypy-c.so', 'libpypy-c.dylib']: + for libname in ['libpypy-c.so', 'libpypy-c.dylib', 'libpypy-c.dll']: dest_library = os.path.join(dest_dir, libname) src_library = os.path.join(src_dir, libname) if os.path.exists(src_library): diff --git a/lib-python/2.7/test/test_urllib2net.py b/lib-python/2.7/test/test_urllib2net.py --- a/lib-python/2.7/test/test_urllib2net.py +++ b/lib-python/2.7/test/test_urllib2net.py @@ -286,7 +286,7 @@ self.assertEqual(u.fp._sock.fp._sock.gettimeout(), 120) u.close() - FTP_HOST = 'ftp://ftp.debian.org/debian/' + FTP_HOST = 'ftp://www.pythontest.net/' def test_ftp_basic(self): self.assertIsNone(socket.getdefaulttimeout()) diff --git a/lib-python/2.7/warnings.py b/lib-python/2.7/warnings.py --- a/lib-python/2.7/warnings.py +++ b/lib-python/2.7/warnings.py @@ -43,11 +43,12 @@ unicodetype = unicode except NameError: unicodetype = () + template = "%s: %s: %s\n" try: message = str(message) except UnicodeEncodeError: - pass - s = "%s: %s: %s\n" % (lineno, category.__name__, message) + template = unicode(template) + s = template % (lineno, category.__name__, message) line = linecache.getline(filename, lineno) if line is None else line if line: line = line.strip() diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py --- a/lib_pypy/_ctypes/array.py +++ b/lib_pypy/_ctypes/array.py @@ -12,7 +12,8 @@ if cls == (_CData,): # this is the Array class defined below res._ffiarray = None return res - if not hasattr(res, '_length_') or not isinstance(res._length_, int): + if not hasattr(res, '_length_') or not isinstance(res._length_, + (int, long)): raise AttributeError( "class must define a '_length_' attribute, " "which must be a positive integer") diff --git a/lib_pypy/_ctypes_test.py b/lib_pypy/_ctypes_test.py --- a/lib_pypy/_ctypes_test.py +++ b/lib_pypy/_ctypes_test.py @@ -21,5 +21,11 @@ with fp: imp.load_module('_ctypes_test', fp, filename, description) except ImportError: + if os.name == 'nt': + # hack around finding compilers on win32 + try: + import setuptools + except ImportError: + pass print('could not find _ctypes_test in %s' % output_dir) _pypy_testcapi.compile_shared('_ctypes_test.c', '_ctypes_test', output_dir) diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_testcapi.py --- a/lib_pypy/_testcapi.py +++ b/lib_pypy/_testcapi.py @@ -16,4 +16,10 @@ with fp: imp.load_module('_testcapi', fp, filename, description) except ImportError: + if os.name == 'nt': + # hack around finding compilers on win32 + try: + import setuptools + except ImportError: + pass _pypy_testcapi.compile_shared(cfile, '_testcapi', output_dir) diff --git a/lib_pypy/_tkinter/app.py b/lib_pypy/_tkinter/app.py --- a/lib_pypy/_tkinter/app.py +++ b/lib_pypy/_tkinter/app.py @@ -180,6 +180,9 @@ if err == tklib.TCL_ERROR: self.raiseTclError() + def interpaddr(self): + return int(tkffi.cast('size_t', self.interp)) + def _var_invoke(self, func, *args, **kwargs): if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread(): # The current thread is not the interpreter thread. diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,11 +1,12 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.11.2 +Version: 1.11.4 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski Author-email: python-cffi at googlegroups.com License: MIT +Description-Content-Type: UNKNOWN Description: CFFI ==== @@ -27,5 +28,7 @@ Classifier: Programming Language :: Python :: 3.2 Classifier: Programming Language :: Python :: 3.3 Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: PyPy diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI from .error import CDefError, FFIError, VerificationError, VerificationMissing -__version__ = "1.11.2" -__version_info__ = (1, 11, 2) +__version__ = "1.11.4" +__version_info__ = (1, 11, 4) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h --- a/lib_pypy/cffi/_cffi_include.h +++ b/lib_pypy/cffi/_cffi_include.h @@ -7,6 +7,16 @@ we can learn about Py_DEBUG from pyconfig.h, but it is unclear if the same works for the other two macros. Py_DEBUG implies them, but not the other way around. + + Issue #350 is still open: on Windows, the code here causes it to link + with PYTHON36.DLL (for example) instead of PYTHON3.DLL. A fix was + attempted in 164e526a5515 and 14ce6985e1c3, but reverted: virtualenv + does not make PYTHON3.DLL available, and so the "correctly" compiled + version would not run inside a virtualenv. We will re-apply the fix + after virtualenv has been fixed for some time. For explanation, see + issue #355. For a workaround if you want PYTHON3.DLL and don't worry + about virtualenv, see issue #350. See also 'py_limited_api' in + setuptools_ext.py. */ #if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API) # include diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h --- a/lib_pypy/cffi/_embedding.h +++ b/lib_pypy/cffi/_embedding.h @@ -247,7 +247,7 @@ if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.11.2" + "\ncompiled with cffi version: 1.11.4" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py --- a/lib_pypy/cffi/recompiler.py +++ b/lib_pypy/cffi/recompiler.py @@ -295,8 +295,9 @@ base_module_name = self.module_name.split('.')[-1] if self.ffi._embedding is not None: prnt('#define _CFFI_MODULE_NAME "%s"' % (self.module_name,)) - prnt('#define _CFFI_PYTHON_STARTUP_CODE %s' % - (self._string_literal(self.ffi._embedding),)) + prnt('static const char _CFFI_PYTHON_STARTUP_CODE[] = {') + self._print_string_literal_in_array(self.ffi._embedding) + prnt('0 };') prnt('#ifdef PYPY_VERSION') prnt('# define _CFFI_PYTHON_STARTUP_FUNC _cffi_pypyinit_%s' % ( base_module_name,)) @@ -1271,17 +1272,18 @@ _generate_cpy_extern_python_plus_c_ctx = \ _generate_cpy_extern_python_ctx - def _string_literal(self, s): - def _char_repr(c): - # escape with a '\' the characters '\', '"' or (for trigraphs) '?' - if c in '\\"?': return '\\' + c - if ' ' <= c < '\x7F': return c - if c == '\n': return '\\n' - return '\\%03o' % ord(c) - lines = [] - for line in s.splitlines(True) or ['']: - lines.append('"%s"' % ''.join([_char_repr(c) for c in line])) - return ' \\\n'.join(lines) + def _print_string_literal_in_array(self, s): + prnt = self._prnt + prnt('// # NB. this is not a string because of a size limit in MSVC') + for line in s.splitlines(True): + prnt(('// ' + line).rstrip()) + printed_line = '' + for c in line: + if len(printed_line) >= 76: + prnt(printed_line) + printed_line = '' + printed_line += '%d,' % (ord(c),) + prnt(printed_line) # ---------- # emitting the opcodes for individual types diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py --- a/lib_pypy/cffi/verifier.py +++ b/lib_pypy/cffi/verifier.py @@ -301,7 +301,6 @@ return suffixes def _ensure_dir(filename): - try: - os.makedirs(os.path.dirname(filename)) - except OSError: - pass + dirname = os.path.dirname(filename) + if dirname and not os.path.isdir(dirname): + os.makedirs(dirname) diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py --- a/lib_pypy/datetime.py +++ b/lib_pypy/datetime.py @@ -17,10 +17,13 @@ """ from __future__ import division -import time as _time +import time as _timemodule import math as _math import struct as _struct +# for cpyext, use these as base classes +from __pypy__._pypydatetime import dateinterop, deltainterop, timeinterop + _SENTINEL = object() def _cmp(x, y): @@ -179,7 +182,7 @@ def _build_struct_time(y, m, d, hh, mm, ss, dstflag): wday = (_ymd2ord(y, m, d) + 6) % 7 dnum = _days_before_month(y, m) + d - return _time.struct_time((y, m, d, hh, mm, ss, wday, dnum, dstflag)) + return _timemodule.struct_time((y, m, d, hh, mm, ss, wday, dnum, dstflag)) def _format_time(hh, mm, ss, us): # Skip trailing microseconds when us==0. @@ -247,7 +250,7 @@ else: push(ch) newformat = "".join(newformat) - return _time.strftime(newformat, timetuple) + return _timemodule.strftime(newformat, timetuple) # Just raise TypeError if the arg isn't None or a string. def _check_tzname(name): @@ -433,7 +436,7 @@ raise TypeError("unsupported type for timedelta %s component: %s" % (tag, type(num))) -class timedelta(object): +class timedelta(deltainterop): """Represent the difference between two datetime objects. Supported operators: @@ -489,7 +492,7 @@ if not -_MAX_DELTA_DAYS <= d <= _MAX_DELTA_DAYS: raise OverflowError("days=%d; must have magnitude <= %d" % (d, _MAX_DELTA_DAYS)) - self = object.__new__(cls) + self = deltainterop.__new__(cls) self._days = d self._seconds = s self._microseconds = us @@ -667,7 +670,7 @@ timedelta.max = timedelta(_MAX_DELTA_DAYS, 24*3600-1, 1000000-1) timedelta.resolution = timedelta(microseconds=1) -class date(object): +class date(dateinterop): """Concrete date type. Constructors: @@ -707,12 +710,12 @@ if month is None and isinstance(year, bytes) and len(year) == 4 and \ 1 <= ord(year[2]) <= 12: # Pickle support - self = object.__new__(cls) + self = dateinterop.__new__(cls) self.__setstate(year) self._hashcode = -1 return self year, month, day = _check_date_fields(year, month, day) - self = object.__new__(cls) + self = dateinterop.__new__(cls) self._year = year self._month = month self._day = day @@ -724,13 +727,13 @@ @classmethod def fromtimestamp(cls, t): "Construct a date from a POSIX timestamp (like time.time())." - y, m, d, hh, mm, ss, weekday, jday, dst = _time.localtime(t) + y, m, d, hh, mm, ss, weekday, jday, dst = _timemodule.localtime(t) return cls(y, m, d) @classmethod def today(cls): "Construct a date from time.time()." - t = _time.time() + t = _timemodule.time() return cls.fromtimestamp(t) @classmethod @@ -1061,7 +1064,7 @@ _tzinfo_class = tzinfo -class time(object): +class time(timeinterop): """Time with time zone. Constructors: @@ -1097,14 +1100,14 @@ """ if isinstance(hour, bytes) and len(hour) == 6 and ord(hour[0]) < 24: # Pickle support - self = object.__new__(cls) + self = timeinterop.__new__(cls) self.__setstate(hour, minute or None) self._hashcode = -1 return self hour, minute, second, microsecond = _check_time_fields( hour, minute, second, microsecond) _check_tzinfo_arg(tzinfo) - self = object.__new__(cls) + self = timeinterop.__new__(cls) self._hour = hour self._minute = minute self._second = second @@ -1408,7 +1411,7 @@ if isinstance(year, bytes) and len(year) == 10 and \ 1 <= ord(year[2]) <= 12: # Pickle support - self = object.__new__(cls) + self = dateinterop.__new__(cls) self.__setstate(year, month) self._hashcode = -1 return self @@ -1416,7 +1419,7 @@ hour, minute, second, microsecond = _check_time_fields( hour, minute, second, microsecond) _check_tzinfo_arg(tzinfo) - self = object.__new__(cls) + self = dateinterop.__new__(cls) self._year = year self._month = month self._day = day @@ -1461,7 +1464,7 @@ A timezone info object may be passed in as well. """ _check_tzinfo_arg(tz) - converter = _time.localtime if tz is None else _time.gmtime + converter = _timemodule.localtime if tz is None else _timemodule.gmtime self = cls._from_timestamp(converter, timestamp, tz) if tz is not None: self = tz.fromutc(self) @@ -1470,7 +1473,7 @@ @classmethod def utcfromtimestamp(cls, t): "Construct a UTC datetime from a POSIX timestamp (like time.time())." - return cls._from_timestamp(_time.gmtime, t, None) + return cls._from_timestamp(_timemodule.gmtime, t, None) @classmethod def _from_timestamp(cls, converter, timestamp, tzinfo): @@ -1493,13 +1496,13 @@ @classmethod def now(cls, tz=None): "Construct a datetime from time.time() and optional time zone info." - t = _time.time() + t = _timemodule.time() return cls.fromtimestamp(t, tz) @classmethod def utcnow(cls): "Construct a UTC datetime from time.time()." - t = _time.time() + t = _timemodule.time() return cls.utcfromtimestamp(t) @classmethod diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py --- a/lib_pypy/resource.py +++ b/lib_pypy/resource.py @@ -20,6 +20,7 @@ or via the attributes ru_utime, ru_stime, ru_maxrss, and so on.""" __metaclass__ = _structseq.structseqtype + name = "resource.struct_rusage" ru_utime = _structseq.structseqfield(0, "user time used") ru_stime = _structseq.structseqfield(1, "system time used") diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst --- a/pypy/doc/build.rst +++ b/pypy/doc/build.rst @@ -149,7 +149,7 @@ xz-devel # For lzma on PyPy3. (XXX plus the SLES11 version of libgdbm-dev and tk-dev) -On Mac OS X:: +On Mac OS X: Most of these build-time dependencies are installed alongside the Developer Tools. However, note that in order for the installation to diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -59,7 +59,7 @@ # General information about the project. project = u'PyPy' -copyright = u'2017, The PyPy Project' +copyright = u'2018, The PyPy Project' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -217,6 +217,7 @@ Alejandro J. Cura Vladimir Kryachko Gabriel + Thomas Hisch Mark Williams Kunal Grover Nathan Taylor @@ -306,8 +307,10 @@ Stanisław Halik Julien Phalip Roman Podoliaka + Steve Papanik Eli Stevens Boglarka Vezer + gabrielg PavloKapyshin Tomer Chachamu Christopher Groskopf @@ -330,11 +333,13 @@ Konrad Delong Dinu Gherman pizi + Tomáš Pružina James Robert Armin Ronacher Diana Popa Mads Kiilerich Brett Cannon + Caleb Hattingh aliceinwire Zooko Wilcox-O Hearn James Lan @@ -355,4 +360,5 @@ Jason Madden Yaroslav Fedevych Even Wiik Thomassen + m at funkyhat.org Stefan Marr diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -355,7 +355,11 @@ containers (as list items or in sets for example), the exact rule of equality used is "``if x is y or x == y``" (on both CPython and PyPy); as a consequence, because all ``nans`` are identical in PyPy, you -cannot have several of them in a set, unlike in CPython. (Issue `#1974`__) +cannot have several of them in a set, unlike in CPython. (Issue `#1974`__). +Another consequence is that ``cmp(float('nan'), float('nan')) == 0``, because +``cmp`` checks with ``is`` first whether the arguments are identical (there is +no good value to return from this call to ``cmp``, because ``cmp`` pretends +that there is a total order on floats, but that is wrong for NaNs). .. __: https://bitbucket.org/pypy/pypy/issue/1974/different-behaviour-for-collections-of @@ -541,6 +545,15 @@ ``del foo.bar`` where ``foo`` is a module (or class) that contains the function ``bar``, is significantly slower than CPython. +* Various built-in functions in CPython accept only positional arguments + and not keyword arguments. That can be considered a long-running + historical detail: newer functions tend to accept keyword arguments + and older function are occasionally fixed to do so as well. In PyPy, + most built-in functions accept keyword arguments (``help()`` shows the + argument names). But don't rely on it too much because future + versions of PyPy may have to rename the arguments if CPython starts + accepting them too. + .. _`is ignored in PyPy`: http://bugs.python.org/issue14621 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/ diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst --- a/pypy/doc/how-to-release.rst +++ b/pypy/doc/how-to-release.rst @@ -62,7 +62,7 @@ * go to pypy/tool/release and run ``force-builds.py `` The following JIT binaries should be built, however, we need more buildbots - windows, linux-32, linux-64, osx64, armhf-raring, armhf-raspberrian, armel, + windows, linux-32, linux-64, osx64, armhf-raspberrian, armel, freebsd64 * wait for builds to complete, make sure there are no failures diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,8 @@ .. toctree:: + release-v5.10.1.rst + release-v5.10.0.rst release-v5.9.0.rst release-v5.8.0.rst release-v5.7.1.rst diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -7,6 +7,7 @@ .. toctree:: whatsnew-head.rst + whatsnew-pypy2-5.10.0.rst whatsnew-pypy2-5.9.0.rst whatsnew-pypy2-5.8.0.rst whatsnew-pypy2-5.7.0.rst diff --git a/pypy/doc/release-v5.10.0.rst b/pypy/doc/release-v5.10.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-v5.10.0.rst @@ -0,0 +1,100 @@ +====================================== +PyPy2.7 and PyPy3.5 v5.10 dual release +====================================== + +The PyPy team is proud to release both PyPy2.7 v5.10 (an interpreter supporting +Python 2.7 syntax), and a final PyPy3.5 v5.10 (an interpreter for Python +3.5 syntax). The two releases are both based on much the same codebase, thus +the dual release. + +This release is an incremental release with very few new features, the main +feature being the final PyPy3.5 release that works on linux and OS X with beta +windows support. It also includes fixes for `vmprof`_ cooperation with greenlets. + +Compared to 5.9, the 5.10 release contains mostly bugfixes and small improvements. +We have in the pipeline big new features coming for PyPy 6.0 that did not make +the release cut and should be available within the next couple months. + +As always, this release is 100% compatible with the previous one and fixed +several issues and bugs raised by the growing community of PyPy users. +As always, we strongly recommend updating. + +There are quite a few important changes that are in the pipeline that did not +make it into the 5.10 release. Most important are speed improvements to cpyext +(which will make numpy and pandas a bit faster) and utf8 branch that changes +internal representation of unicode to utf8, which should help especially the +Python 3.5 version of PyPy. + +This release concludes the Mozilla Open Source `grant`_ for having a compatible +PyPy 3.5 release and we're very grateful for that. Of course, we will continue +to improve PyPy 3.5 and probably move to 3.6 during the course of 2018. + +You can download the v5.10 releases here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project. + +We would also like to thank our contributors and +encourage new people to join the project. PyPy has many +layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation +improvements, tweaking popular `modules`_ to run on pypy, or general `help`_ +with making RPython's JIT even better. + +.. _vmprof: http://vmprof.readthedocs.io +.. _grant: https://morepypy.blogspot.com/2016/08/pypy-gets-funding-from-mozilla-for.html +.. _`PyPy`: index.html +.. _`RPython`: https://rpython.readthedocs.org +.. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly +.. _`help`: project-ideas.html + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7 and CPython 3.5. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +The PyPy release supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html + +Changelog +========= + +* improve ssl handling on windows for pypy3 (makes pip work) +* improve unicode handling in various error reporters +* fix vmprof cooperation with greenlets +* fix some things in cpyext +* test and document the cmp(nan, nan) == 0 behaviour +* don't crash when calling sleep with inf or nan +* fix bugs in _io module +* inspect.isbuiltin() now returns True for functions implemented in C +* allow the sequences future-import, docstring, future-import for CPython bug compatibility +* Issue #2699: non-ascii messages in warnings +* posix.lockf +* fixes for FreeBSD platform +* add .debug files, so builds contain debugging info, instead of being stripped +* improvements to cppyy +* issue #2677 copy pure c PyBuffer_{From,To}Contiguous from cpython +* issue #2682, split firstword on any whitespace in sqlite3 +* ctypes: allow ptr[0] = foo when ptr is a pointer to struct +* matplotlib will work with tkagg backend once `matplotlib pr #9356`_ is merged +* improvements to utf32 surrogate handling +* cffi version bump to 1.11.2 + +.. _`matplotlib pr #9356`: https://github.com/matplotlib/matplotlib/pull/9356 diff --git a/pypy/doc/release-v5.10.1.rst b/pypy/doc/release-v5.10.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-v5.10.1.rst @@ -0,0 +1,63 @@ +=========== +PyPy 5.10.1 +=========== + +We have released a bugfix PyPy3.5-v5.10.1 +due to the following issues: + + * Fix ``time.sleep(float('nan')`` which would hang on windows + + * Fix missing ``errno`` constants on windows + + * Fix issue 2718_ for the REPL on linux + + * Fix an overflow in converting 3 secs to nanosecs (issue 2717_ ) + + * Flag kwarg to ``os.setxattr`` had no effect + + * Fix the winreg module for unicode entries in the registry on windows + +Note that many of these fixes are for our new beta verison of PyPy3.5 on +windows. There may be more unicode problems in the windows beta version +especially around the subject of directory- and file-names with non-ascii +characters. + +Our downloads are available now. On macos, we recommend you wait for the +Homebrew_ package. + +Thanks to those who reported the issues. + +.. _2718: https://bitbucket.org/pypy/pypy/issues/2718 +.. _2717: https://bitbucket.org/pypy/pypy/issues/2717 +.. _Homebrew: http://brewformulas.org/Pypy + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7 and CPython 3.5. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +This PyPy 3.5 release supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html + +Please update, and continue to help us make PyPy better. + +Cheers + +The PyPy Team + diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py --- a/pypy/doc/tool/makecontributor.py +++ b/pypy/doc/tool/makecontributor.py @@ -81,6 +81,7 @@ 'Yasir Suhail':['yasirs'], 'Squeaky': ['squeaky'], "Amaury Forgeot d'Arc": ['amauryfa at gmail.com'], + "Dodan Mihai": ['mihai.dodan at gmail.com'], } alias_map = {} diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -1,12 +1,16 @@ -=========================== -What's new in PyPy2.7 5.10+ -=========================== - -.. this is a revision shortly after release-pypy2.7-v5.9.0 -.. startrev:d56dadcef996 - -.. branch: cppyy-packaging -Cleanup and improve cppyy packaging - -.. branch: docs-osx-brew-openssl - +=========================== +What's new in PyPy2.7 5.10+ +=========================== + +.. this is a revision shortly after release-pypy2.7-v5.10.0 +.. startrev: 6b024edd9d12 + +.. branch: cpyext-avoid-roundtrip + +Big refactoring of some cpyext code, which avoids a lot of nonsense when +calling C from Python and vice-versa: the result is a big speedup in +function/method calls, up to 6 times faster. + +.. branch: cpyext-datetime2 + +Support ``tzinfo`` field on C-API datetime objects, fixes latest pandas HEAD diff --git a/pypy/doc/whatsnew-pypy2-5.10.0.rst b/pypy/doc/whatsnew-pypy2-5.10.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-pypy2-5.10.0.rst @@ -0,0 +1,42 @@ +========================== +What's new in PyPy2.7 5.10 +========================== + +.. this is a revision shortly after release-pypy2.7-v5.9.0 +.. startrev:d56dadcef996 + + +.. branch: cppyy-packaging + +Cleanup and improve cppyy packaging + +.. branch: docs-osx-brew-openssl + +.. branch: keep-debug-symbols + +Add a smartstrip tool, which can optionally keep the debug symbols in a +separate file, instead of just stripping them away. Use it in packaging + +.. branch: bsd-patches + +Fix failures on FreeBSD, contributed by David Naylor as patches on the issue +tracker (issues 2694, 2695, 2696, 2697) + +.. branch: run-extra-tests + +Run extra_tests/ in buildbot + +.. branch: vmprof-0.4.10 + +Upgrade the _vmprof backend to vmprof 0.4.10 + +.. branch: fix-vmprof-stacklet-switch +.. branch: fix-vmprof-stacklet-switch-2 +Fix a vmprof+continulets (i.e. greenelts, eventlet, gevent, ...) + +.. branch: win32-vcvars + +.. branch: rdict-fast-hash + +Make it possible to declare that the hash function of an r_dict is fast in RPython. + diff --git a/pypy/doc/whatsnew-pypy2-5.6.0.rst b/pypy/doc/whatsnew-pypy2-5.6.0.rst --- a/pypy/doc/whatsnew-pypy2-5.6.0.rst +++ b/pypy/doc/whatsnew-pypy2-5.6.0.rst @@ -101,7 +101,7 @@ .. branch: newinitwarn -Match CPython's stricter handling of __new/init__ arguments +Match CPython's stricter handling of ``__new__``/``__init__`` arguments .. branch: openssl-1.1 diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -11,7 +11,7 @@ To build pypy-c you need a working python environment, and a C compiler. It is possible to translate with a CPython 2.6 or later, but this is not -the preferred way, because it will take a lot longer to run � depending +the preferred way, because it will take a lot longer to run – depending on your architecture, between two and three times as long. So head to `our downloads`_ and get the latest stable version. @@ -25,8 +25,10 @@ This compiler, while the standard one for Python 2.7, is deprecated. Microsoft has made it available as the `Microsoft Visual C++ Compiler for Python 2.7`_ (the link -was checked in Nov 2016). Note that the compiler suite will be installed in -``C:\Users\\AppData\Local\Programs\Common\Microsoft\Visual C++ for Python``. +was checked in Nov 2016). Note that the compiler suite may be installed in +``C:\Users\\AppData\Local\Programs\Common\Microsoft\Visual C++ for Python`` +or in +``C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python``. A current version of ``setuptools`` will be able to find it there. For Windows 10, you must right-click the download, and under ``Properties`` -> ``Compatibility`` mark it as ``Run run this program in comatibility mode for`` @@ -41,7 +43,6 @@ ----------------------------------- We routinely test translation using v9, also known as Visual Studio 2008. -Our buildbot is still using the Express Edition, not the compiler noted above. Other configurations may work as well. The translation scripts will set up the appropriate environment variables @@ -81,6 +82,31 @@ .. _build instructions: http://pypy.org/download.html#building-from-source +Setting Up Visual Studio for building SSL in Python3 +---------------------------------------------------- + +On Python3, the ``ssl`` module is based on ``cffi``, and requires a build step after +translation. However ``distutils`` does not support the Micorosft-provided Visual C +compiler, and ``cffi`` depends on ``distutils`` to find the compiler. The +traditional solution to this problem is to install the ``setuptools`` module +via running ``-m ensurepip`` which installs ``pip`` and ``setuptools``. However +``pip`` requires ``ssl``. So we have a chicken-and-egg problem: ``ssl`` depends on +``cffi`` which depends on ``setuptools``, which depends on ``ensurepip``, which +depends on ``ssl``. + +In order to solve this, the buildbot sets an environment varaible that helps +``distutils`` find the compiler without ``setuptools``:: + + set VS90COMNTOOLS=C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python\9.0\VC\bin + +or whatever is appropriate for your machine. Note that this is not enough, you +must also copy the ``vcvarsall.bat`` file fron the ``...\9.0`` directory to the +``...\9.0\VC`` directory, and edit it, changing the lines that set +``VCINSTALLDIR`` and ``WindowsSdkDir``:: + + set VCINSTALLDIR=%~dp0\ + set WindowsSdkDir=%~dp0\..\WinSDK\ + Preparing Windows for the large build ------------------------------------- diff --git a/pypy/goal/getnightly.py b/pypy/goal/getnightly.py --- a/pypy/goal/getnightly.py +++ b/pypy/goal/getnightly.py @@ -15,7 +15,7 @@ arch = 'linux' cmd = 'wget "%s"' TAR_OPTIONS += ' --wildcards' - binfiles = "'*/bin/pypy' '*/bin/libpypy-c.so'" + binfiles = "'*/bin/pypy*' '*/bin/libpypy-c.so*'" if os.uname()[-1].startswith('arm'): arch += '-armhf-raspbian' elif sys.platform.startswith('darwin'): diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py --- a/pypy/interpreter/astcompiler/assemble.py +++ b/pypy/interpreter/astcompiler/assemble.py @@ -1,7 +1,7 @@ """Python control flow graph generation and bytecode assembly.""" +import math import os -from rpython.rlib import rfloat from rpython.rlib.objectmodel import we_are_translated from pypy.interpreter.astcompiler import ast, misc, symtable @@ -266,7 +266,7 @@ w_type = space.type(obj) if space.is_w(w_type, space.w_float): val = space.float_w(obj) - if val == 0.0 and rfloat.copysign(1., val) < 0: + if val == 0.0 and math.copysign(1., val) < 0: w_key = space.newtuple([obj, space.w_float, space.w_None]) else: w_key = space.newtuple([obj, space.w_float]) @@ -276,9 +276,9 @@ real = space.float_w(w_real) imag = space.float_w(w_imag) real_negzero = (real == 0.0 and - rfloat.copysign(1., real) < 0) + math.copysign(1., real) < 0) imag_negzero = (imag == 0.0 and - rfloat.copysign(1., imag) < 0) + math.copysign(1., imag) < 0) if real_negzero and imag_negzero: tup = [obj, space.w_complex, space.w_None, space.w_None, space.w_None] diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py --- a/pypy/interpreter/astcompiler/test/test_astbuilder.py +++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py @@ -1246,3 +1246,7 @@ exc = py.test.raises(SyntaxError, self.get_ast, input).value assert exc.msg == ("(unicode error) 'unicodeescape' codec can't decode" " bytes in position 0-1: truncated \\xXX escape") + input = "u'\\x1'" + exc = py.test.raises(SyntaxError, self.get_ast, input).value + assert exc.msg == ("(unicode error) 'unicodeescape' codec can't decode" + " bytes in position 0-2: truncated \\xXX escape") diff --git a/pypy/interpreter/pyparser/future.py b/pypy/interpreter/pyparser/future.py --- a/pypy/interpreter/pyparser/future.py +++ b/pypy/interpreter/pyparser/future.py @@ -85,13 +85,17 @@ # permissive parsing of the given list of tokens; it relies on # the real parsing done afterwards to give errors. it.skip_newlines() - it.skip_name("r") or it.skip_name("u") or it.skip_name("ru") - if it.skip(pygram.tokens.STRING): - it.skip_newlines() - while (it.skip_name("from") and + docstring_possible = True + while True: + it.skip_name("r") or it.skip_name("u") or it.skip_name("ru") + if docstring_possible and it.skip(pygram.tokens.STRING): + it.skip_newlines() + docstring_possible = False + if not (it.skip_name("from") and it.skip_name("__future__") and it.skip_name("import")): + break it.skip(pygram.tokens.LPAR) # optionally # return in 'last_position' any line-column pair that points # somewhere inside the last __future__ import statement diff --git a/pypy/interpreter/pyparser/test/test_future.py b/pypy/interpreter/pyparser/test/test_future.py --- a/pypy/interpreter/pyparser/test/test_future.py +++ b/pypy/interpreter/pyparser/test/test_future.py @@ -208,3 +208,13 @@ 'from __future__ import with_statement;') f = run(s, (2, 23)) assert f == fut.CO_FUTURE_DIVISION | fut.CO_FUTURE_WITH_STATEMENT + +def test_future_doc_future(): + # for some reason people do this :-[ + s = ''' +from __future__ import generators +"Docstring" +from __future__ import division + ''' + f = run(s, (4, 24)) + assert f == fut.CO_FUTURE_DIVISION | fut.CO_GENERATOR_ALLOWED diff --git a/pypy/interpreter/test/test_unicodehelper.py b/pypy/interpreter/test/test_unicodehelper.py --- a/pypy/interpreter/test/test_unicodehelper.py +++ b/pypy/interpreter/test/test_unicodehelper.py @@ -1,4 +1,7 @@ -from pypy.interpreter.unicodehelper import encode_utf8, decode_utf8 +import pytest +import struct +from pypy.interpreter.unicodehelper import ( + encode_utf8, decode_utf8, unicode_encode_utf_32_be) class FakeSpace: pass @@ -24,3 +27,23 @@ assert map(ord, got) == [0xd800, 0xdc00] got = decode_utf8(space, "\xf0\x90\x80\x80") assert map(ord, got) == [0x10000] + + at pytest.mark.parametrize('unich', [u"\ud800", u"\udc80"]) +def test_utf32_surrogates(unich): + assert (unicode_encode_utf_32_be(unich, 1, None) == + struct.pack('>i', ord(unich))) + with pytest.raises(UnicodeEncodeError): + unicode_encode_utf_32_be(unich, 1, None, allow_surrogates=False) + + def replace_with(ru, rs): + def errorhandler(errors, enc, msg, u, startingpos, endingpos): + if errors == 'strict': + raise UnicodeEncodeError(enc, u, startingpos, endingpos, msg) + return ru, rs, endingpos + return unicode_encode_utf_32_be( + u"<%s>" % unich, 3, None, + errorhandler, allow_surrogates=False) + + assert replace_with(u'rep', None) == u''.encode('utf-32-be') + assert (replace_with(None, '\xca\xfe\xca\xfe') == + '\x00\x00\x00<\xca\xfe\xca\xfe\x00\x00\x00>') diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -1,7 +1,11 @@ +from rpython.rlib.objectmodel import specialize +from rpython.rlib.rarithmetic import intmask +from rpython.rlib.rstring import StringBuilder, UnicodeBuilder +from rpython.rlib import runicode +from rpython.rlib.runicode import ( + default_unicode_error_encode, default_unicode_error_decode, + MAXUNICODE, BYTEORDER, BYTEORDER2, UNICHR) from pypy.interpreter.error import OperationError -from rpython.rlib.objectmodel import specialize -from rpython.rlib import runicode -from pypy.module._codecs import interp_codecs @specialize.memo() def decode_error_handler(space): @@ -37,6 +41,7 @@ # These functions take and return unwrapped rpython strings and unicodes def decode_unicode_escape(space, string): + from pypy.module._codecs import interp_codecs state = space.fromcache(interp_codecs.CodecState) unicodedata_handler = state.get_unicodedata_handler(space) result, consumed = runicode.str_decode_unicode_escape( @@ -71,3 +76,229 @@ uni, len(uni), "strict", errorhandler=None, allow_surrogates=True) + +# ____________________________________________________________ +# utf-32 + +def str_decode_utf_32(s, size, errors, final=True, + errorhandler=None): + result, length, byteorder = str_decode_utf_32_helper( + s, size, errors, final, errorhandler, "native") + return result, length + +def str_decode_utf_32_be(s, size, errors, final=True, + errorhandler=None): + result, length, byteorder = str_decode_utf_32_helper( + s, size, errors, final, errorhandler, "big") + return result, length + +def str_decode_utf_32_le(s, size, errors, final=True, + errorhandler=None): + result, length, byteorder = str_decode_utf_32_helper( + s, size, errors, final, errorhandler, "little") + return result, length + +def py3k_str_decode_utf_32(s, size, errors, final=True, + errorhandler=None): + result, length, byteorder = str_decode_utf_32_helper( + s, size, errors, final, errorhandler, "native", 'utf-32-' + BYTEORDER2) + return result, length + +def py3k_str_decode_utf_32_be(s, size, errors, final=True, + errorhandler=None): + result, length, byteorder = str_decode_utf_32_helper( + s, size, errors, final, errorhandler, "big", 'utf-32-be') + return result, length + +def py3k_str_decode_utf_32_le(s, size, errors, final=True, + errorhandler=None): + result, length, byteorder = str_decode_utf_32_helper( + s, size, errors, final, errorhandler, "little", 'utf-32-le') + return result, length + +BOM32_DIRECT = intmask(0x0000FEFF) +BOM32_REVERSE = intmask(0xFFFE0000) + +def str_decode_utf_32_helper(s, size, errors, final=True, + errorhandler=None, + byteorder="native", + public_encoding_name='utf32'): + if errorhandler is None: + errorhandler = default_unicode_error_decode + bo = 0 + + if BYTEORDER == 'little': + iorder = [0, 1, 2, 3] + else: + iorder = [3, 2, 1, 0] + + # Check for BOM marks (U+FEFF) in the input and adjust current + # byte order setting accordingly. In native mode, the leading BOM + # mark is skipped, in all other modes, it is copied to the output + # stream as-is (giving a ZWNBSP character). + pos = 0 + if byteorder == 'native': + if size >= 4: + bom = intmask( + (ord(s[iorder[3]]) << 24) | (ord(s[iorder[2]]) << 16) | + (ord(s[iorder[1]]) << 8) | ord(s[iorder[0]])) + if BYTEORDER == 'little': + if bom == BOM32_DIRECT: + pos += 4 + bo = -1 + elif bom == BOM32_REVERSE: + pos += 4 + bo = 1 + else: + if bom == BOM32_DIRECT: + pos += 4 + bo = 1 + elif bom == BOM32_REVERSE: + pos += 4 + bo = -1 + elif byteorder == 'little': + bo = -1 + else: + bo = 1 + if size == 0: + return u'', 0, bo + if bo == -1: + # force little endian + iorder = [0, 1, 2, 3] + elif bo == 1: + # force big endian + iorder = [3, 2, 1, 0] + + result = UnicodeBuilder(size // 4) + + while pos < size: + # remaining bytes at the end? (size should be divisible by 4) + if len(s) - pos < 4: + if not final: + break + r, pos = errorhandler(errors, public_encoding_name, + "truncated data", + s, pos, len(s)) + result.append(r) + if len(s) - pos < 4: + break + continue + ch = ((ord(s[pos + iorder[3]]) << 24) | (ord(s[pos + iorder[2]]) << 16) | + (ord(s[pos + iorder[1]]) << 8) | ord(s[pos + iorder[0]])) + if ch >= 0x110000: + r, pos = errorhandler(errors, public_encoding_name, + "codepoint not in range(0x110000)", + s, pos, len(s)) + result.append(r) + continue + + if MAXUNICODE < 65536 and ch >= 0x10000: + ch -= 0x10000L + result.append(unichr(0xD800 + (ch >> 10))) + result.append(unichr(0xDC00 + (ch & 0x03FF))) + else: + result.append(UNICHR(ch)) + pos += 4 + return result.build(), pos, bo + +def _STORECHAR32(result, CH, byteorder): + c0 = chr(((CH) >> 24) & 0xff) + c1 = chr(((CH) >> 16) & 0xff) + c2 = chr(((CH) >> 8) & 0xff) + c3 = chr((CH) & 0xff) + if byteorder == 'little': + result.append(c3) + result.append(c2) + result.append(c1) + result.append(c0) + else: + result.append(c0) + result.append(c1) + result.append(c2) + result.append(c3) + +def unicode_encode_utf_32_helper(s, size, errors, + errorhandler=None, + allow_surrogates=True, + byteorder='little', + public_encoding_name='utf32'): + if errorhandler is None: + errorhandler = default_unicode_error_encode + if size == 0: + if byteorder == 'native': + result = StringBuilder(4) + _STORECHAR32(result, 0xFEFF, BYTEORDER) + return result.build() + return "" + + result = StringBuilder(size * 4 + 4) + if byteorder == 'native': + _STORECHAR32(result, 0xFEFF, BYTEORDER) + byteorder = BYTEORDER + + pos = 0 + while pos < size: + ch = ord(s[pos]) + pos += 1 + ch2 = 0 + if not allow_surrogates and 0xD800 <= ch < 0xE000: + ru, rs, pos = errorhandler( + errors, public_encoding_name, 'surrogates not allowed', + s, pos - 1, pos) + if rs is not None: + # py3k only + if len(rs) % 4 != 0: + errorhandler( + 'strict', public_encoding_name, 'surrogates not allowed', + s, pos - 1, pos) + result.append(rs) + continue + for ch in ru: + if ord(ch) < 0xD800: + _STORECHAR32(result, ord(ch), byteorder) + else: + errorhandler( + 'strict', public_encoding_name, + 'surrogates not allowed', s, pos - 1, pos) + continue + if 0xD800 <= ch < 0xDC00 and MAXUNICODE < 65536 and pos < size: + ch2 = ord(s[pos]) + if 0xDC00 <= ch2 < 0xE000: + ch = (((ch & 0x3FF) << 10) | (ch2 & 0x3FF)) + 0x10000 + pos += 1 + _STORECHAR32(result, ch, byteorder) + + return result.build() + +def unicode_encode_utf_32(s, size, errors, + errorhandler=None, allow_surrogates=True): + return unicode_encode_utf_32_helper(s, size, errors, errorhandler, + allow_surrogates, "native") + +def unicode_encode_utf_32_be(s, size, errors, + errorhandler=None, allow_surrogates=True): + return unicode_encode_utf_32_helper(s, size, errors, errorhandler, + allow_surrogates, "big") + +def unicode_encode_utf_32_le(s, size, errors, + errorhandler=None, allow_surrogates=True): + return unicode_encode_utf_32_helper(s, size, errors, errorhandler, + allow_surrogates, "little") + +def py3k_unicode_encode_utf_32(s, size, errors, + errorhandler=None, allow_surrogates=True): + return unicode_encode_utf_32_helper(s, size, errors, errorhandler, + allow_surrogates, "native", + 'utf-32-' + BYTEORDER2) + +def py3k_unicode_encode_utf_32_be(s, size, errors, + errorhandler=None, allow_surrogates=True): + return unicode_encode_utf_32_helper(s, size, errors, errorhandler, + allow_surrogates, "big", + 'utf-32-be') + +def py3k_unicode_encode_utf_32_le(s, size, errors, + errorhandler=None, allow_surrogates=True): + return unicode_encode_utf_32_helper(s, size, errors, errorhandler, + allow_surrogates, "little", + 'utf-32-le') diff --git a/pypy/module/__builtin__/operation.py b/pypy/module/__builtin__/operation.py --- a/pypy/module/__builtin__/operation.py +++ b/pypy/module/__builtin__/operation.py @@ -2,11 +2,13 @@ Interp-level implementation of the basic space operations. """ +import math + from pypy.interpreter import gateway from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.gateway import unwrap_spec, WrappedDefault from rpython.rlib.runicode import UNICHR -from rpython.rlib.rfloat import isfinite, isinf, round_double, round_away +from rpython.rlib.rfloat import isfinite, round_double, round_away from rpython.rlib import rfloat import __builtin__ @@ -151,7 +153,7 @@ else: # finite x, and ndigits is not unreasonably large z = round_double(number, ndigits) - if isinf(z): + if math.isinf(z): raise oefmt(space.w_OverflowError, "rounded value too large to represent") return space.newfloat(z) diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py --- a/pypy/module/__builtin__/test/test_builtin.py +++ b/pypy/module/__builtin__/test/test_builtin.py @@ -404,6 +404,7 @@ def test_cmp(self): + assert cmp(float('nan'), float('nan')) == 0 assert cmp(9,9) == 0 assert cmp(0,9) < 0 assert cmp(9,0) > 0 diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -58,6 +58,14 @@ } +class PyPyDateTime(MixedModule): + appleveldefs = {} + interpleveldefs = { + 'dateinterop': 'interp_pypydatetime.W_DateTime_Date', + 'timeinterop' : 'interp_pypydatetime.W_DateTime_Time', + 'deltainterop' : 'interp_pypydatetime.W_DateTime_Delta', + } + class Module(MixedModule): """ PyPy specific "magic" functions. A lot of them are experimental and subject to change, many are internal. """ @@ -108,6 +116,7 @@ "thread": ThreadModule, "intop": IntOpModule, "os": OsModule, + '_pypydatetime': PyPyDateTime, } def setup_after_space_initialization(self): diff --git a/pypy/module/__pypy__/interp_pypydatetime.py b/pypy/module/__pypy__/interp_pypydatetime.py new file mode 100644 --- /dev/null +++ b/pypy/module/__pypy__/interp_pypydatetime.py @@ -0,0 +1,24 @@ +from pypy.interpreter.baseobjspace import W_Root +from pypy.interpreter.typedef import TypeDef +from pypy.interpreter.gateway import interp2app +from rpython.tool.sourcetools import func_with_new_name + +def create_class(name): + class W_Class(W_Root): + 'builtin base clasee for datetime.%s to allow interop with cpyext' % name + def descr_new__(space, w_type): + return space.allocate_instance(W_Class, w_type) + + W_Class.typedef = TypeDef(name, + __new__ = interp2app(func_with_new_name( + W_Class.descr_new__.im_func, + '%s_new' % (name,))), + ) + W_Class.typedef.acceptable_as_base_class = True + return W_Class + +W_DateTime_Time = create_class('pypydatetime_time') +W_DateTime_Date = create_class('pypydatetime_date') +W_DateTime_Delta = create_class('pypydatetime_delta') + + diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -3,7 +3,7 @@ from rpython.rlib import rdynload, clibffi from rpython.rtyper.lltypesystem import rffi -VERSION = "1.11.2" +VERSION = "1.11.4" FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI try: diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1,7 +1,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.11.2", ("This test_c.py file is for testing a version" +assert __version__ == "1.11.4", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py --- a/pypy/module/_cffi_backend/test/test_recompiler.py +++ b/pypy/module/_cffi_backend/test/test_recompiler.py @@ -8,7 +8,8 @@ @unwrap_spec(cdef='text', module_name='text', source='text', packed=int) def prepare(space, cdef, module_name, source, w_includes=None, - w_extra_source=None, w_min_version=None, packed=False): + w_extra_source=None, w_min_version=None, packed=False, + w_extra_compile_args=None): try: import cffi from cffi import FFI # <== the system one, which @@ -55,10 +56,14 @@ sources = [] if w_extra_source is not None: sources.append(space.str_w(w_extra_source)) + kwargs = {} + if w_extra_compile_args is not None: + kwargs['extra_compile_args'] = space.unwrap(w_extra_compile_args) ext = ffiplatform.get_extension(c_file, module_name, include_dirs=[str(rdir)], export_symbols=['_cffi_pypyinit_' + base_module_name], - sources=sources) + sources=sources, + **kwargs) ffiplatform.compile(str(rdir), ext) for extension in ['so', 'pyd', 'dylib']: @@ -2054,3 +2059,51 @@ "Such structs are only supported as return value if the function is " "'API mode' and non-variadic (i.e. declared inside ffibuilder.cdef()" "+ffibuilder.set_source() and not taking a final '...' argument)") + + def test_gcc_visibility_hidden(self): + import sys + if sys.platform == 'win32': + skip("test for gcc/clang") + ffi, lib = self.prepare(""" + int f(int); + """, "test_gcc_visibility_hidden", """ + int f(int a) { return a + 40; } + """, extra_compile_args=['-fvisibility=hidden']) + assert lib.f(2) == 42 + + def test_override_default_definition(self): + ffi, lib = self.prepare(""" + typedef long int16_t, char16_t; + """, "test_override_default_definition", """ + """) + assert ffi.typeof("int16_t") is ffi.typeof("char16_t") is ffi.typeof("long") + + def test_char16_char32_plain_c(self): + ffi, lib = self.prepare(""" + char16_t foo_2bytes(char16_t); + char32_t foo_4bytes(char32_t); + """, "test_char16_char32_type_nocpp", """ + #if !defined(__cplusplus) || (!defined(_LIBCPP_VERSION) && __cplusplus < 201103L) + typedef uint_least16_t char16_t; + typedef uint_least32_t char32_t; + #endif + + char16_t foo_2bytes(char16_t a) { return (char16_t)(a + 42); } + char32_t foo_4bytes(char32_t a) { return (char32_t)(a + 42); } + """, min_version=(1, 11, 0)) + assert lib.foo_2bytes(u'\u1234') == u'\u125e' + assert lib.foo_4bytes(u'\u1234') == u'\u125e' + assert lib.foo_4bytes(u'\U00012345') == u'\U0001236f' + raises(TypeError, lib.foo_2bytes, u'\U00012345') + raises(TypeError, lib.foo_2bytes, 1234) + raises(TypeError, lib.foo_4bytes, 1234) + + def test_loader_spec(self): + import sys + ffi, lib = self.prepare("", "test_loader_spec", "") + if sys.version_info < (3,): + assert not hasattr(lib, '__loader__') + assert not hasattr(lib, '__spec__') + else: + assert lib.__loader__ is None + assert lib.__spec__ is None diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -1,10 +1,12 @@ from rpython.rlib import jit from rpython.rlib.objectmodel import we_are_translated, not_rpython from rpython.rlib.rstring import UnicodeBuilder +from rpython.rlib import runicode from rpython.rlib.runicode import code_to_unichr, MAXUNICODE from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault +from pypy.interpreter import unicodehelper class VersionTag(object): @@ -210,7 +212,8 @@ def xmlcharrefreplace_errors(space, w_exc): check_exception(space, w_exc) if space.isinstance_w(w_exc, space.w_UnicodeEncodeError): - obj = space.realunicode_w(space.getattr(w_exc, space.newtext('object'))) + w_obj = space.getattr(w_exc, space.newtext('object')) + obj = space.realunicode_w(w_obj) start = space.int_w(space.getattr(w_exc, space.newtext('start'))) w_end = space.getattr(w_exc, space.newtext('end')) end = space.int_w(w_end) @@ -236,7 +239,8 @@ def backslashreplace_errors(space, w_exc): check_exception(space, w_exc) if space.isinstance_w(w_exc, space.w_UnicodeEncodeError): - obj = space.realunicode_w(space.getattr(w_exc, space.newtext('object'))) + w_obj = space.getattr(w_exc, space.newtext('object')) + obj = space.realunicode_w(w_obj) start = space.int_w(space.getattr(w_exc, space.newtext('start'))) w_end = space.getattr(w_exc, space.newtext('end')) end = space.int_w(w_end) @@ -363,19 +367,23 @@ raise oefmt(space.w_TypeError, "handler must be callable") # ____________________________________________________________ -# delegation to runicode +# delegation to runicode/unicodehelper -from rpython.rlib import runicode +def _find_implementation(impl_name): + try: + func = getattr(unicodehelper, impl_name) + except AttributeError: + func = getattr(runicode, impl_name) + return func def make_encoder_wrapper(name): rname = "unicode_encode_%s" % (name.replace("_encode", ""), ) - assert hasattr(runicode, rname) + func = _find_implementation(rname) @unwrap_spec(uni=unicode, errors='text_or_none') def wrap_encoder(space, uni, errors="strict"): if errors is None: errors = 'strict' state = space.fromcache(CodecState) - func = getattr(runicode, rname) result = func(uni, len(uni), errors, state.encode_error_handler) return space.newtuple([space.newbytes(result), space.newint(len(uni))]) wrap_encoder.func_name = rname @@ -383,7 +391,7 @@ def make_decoder_wrapper(name): rname = "str_decode_%s" % (name.replace("_decode", ""), ) - assert hasattr(runicode, rname) + func = _find_implementation(rname) @unwrap_spec(string='bufferstr', errors='text_or_none', w_final=WrappedDefault(False)) def wrap_decoder(space, string, errors="strict", w_final=None): @@ -391,7 +399,6 @@ errors = 'strict' final = space.is_true(w_final) state = space.fromcache(CodecState) - func = getattr(runicode, rname) result, consumed = func(string, len(string), errors, final, state.decode_error_handler) return space.newtuple([space.newunicode(result), space.newint(consumed)]) diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -115,10 +115,10 @@ From pypy.commits at gmail.com Fri Apr 20 20:33:16 2018 From: pypy.commits at gmail.com (wlav) Date: Fri, 20 Apr 2018 17:33:16 -0700 (PDT) Subject: [pypy-commit] pypy cppyy-packaging: upgrade to backend 0.6.0 Message-ID: <5ada86cc.1c69fb81.78f2.7fa6@mx.google.com> Author: Wim Lavrijsen Branch: cppyy-packaging Changeset: r94404:e98cf77f5b72 Date: 2018-04-20 16:00 -0700 http://bitbucket.org/pypy/pypy/changeset/e98cf77f5b72/ Log: upgrade to backend 0.6.0 diff --git a/pypy/module/_cppyy/capi/loadable_capi.py b/pypy/module/_cppyy/capi/loadable_capi.py --- a/pypy/module/_cppyy/capi/loadable_capi.py +++ b/pypy/module/_cppyy/capi/loadable_capi.py @@ -121,11 +121,11 @@ # TODO: the following need to match up with the globally defined C_XYZ low-level # types (see capi/__init__.py), but by using strings here, that isn't guaranteed - c_opaque_ptr = state.c_ulong + c_opaque_ptr = state.c_ulong # not ptrdiff_t (which is signed) c_scope = c_opaque_ptr c_type = c_scope - c_object = c_opaque_ptr + c_object = c_opaque_ptr # not voidp (to stick with one handle type) c_method = c_opaque_ptr c_index = state.c_long c_index_array = state.c_voidp @@ -150,16 +150,17 @@ self.capi_call_ifaces = { # name to opaque C++ scope representation - 'num_scopes' : ([c_scope], c_int), - 'scope_name' : ([c_scope, c_int], c_ccharp), - 'resolve_name' : ([c_ccharp], c_ccharp), + 'resolve_enum' : ([c_ccharp], c_ccharp), 'get_scope' : ([c_ccharp], c_scope), 'actual_class' : ([c_type, c_object], c_type), + 'size_of_klass' : ([c_type], c_size_t), + 'size_of_type' : ([c_ccharp], c_size_t), # memory management 'allocate' : ([c_type], c_object), 'deallocate' : ([c_type, c_object], c_void), + 'construct' : ([c_type], c_object), 'destruct' : ([c_type, c_object], c_void), # method/function dispatching @@ -182,7 +183,8 @@ 'constructor' : ([c_method, c_object, c_int, c_voidp], c_object), 'call_o' : ([c_method, c_object, c_int, c_voidp, c_type], c_object), - 'get_function_address' : ([c_scope, c_index], c_voidp), # TODO: verify + 'function_address_from_index' : ([c_scope, c_index], c_voidp), # TODO: verify + 'function_address_from_method' : ([c_method], c_voidp), # id. # handling of function argument buffer 'allocate_function_args' : ([c_int], c_voidp), @@ -196,6 +198,8 @@ 'is_abstract' : ([c_type], c_int), 'is_enum' : ([c_ccharp], c_int), + 'get_all_cpp_names' : ([c_scope, c_voidp], c_voidp), # const char** + # type/class reflection information 'final_name' : ([c_type], c_ccharp), 'scoped_final_name' : ([c_type], c_ccharp), @@ -208,10 +212,10 @@ # method/function reflection information 'num_methods' : ([c_scope], c_int), - 'method_index_at' : ([c_scope, c_int], c_index), 'method_indices_from_name' : ([c_scope, c_ccharp], c_index_array), 'method_name' : ([c_scope, c_index], c_ccharp), + 'method_mangled_name' : ([c_scope, c_index], c_ccharp), 'method_result_type' : ([c_scope, c_index], c_ccharp), 'method_num_args' : ([c_scope, c_index], c_int), 'method_req_args' : ([c_scope, c_index], c_int), @@ -219,7 +223,9 @@ 'method_arg_default' : ([c_scope, c_index, c_int], c_ccharp), 'method_signature' : ([c_scope, c_index, c_int], c_ccharp), 'method_prototype' : ([c_scope, c_index, c_int], c_ccharp), + 'is_const_method' : ([c_method], c_int), + 'exists_method_template' : ([c_scope, c_ccharp], c_int), 'method_is_template' : ([c_scope, c_index], c_int), 'method_num_template_args' : ([c_scope, c_index], c_int), 'method_template_arg_name' : ([c_scope, c_index, c_index], c_ccharp), @@ -228,7 +234,9 @@ 'get_global_operator' : ([c_scope, c_scope, c_scope, c_ccharp], c_index), # method properties + 'is_public_method' : ([c_type, c_index], c_int), 'is_constructor' : ([c_type, c_index], c_int), + 'is_destructor' : ([c_type, c_index], c_int), 'is_staticmethod' : ([c_type, c_index], c_int), # data member reflection information @@ -236,12 +244,14 @@ 'datamember_name' : ([c_scope, c_int], c_ccharp), 'datamember_type' : ([c_scope, c_int], c_ccharp), 'datamember_offset' : ([c_scope, c_int], c_ptrdiff_t), - 'datamember_index' : ([c_scope, c_ccharp], c_int), # data member properties 'is_publicdata' : ([c_scope, c_int], c_int), 'is_staticdata' : ([c_scope, c_int], c_int), + 'is_const_data' : ([c_scope, c_int], c_int), + 'is_enum_data' : ([c_scope, c_int], c_int), + 'get_dimension_size' : ([c_scope, c_int, c_int], c_int), # misc helpers 'strtoll' : ([c_ccharp], c_llong), @@ -328,25 +338,27 @@ return rffi.cast(rffi.CCHARP, ptr) # name to opaque C++ scope representation ------------------------------------ -def c_num_scopes(space, cppscope): - return space.int_w(call_capi(space, 'num_scopes', [_ArgH(cppscope.handle)])) -def c_scope_name(space, cppscope, iscope): - args = [_ArgH(cppscope.handle), _ArgL(iscope)] - return charp2str_free(space, call_capi(space, 'scope_name', args)) - def c_resolve_name(space, name): return charp2str_free(space, call_capi(space, 'resolve_name', [_ArgS(name)])) +def c_resolve_enum(space, name): + return charp2str_free(space, call_capi(space, 'resolve_enum', [_ArgS(name)])) def c_get_scope_opaque(space, name): return rffi.cast(C_SCOPE, space.uint_w(call_capi(space, 'get_scope', [_ArgS(name)]))) def c_actual_class(space, cppclass, cppobj): args = [_ArgH(cppclass.handle), _ArgH(cppobj)] return rffi.cast(C_TYPE, space.uint_w(call_capi(space, 'actual_class', args))) +def c_size_of_klass(space, cppclass): + return _cdata_to_size_t(space, call_capi(space, 'size_of_klass', [_ArgH(cppclass.handle)])) +def c_size_of_type(space, name): + return _cdata_to_size_t(space, call_capi(space, 'size_of_type', [_ArgS(name)])) # memory management ---------------------------------------------------------- def c_allocate(space, cppclass): return _cdata_to_cobject(space, call_capi(space, 'allocate', [_ArgH(cppclass.handle)])) def c_deallocate(space, cppclass, cppobject): call_capi(space, 'deallocate', [_ArgH(cppclass.handle), _ArgH(cppobject)]) +def c_construct(space, cppclass): + return _cdata_to_cobject(space, call_capi(space, 'construct', [_ArgH(cppclass.handle)])) def c_destruct(space, cppclass, cppobject): call_capi(space, 'destruct', [_ArgH(cppclass.handle), _ArgH(cppobject)]) @@ -391,7 +403,7 @@ w_cstr = call_capi(space, 'call_s', [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs), _ArgP(rffi.cast(rffi.VOIDP, length))]) - cstr_len = intmask(length[0]) + cstr_len = int(intmask(length[0])) finally: lltype.free(length, flavor='raw') return _cdata_to_ccharp(space, w_cstr), cstr_len @@ -403,10 +415,13 @@ args = [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs), _ArgH(cppclass.handle)] return _cdata_to_cobject(space, call_capi(space, 'call_o', args)) -def c_get_function_address(space, cppscope, index): +def c_function_address_from_index(space, cppscope, index): args = [_ArgH(cppscope.handle), _ArgL(index)] return rffi.cast(C_FUNC_PTR, - _cdata_to_ptr(space, call_capi(space, 'get_function_address', args))) + _cdata_to_ptr(space, call_capi(space, 'function_address_from_index', args))) +def c_function_address_from_method(space, cppmethod): + return rffi.cast(C_FUNC_PTR, + _cdata_to_ptr(space, call_capi(space, 'function_address_from_method', _ArgH(cppmethod)))) # handling of function argument buffer --------------------------------------- def c_allocate_function_args(space, size): @@ -430,6 +445,23 @@ def c_is_enum(space, name): return space.bool_w(call_capi(space, 'is_enum', [_ArgS(name)])) +def c_get_all_cpp_names(space, scope): + sz = lltype.malloc(rffi.SIZE_TP.TO, 1, flavor='raw', zero=True) + try: + args = [_ArgH(scope.handle), _ArgP(rffi.cast(rffi.VOIDP, sz))] + rawnames = rffi.cast(rffi.CCHARPP, + _cdata_to_ptr(space, call_capi(space, 'get_all_cpp_names', args))) + count = int(intmask(sz[0])) + finally: + lltype.free(sz, flavor='raw') + allnames = [] + for i in range(count): + pystr = rffi.charp2str(rawnames[i]) + c_free(space, rffi.cast(rffi.VOIDP, rawnames[i])) # c_free defined below + allnames.append(pystr) + c_free(space, rffi.cast(rffi.VOIDP, rawnames)) # id. + return allnames + # type/class reflection information ------------------------------------------ def c_final_name(space, cpptype): return charp2str_free(space, call_capi(space, 'final_name', [_ArgH(cpptype)])) @@ -462,13 +494,10 @@ def c_num_methods(space, cppscope): args = [_ArgH(cppscope.handle)] return space.int_w(call_capi(space, 'num_methods', args)) -def c_method_index_at(space, cppscope, imethod): - args = [_ArgH(cppscope.handle), _ArgL(imethod)] - return space.int_w(call_capi(space, 'method_index_at', args)) def c_method_indices_from_name(space, cppscope, name): args = [_ArgH(cppscope.handle), _ArgS(name)] indices = rffi.cast(C_INDEX_ARRAY, - _cdata_to_ptr(space, call_capi(space, 'method_indices_from_name', args))) + _cdata_to_ptr(space, call_capi(space, 'method_indices_from_name', args))) if not indices: return [] py_indices = [] @@ -506,6 +535,9 @@ args = [_ArgH(cppscope.handle), _ArgL(index), _ArgL(show_formalargs)] return charp2str_free(space, call_capi(space, 'method_prototype', args)) +def c_exists_method_template(space, cppscope, name): + args = [_ArgH(cppscope.handle), _ArgS(name)] + return space.bool_w(call_capi(space, 'exists_method_template', args)) def c_method_is_template(space, cppscope, index): args = [_ArgH(cppscope.handle), _ArgL(index)] return space.bool_w(call_capi(space, 'method_is_template', args)) @@ -531,9 +563,15 @@ return rffi.cast(WLAVC_INDEX, -1) # method properties ---------------------------------------------------------- +def c_is_public_method(space, cppclass, index): + args = [_ArgH(cppclass.handle), _ArgL(index)] + return space.bool_w(call_capi(space, 'is_public_method', args)) def c_is_constructor(space, cppclass, index): args = [_ArgH(cppclass.handle), _ArgL(index)] return space.bool_w(call_capi(space, 'is_constructor', args)) +def c_is_destructor(space, cppclass, index): + args = [_ArgH(cppclass.handle), _ArgL(index)] + return space.bool_w(call_capi(space, 'is_destructor', args)) def c_is_staticmethod(space, cppclass, index): args = [_ArgH(cppclass.handle), _ArgL(index)] return space.bool_w(call_capi(space, 'is_staticmethod', args)) @@ -562,6 +600,15 @@ def c_is_staticdata(space, cppscope, datamember_index): args = [_ArgH(cppscope.handle), _ArgL(datamember_index)] return space.bool_w(call_capi(space, 'is_staticdata', args)) +def c_is_const_data(space, cppscope, datamember_index): + args = [_ArgH(cppscope.handle), _ArgL(datamember_index)] + return space.bool_w(call_capi(space, 'is_const_data', args)) +def c_is_enum_data(space, cppscope, datamember_index): + args = [_ArgH(cppscope.handle), _ArgL(datamember_index)] + return space.bool_w(call_capi(space, 'is_enum_data', args)) +def c_get_dimension_size(space, cppscope, datamember_index, dim_idx): + args = [_ArgH(cppscope.handle), _ArgL(datamember_index), _ArgL(dim_idx)] + return space.bool_w(call_capi(space, 'get_dimension_size', args)) # misc helpers --------------------------------------------------------------- def c_strtoll(space, svalue): @@ -585,7 +632,7 @@ try: w_cstr = call_capi(space, 'stdstring2charp', [_ArgH(cppstr), _ArgP(rffi.cast(rffi.VOIDP, sz))]) - cstr_len = intmask(sz[0]) + cstr_len = int(intmask(sz[0])) finally: lltype.free(sz, flavor='raw') return rffi.charpsize2str(_cdata_to_ccharp(space, w_cstr), cstr_len) diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py --- a/pypy/module/_cppyy/converter.py +++ b/pypy/module/_cppyy/converter.py @@ -749,8 +749,6 @@ return InstancePtrPtrConverter(space, clsdecl) elif compound == "": return InstanceConverter(space, clsdecl) - elif capi.c_is_enum(space, clean_name): - return _converters['unsigned'](space, default) # 5) void* or void converter (which fails on use) if 0 <= compound.find('*'): diff --git a/pypy/module/_cppyy/executor.py b/pypy/module/_cppyy/executor.py --- a/pypy/module/_cppyy/executor.py +++ b/pypy/module/_cppyy/executor.py @@ -289,8 +289,6 @@ return InstancePtrExecutor(space, cppclass) elif compound == '**' or compound == '*&': return InstancePtrPtrExecutor(space, cppclass) - elif capi.c_is_enum(space, clean_name): - return _executors['internal_enum_type_t'](space, None) # 4) additional special cases if compound == '*': diff --git a/pypy/module/_cppyy/include/capi.h b/pypy/module/_cppyy/include/capi.h --- a/pypy/module/_cppyy/include/capi.h +++ b/pypy/module/_cppyy/include/capi.h @@ -8,26 +8,27 @@ extern "C" { #endif // ifdef __cplusplus - typedef unsigned long cppyy_scope_t; + typedef ptrdiff_t cppyy_scope_t; typedef cppyy_scope_t cppyy_type_t; - typedef unsigned long cppyy_object_t; - typedef unsigned long cppyy_method_t; + typedef void* cppyy_object_t; + typedef ptrdiff_t cppyy_method_t; + typedef long cppyy_index_t; typedef void* cppyy_funcaddr_t; /* name to opaque C++ scope representation -------------------------------- */ RPY_EXTERN - int cppyy_num_scopes(cppyy_scope_t parent); + char* cppyy_resolve_name(const char* cppitem_name); RPY_EXTERN - char* cppyy_scope_name(cppyy_scope_t parent, cppyy_index_t iscope); - RPY_EXTERN - char* cppyy_resolve_name(const char* cppitem_name); + char* cppyy_resolve_enum(const char* enum_type); RPY_EXTERN cppyy_scope_t cppyy_get_scope(const char* scope_name); RPY_EXTERN cppyy_type_t cppyy_actual_class(cppyy_type_t klass, cppyy_object_t obj); RPY_EXTERN - size_t cppyy_size_of(cppyy_type_t klass); + size_t cppyy_size_of_klass(cppyy_type_t klass); + RPY_EXTERN + size_t cppyy_size_of_type(const char* type_name); /* memory management ------------------------------------------------------ */ RPY_EXTERN @@ -35,48 +36,53 @@ RPY_EXTERN void cppyy_deallocate(cppyy_type_t type, cppyy_object_t self); RPY_EXTERN + cppyy_object_t cppyy_construct(cppyy_type_t type); + RPY_EXTERN void cppyy_destruct(cppyy_type_t type, cppyy_object_t self); /* method/function dispatching -------------------------------------------- */ RPY_EXTERN - void cppyy_call_v(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); + void cppyy_call_v(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN unsigned char cppyy_call_b(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN - char cppyy_call_c(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); + char cppyy_call_c(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN - short cppyy_call_h(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); + short cppyy_call_h(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN - int cppyy_call_i(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); + int cppyy_call_i(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN - long cppyy_call_l(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); + long cppyy_call_l(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN long long cppyy_call_ll(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN - float cppyy_call_f(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); + float cppyy_call_f(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN double cppyy_call_d(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN long double cppyy_call_ld(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN - void* cppyy_call_r(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); + void* cppyy_call_r(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN - char* cppyy_call_s(cppyy_method_t method, cppyy_object_t self, int nargs, void* args, size_t* length); - + char* cppyy_call_s(cppyy_method_t method, cppyy_object_t self, int nargs, void* args, size_t* length); RPY_EXTERN cppyy_object_t cppyy_constructor(cppyy_method_t method, cppyy_type_t klass, int nargs, void* args); RPY_EXTERN + void cppyy_destructor(cppyy_type_t type, cppyy_object_t self); + RPY_EXTERN cppyy_object_t cppyy_call_o(cppyy_method_t method, cppyy_object_t self, int nargs, void* args, cppyy_type_t result_type); RPY_EXTERN - cppyy_funcaddr_t cppyy_get_function_address(cppyy_scope_t scope, cppyy_index_t idx); + cppyy_funcaddr_t cppyy_function_address_from_index(cppyy_scope_t scope, cppyy_index_t idx); + RPY_EXTERN + cppyy_funcaddr_t cppyy_function_address_from_method(cppyy_method_t method); /* handling of function argument buffer ----------------------------------- */ RPY_EXTERN - void* cppyy_allocate_function_args(int nargs); + void* cppyy_allocate_function_args(int nargs); RPY_EXTERN - void cppyy_deallocate_function_args(void* args); + void cppyy_deallocate_function_args(void* args); RPY_EXTERN size_t cppyy_function_arg_sizeof(); RPY_EXTERN @@ -92,6 +98,9 @@ RPY_EXTERN int cppyy_is_enum(const char* type_name); + RPY_EXTERN + const char** cppyy_get_all_cpp_names(cppyy_scope_t scope, size_t* count); + /* class reflection information ------------------------------------------- */ RPY_EXTERN char* cppyy_final_name(cppyy_type_t type); @@ -105,6 +114,10 @@ char* cppyy_base_name(cppyy_type_t type, int base_index); RPY_EXTERN int cppyy_is_subtype(cppyy_type_t derived, cppyy_type_t base); + RPY_EXTERN + int cppyy_smartptr_info(const char* name, cppyy_type_t* raw, cppyy_method_t* deref); + RPY_EXTERN + void cppyy_add_smartptr_type(const char* type_name); /* calculate offsets between declared and actual type, up-cast: direction > 0; down-cast: direction < 0 */ RPY_EXTERN @@ -114,8 +127,6 @@ RPY_EXTERN int cppyy_num_methods(cppyy_scope_t scope); RPY_EXTERN - cppyy_index_t cppyy_method_index_at(cppyy_scope_t scope, int imeth); - RPY_EXTERN cppyy_index_t* cppyy_method_indices_from_name(cppyy_scope_t scope, const char* name); RPY_EXTERN @@ -136,8 +147,12 @@ char* cppyy_method_signature(cppyy_scope_t scope, cppyy_index_t idx, int show_formalargs); RPY_EXTERN char* cppyy_method_prototype(cppyy_scope_t scope, cppyy_index_t idx, int show_formalargs); + RPY_EXTERN + int cppyy_is_const_method(cppyy_method_t); RPY_EXTERN + int cppyy_exists_method_template(cppyy_scope_t scope, const char* name); + RPY_EXTERN int cppyy_method_is_template(cppyy_scope_t scope, cppyy_index_t idx); RPY_EXTERN int cppyy_method_num_template_args(cppyy_scope_t scope, cppyy_index_t idx); @@ -169,15 +184,20 @@ char* cppyy_datamember_type(cppyy_scope_t scope, int datamember_index); RPY_EXTERN ptrdiff_t cppyy_datamember_offset(cppyy_scope_t scope, int datamember_index); - RPY_EXTERN int cppyy_datamember_index(cppyy_scope_t scope, const char* name); /* data member properties ------------------------------------------------- */ RPY_EXTERN - int cppyy_is_publicdata(cppyy_type_t type, int datamember_index); + int cppyy_is_publicdata(cppyy_type_t type, cppyy_index_t datamember_index); RPY_EXTERN - int cppyy_is_staticdata(cppyy_type_t type, int datamember_index); + int cppyy_is_staticdata(cppyy_type_t type, cppyy_index_t datamember_index); + RPY_EXTERN + int cppyy_is_const_data(cppyy_scope_t scope, cppyy_index_t idata); + RPY_EXTERN + int cppyy_is_enum_data(cppyy_scope_t scope, cppyy_index_t idata); + RPY_EXTERN + int cppyy_get_dimension_size(cppyy_scope_t scope, cppyy_index_t idata, int dimension); /* misc helpers ----------------------------------------------------------- */ RPY_EXTERN @@ -197,7 +217,7 @@ RPY_EXTERN const char* cppyy_stdvector_valuetype(const char* clname); RPY_EXTERN - size_t cppyy_stdvector_valuesize(const char* clname); + size_t cppyy_stdvector_valuesize(const char* clname); #ifdef __cplusplus } diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py --- a/pypy/module/_cppyy/interp_cppyy.py +++ b/pypy/module/_cppyy/interp_cppyy.py @@ -322,7 +322,7 @@ # Each CPPMethod corresponds one-to-one to a C++ equivalent and cppthis # has been offset to the matching class. Hence, the libffi pointer is # uniquely defined and needs to be setup only once. - funcaddr = capi.c_get_function_address(self.space, self.scope, self.index) + funcaddr = capi.c_function_address_from_index(self.space, self.scope, self.index) if funcaddr and cppthis: # methods only for now state = self.space.fromcache(ffitypes.State) @@ -845,24 +845,11 @@ return self.space.w_True def ns__dir__(self): - # Collect a list of everything (currently) available in the namespace. - # The backend can filter by returning empty strings. Special care is - # taken for functions, which need not be unique (overloading). - alldir = [] - for i in range(capi.c_num_scopes(self.space, self)): - sname = capi.c_scope_name(self.space, self, i) - if sname: alldir.append(self.space.newtext(sname)) - allmeth = {} - for i in range(capi.c_num_methods(self.space, self)): - idx = capi.c_method_index_at(self.space, self, i) - mname = capi.c_method_name(self.space, self, idx) - if mname: allmeth.setdefault(mname, 0) - for m in allmeth.keys(): - alldir.append(self.space.newtext(m)) - for i in range(capi.c_num_datamembers(self.space, self)): - dname = capi.c_datamember_name(self.space, self, i) - if dname: alldir.append(self.space.newtext(dname)) - return self.space.newlist(alldir) + alldir = capi.c_get_all_cpp_names(self.space, self) + w_alldir = self.space.newlist([]) + for name in alldir: + w_alldir.append(self.space.wrap(name)) + return w_alldir def missing_attribute_error(self, name): return oefmt(self.space.w_AttributeError, @@ -890,8 +877,7 @@ def _build_methods(self): assert len(self.methods) == 0 methods_temp = {} - for i in range(capi.c_num_methods(self.space, self)): - idx = capi.c_method_index_at(self.space, self, i) + for idx in range(capi.c_num_methods(self.space, self)): if capi.c_is_constructor(self.space, self, idx): pyname = '__init__' else: diff --git a/pypy/module/_cppyy/pythonify.py b/pypy/module/_cppyy/pythonify.py --- a/pypy/module/_cppyy/pythonify.py +++ b/pypy/module/_cppyy/pythonify.py @@ -438,9 +438,8 @@ gbl.std.move = _cppyy.move # install a type for enums to refer to - # TODO: this is correct for C++98, not for C++11 and in general there will - # be the same issue for all typedef'd builtin types setattr(gbl, 'internal_enum_type_t', int) + setattr(gbl, 'unsigned int', int) # if resolved # install for user access _cppyy.gbl = gbl diff --git a/pypy/module/_cppyy/src/dummy_backend.cxx b/pypy/module/_cppyy/src/dummy_backend.cxx --- a/pypy/module/_cppyy/src/dummy_backend.cxx +++ b/pypy/module/_cppyy/src/dummy_backend.cxx @@ -407,10 +407,6 @@ /* name to opaque C++ scope representation -------------------------------- */ -int cppyy_num_scopes(cppyy_scope_t handle) { - return 0; -} - char* cppyy_resolve_name(const char* cppitem_name) { return cppstring_to_cstring(cppitem_name); } @@ -851,10 +847,13 @@ return (cppyy_object_t)result; } -cppyy_funcaddr_t cppyy_get_function_address(cppyy_scope_t /* scope */, cppyy_index_t /* idx */) { +cppyy_funcaddr_t cppyy_function_address_from_index(cppyy_scope_t /* scope */, cppyy_index_t /* idx */) { return (cppyy_funcaddr_t)0; } +cppyy_funcaddr_t cppyy_function_address_from_method(cppyy_method_t /* method */) { + return (cppyy_funcaddr_t)0; +} /* handling of function argument buffer ----------------------------------- */ void* cppyy_allocate_function_args(int nargs) { @@ -926,10 +925,6 @@ return s_scopes[handle].m_methods.size(); } -cppyy_index_t cppyy_method_index_at(cppyy_scope_t /* scope */, int imeth) { - return (cppyy_index_t)imeth; -} - char* cppyy_method_name(cppyy_scope_t handle, cppyy_index_t method_index) { return cppstring_to_cstring(s_scopes[handle].m_methods[(int)method_index].m_name); } @@ -978,8 +973,17 @@ return (cppyy_method_t)0; } +cppyy_index_t cppyy_get_global_operator(cppyy_scope_t /* scope */, + cppyy_scope_t /* lc */, cppyy_scope_t /* rc */, const char* /* op */) { + return (cppyy_index_t)-1; +} + /* method properties ----------------------------------------------------- */ +int cppyy_is_publicmethod(cppyy_type_t /* handle */, cppyy_index_t /* method_index */) { + return 1; +} + int cppyy_is_constructor(cppyy_type_t handle, cppyy_index_t method_index) { if (s_scopes.find(handle) != s_scopes.end()) return s_scopes[handle].m_methods[method_index].m_type == kConstructor; @@ -987,6 +991,10 @@ return 0; } +int cppyy_is_destructor(cppyy_type_t /* handle */, cppyy_index_t /* method_index */) { + return 0; +} + int cppyy_is_staticmethod(cppyy_type_t handle, cppyy_index_t method_index) { if (s_scopes.find(handle) != s_scopes.end()) return s_scopes[handle].m_methods[method_index].m_type == kStatic; @@ -1014,14 +1022,22 @@ /* data member properties ------------------------------------------------ */ -int cppyy_is_publicdata(cppyy_scope_t handle, int idatambr) { +int cppyy_is_publicdata(cppyy_scope_t /* handle */, cppyy_index_t /* idatambr */) { return 1; } -int cppyy_is_staticdata(cppyy_scope_t handle, int idatambr) { +int cppyy_is_staticdata(cppyy_scope_t handle, cppyy_index_t idatambr) { return s_scopes[handle].m_datambrs[idatambr].m_isstatic; } +int cppyy_is_const_data(cppyy_scope_t /* handle */, cppyy_index_t /* idatambr */) { + return 0; +} + +int cppyy_is_enum_data(cppyy_scope_t /* handle */, cppyy_index_t /* idatambr */) { + return 0; +} + /* misc helpers ----------------------------------------------------------- */ #if defined(_MSC_VER) From pypy.commits at gmail.com Fri Apr 20 20:33:14 2018 From: pypy.commits at gmail.com (wlav) Date: Fri, 20 Apr 2018 17:33:14 -0700 (PDT) Subject: [pypy-commit] pypy cppyy-packaging: merge default into branch Message-ID: <5ada86ca.87981c0a.fc177.d115@mx.google.com> Author: Wim Lavrijsen Branch: cppyy-packaging Changeset: r94403:60b72b97202f Date: 2018-04-19 10:47 -0700 http://bitbucket.org/pypy/pypy/changeset/60b72b97202f/ Log: merge default into branch diff too long, truncating to 2000 out of 19728 lines diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -6,36 +6,36 @@ Except when otherwise stated (look for LICENSE files in directories or information at the beginning of each file) all software and documentation in the 'rpython', 'pypy', 'ctype_configure', 'dotviewer', 'demo', 'lib_pypy', -'py', and '_pytest' directories is licensed as follows: +'py', and '_pytest' directories is licensed as follows: The MIT License - Permission is hereby granted, free of charge, to any person - obtaining a copy of this software and associated documentation - files (the "Software"), to deal in the Software without - restriction, including without limitation the rights to use, - copy, modify, merge, publish, distribute, sublicense, and/or - sell copies of the Software, and to permit persons to whom the + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation + files (the "Software"), to deal in the Software without + restriction, including without limitation the rights to use, + copy, modify, merge, publish, distribute, sublicense, and/or + sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included + The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. PyPy Copyright holders 2003-2018 ------------------------------------ +-------------------------------- Except when otherwise stated (look for LICENSE files or information at the beginning of each file) the files in the 'pypy' directory are each -copyrighted by one or more of the following people and organizations: +copyrighted by one or more of the following people and organizations: Armin Rigo Maciej Fijalkowski @@ -89,13 +89,13 @@ Niko Matsakis Alexander Hesse Ludovic Aubry + stian Jacob Hallen Jason Creighton Mark Young Alex Martelli Spenser Bauman Michal Bendowski - stian Jan de Mooij Tyler Wade Vincent Legoll @@ -123,10 +123,10 @@ Wenzhu Man Konstantin Lopuhin John Witulski + Jeremy Thurgood Greg Price Ivan Sichmann Freitas Dario Bertini - Jeremy Thurgood Mark Pearse Simon Cross Tobias Pape @@ -145,18 +145,19 @@ Adrian Kuhn tav Georg Brandl + Joannah Nanjekye Bert Freudenberg Stian Andreassen Wanja Saatkamp Mike Blume - Joannah Nanjekye Gerald Klix Oscar Nierstrasz Rami Chowdhury Stefan H. Muller + Dodan Mihai Tim Felgentreff Eugene Oden - Dodan Mihai + Colin Valliant Jeff Terrace Henry Mason Vasily Kuznetsov @@ -225,12 +226,14 @@ Vaibhav Sood Reuben Cummings Attila Gobi + Floris Bruynooghe Christopher Pope Tristan Arthur Christian Tismer Dan Stromberg Carl Meyer Florin Papa + Arianna Avanzini Jens-Uwe Mager Valentina Mukhamedzhanova Stefano Parmesan @@ -250,9 +253,11 @@ Alejandro J. Cura Vladimir Kryachko Gabriel + Thomas Hisch Mark Williams Kunal Grover Nathan Taylor + Barry Hart Travis Francis Athougies Yasir Suhail Sergey Kishchenko @@ -260,6 +265,7 @@ Lutz Paelike Ian Foote Philipp Rustemeuer + Logan Chien Catalin Gabriel Manciu Jacob Oscarson Ryan Gonzalez @@ -295,7 +301,6 @@ Akira Li Gustavo Niemeyer Rafał Gałczyński - Logan Chien Lucas Stadler roberto at goyle Matt Bogosian @@ -308,6 +313,7 @@ Anna Katrina Dominguez Kim Jin Su Amber Brown + Miro Hrončok Anthony Sottile Nate Bragg Ben Darnell @@ -315,7 +321,6 @@ Godefroid Chappelle Julian Berman Michael Hudson-Doyle - Floris Bruynooghe Stephan Busemann Dan Colish timo @@ -357,6 +362,7 @@ Michael Chermside Anna Ravencroft remarkablerocket + Pauli Virtanen Petre Vijiac Berker Peksag Christian Muirhead @@ -381,6 +387,7 @@ Graham Markall Dan Loewenherz werat + Andrew Stepanov Niclas Olofsson Chris Pressey Tobias Diaz @@ -395,14 +402,14 @@ m at funkyhat.org Stefan Marr - Heinrich-Heine University, Germany + Heinrich-Heine University, Germany Open End AB (formerly AB Strakt), Sweden - merlinux GmbH, Germany - tismerysoft GmbH, Germany - Logilab Paris, France - DFKI GmbH, Germany + merlinux GmbH, Germany + tismerysoft GmbH, Germany + Logilab Paris, France + DFKI GmbH, Germany Impara, Germany - Change Maker, Sweden + Change Maker, Sweden University of California Berkeley, USA Google Inc. King's College London @@ -410,14 +417,14 @@ The PyPy Logo as used by http://speed.pypy.org and others was created by Samuel Reis and is distributed on terms of Creative Commons Share Alike License. - -License for 'lib-python/2.7' -============================ + +License for 'lib-python/2.7, lib-python/3' +========================================== Except when otherwise stated (look for LICENSE files or copyright/license -information at the beginning of each file) the files in the 'lib-python/2.7' +information at the beginning of each file) the files in the 'lib-python' directory are all copyrighted by the Python Software Foundation and licensed -under the terms that you can find here: https://docs.python.org/2/license.html +under the terms that you can find here: https://docs.python.org/3/license.html License for 'pypy/module/unicodedata/' ====================================== @@ -441,9 +448,9 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/README.rst b/README.rst --- a/README.rst +++ b/README.rst @@ -4,42 +4,40 @@ Welcome to PyPy! -PyPy is both an implementation of the Python programming language, and -an extensive compiler framework for dynamic language implementations. -You can build self-contained Python implementations which execute -independently from CPython. +PyPy is an interperter that implements the Python programming language, based +on the RPython compiler framework for dynamic language implementations. -The home page is: +The home page for the interpreter is: http://pypy.org/ -If you want to help developing PyPy, this document might help you: +If you want to help developing PyPy, this documentation might help you: http://doc.pypy.org/ -It will also point you to the rest of the documentation which is generated -from files in the pypy/doc directory within the source repositories. Enjoy -and send us feedback! +More documentation about the RPython framework can be found here - the pypy-dev team + http://rpython.readthedocs.io +The source for the documentation is in the pypy/doc directory + +Using PyPy instead of CPython +============================= + +Please read the information at http://pypy.org to find the correct way to +download and use PyPy as an alternative to CPython. Building ======== -First switch to or download the correct branch. The basic choices are -``default`` for Python 2.7 and, for Python 3.X, the corresponding py3.X -branch (e.g. ``py3.5``). +Building PyPy is not the recommended way to obtain the PyPy alternative python +interpreter. It is time-consuming and requires significant computing resources. +More information can be found here -Build with: + http://doc.pypy.org/en/latest/build.html -.. code-block:: console +Enjoy and send us feedback! - $ rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py + the pypy-dev team -This ends up with a ``pypy-c`` or ``pypy3-c`` binary in the main pypy -directory. We suggest to use virtualenv with the resulting -pypy-c/pypy3-c as the interpreter; you can find more details about -various installation schemes here: - http://doc.pypy.org/en/latest/install.html diff --git a/pypy/module/test_lib_pypy/pyrepl/__init__.py b/extra_tests/test_pyrepl/__init__.py rename from pypy/module/test_lib_pypy/pyrepl/__init__.py rename to extra_tests/test_pyrepl/__init__.py --- a/pypy/module/test_lib_pypy/pyrepl/__init__.py +++ b/extra_tests/test_pyrepl/__init__.py @@ -1,3 +1,1 @@ -import sys -import lib_pypy.pyrepl -sys.modules['pyrepl'] = sys.modules['lib_pypy.pyrepl'] + diff --git a/pypy/module/test_lib_pypy/pyrepl/infrastructure.py b/extra_tests/test_pyrepl/infrastructure.py rename from pypy/module/test_lib_pypy/pyrepl/infrastructure.py rename to extra_tests/test_pyrepl/infrastructure.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_basic.py b/extra_tests/test_pyrepl/test_basic.py rename from pypy/module/test_lib_pypy/pyrepl/test_basic.py rename to extra_tests/test_pyrepl/test_basic.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_bugs.py b/extra_tests/test_pyrepl/test_bugs.py rename from pypy/module/test_lib_pypy/pyrepl/test_bugs.py rename to extra_tests/test_pyrepl/test_bugs.py diff --git a/extra_tests/test_pyrepl/test_functional.py b/extra_tests/test_pyrepl/test_functional.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_pyrepl/test_functional.py @@ -0,0 +1,28 @@ +# Copyright 2000-2007 Michael Hudson-Doyle +# Maciek Fijalkowski +# License: MIT +# some functional tests, to see if this is really working + +import pytest +import sys + + + at pytest.fixture() +def child(): + try: + import pexpect + except ImportError: + pytest.skip("no pexpect module") + except SyntaxError: + pytest.skip('pexpect wont work on py3k') + child = pexpect.spawn(sys.executable, ['-S'], timeout=10) + child.logfile = sys.stdout + child.sendline('from pyrepl.python_reader import main') + child.sendline('main()') + return child + + +def test_basic(child): + child.sendline('a = 3') + child.sendline('a') + child.expect('3') diff --git a/pypy/module/test_lib_pypy/pyrepl/test_keymap.py b/extra_tests/test_pyrepl/test_keymap.py rename from pypy/module/test_lib_pypy/pyrepl/test_keymap.py rename to extra_tests/test_pyrepl/test_keymap.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_reader.py b/extra_tests/test_pyrepl/test_reader.py rename from pypy/module/test_lib_pypy/pyrepl/test_reader.py rename to extra_tests/test_pyrepl/test_reader.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_readline.py b/extra_tests/test_pyrepl/test_readline.py rename from pypy/module/test_lib_pypy/pyrepl/test_readline.py rename to extra_tests/test_pyrepl/test_readline.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_wishes.py b/extra_tests/test_pyrepl/test_wishes.py rename from pypy/module/test_lib_pypy/pyrepl/test_wishes.py rename to extra_tests/test_pyrepl/test_wishes.py diff --git a/get_externals.py b/get_externals.py new file mode 100644 --- /dev/null +++ b/get_externals.py @@ -0,0 +1,69 @@ +'''Get external dependencies for building PyPy +they will end up in the platform.host().basepath, something like repo-root/external +''' + +from __future__ import print_function + +import argparse +import os +import zipfile +from subprocess import Popen, PIPE +from rpython.translator.platform import host + +def runcmd(cmd, verbose): + stdout = stderr = '' + report = False + try: + p = Popen(cmd, stdout=PIPE, stderr=PIPE) + stdout, stderr = p.communicate() + if p.wait() != 0 or verbose: + report = True + except Exception as e: + stderr = str(e) + '\n' + stderr + report = True + if report: + print('running "%s" returned\n%s\n%s' % (' '.join(cmd), stdout, stderr)) + if stderr: + raise RuntimeError(stderr) + +def checkout_repo(dest='externals', org='pypy', branch='default', verbose=False): + url = 'https://bitbucket.org/{}/externals'.format(org) + if not os.path.exists(dest): + cmd = ['hg','clone',url,dest] + runcmd(cmd, verbose) + cmd = ['hg','-R', dest, 'update',branch] + runcmd(cmd, verbose) + +def extract_zip(externals_dir, zip_path): + with zipfile.ZipFile(os.fspath(zip_path)) as zf: + zf.extractall(os.fspath(externals_dir)) + return externals_dir / zf.namelist()[0].split('/')[0] + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument('-v', '--verbose', action='store_true') + p.add_argument('-O', '--organization', + help='Organization owning the deps repos', default='pypy') + p.add_argument('-e', '--externals', default=host.externals, + help='directory in which to store dependencies', + ) + p.add_argument('-b', '--branch', default=host.externals_branch, + help='branch to check out', + ) + p.add_argument('-p', '--platform', default=None, + help='someday support cross-compilation, ignore for now', + ) + return p.parse_args() + + +def main(): + args = parse_args() + checkout_repo( + dest=args.externals, + org=args.organization, + branch=args.branch, + verbose=args.verbose, + ) + +if __name__ == '__main__': + main() diff --git a/lib-python/2.7/re.py b/lib-python/2.7/re.py --- a/lib-python/2.7/re.py +++ b/lib-python/2.7/re.py @@ -225,7 +225,7 @@ _pattern_type = type(sre_compile.compile("", 0)) -_MAXCACHE = 100 +_MAXCACHE = 1000 def _compile(*key): # internal: compile pattern diff --git a/lib-python/2.7/test/test_eof.py b/lib-python/2.7/test/test_eof.py --- a/lib-python/2.7/test/test_eof.py +++ b/lib-python/2.7/test/test_eof.py @@ -5,7 +5,7 @@ class EOFTestCase(unittest.TestCase): def test_EOFC(self): - expect = "EOL while scanning string literal (, line 1)" + expect = "end of line (EOL) while scanning string literal (, line 1)" try: eval("""'this is a test\ """) @@ -15,7 +15,7 @@ raise test_support.TestFailed def test_EOFS(self): - expect = ("EOF while scanning triple-quoted string literal " + expect = ("end of file (EOF) while scanning triple-quoted string literal " "(, line 1)") try: eval("""'''this is a test""") diff --git a/lib-python/2.7/test/test_generators.py b/lib-python/2.7/test/test_generators.py --- a/lib-python/2.7/test/test_generators.py +++ b/lib-python/2.7/test/test_generators.py @@ -398,7 +398,10 @@ 0 >>> type(i.gi_frame) ->>> i.gi_running = 42 + +PyPy prints "readonly attribute 'gi_running'" so ignore the exception detail + +>>> i.gi_running = 42 # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ... TypeError: readonly attribute diff --git a/lib-python/2.7/test/test_genexps.py b/lib-python/2.7/test/test_genexps.py --- a/lib-python/2.7/test/test_genexps.py +++ b/lib-python/2.7/test/test_genexps.py @@ -87,7 +87,7 @@ >>> dict(a = i for i in xrange(10)) Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: invalid syntax (expected ')') Verify that parenthesis are required when used as a keyword argument value diff --git a/lib-python/2.7/test/test_traceback.py b/lib-python/2.7/test/test_traceback.py --- a/lib-python/2.7/test/test_traceback.py +++ b/lib-python/2.7/test/test_traceback.py @@ -123,10 +123,7 @@ self.assertEqual(len(err), 4) self.assertEqual(err[1].strip(), "print(2)") self.assertIn("^", err[2]) - if check_impl_detail(): - self.assertEqual(err[1].find("p"), err[2].find("^")) - if check_impl_detail(pypy=True): - self.assertEqual(err[1].find("2)") + 1, err[2].find("^")) + self.assertEqual(err[1].find("p"), err[2].find("^")) def test_base_exception(self): # Test that exceptions derived from BaseException are formatted right diff --git a/lib-python/2.7/threading.py b/lib-python/2.7/threading.py --- a/lib-python/2.7/threading.py +++ b/lib-python/2.7/threading.py @@ -351,6 +351,21 @@ # forward-compatibility reasons we do the same. waiter.acquire() gotit = True + except AttributeError: + # someone patched the 'waiter' class, probably. + # Fall back to the standard CPython logic. + # See the CPython lib for the comments about it... + endtime = _time() + timeout + delay = 0.0005 # 500 us -> initial delay of 1 ms + while True: + gotit = waiter.acquire(0) + if gotit: + break + remaining = endtime - _time() + if remaining <= 0: + break + delay = min(delay * 2, remaining, .05) + _sleep(delay) else: gotit = waiter.acquire(False) if not gotit: diff --git a/lib_pypy/_pypy_testcapi.py b/lib_pypy/_pypy_testcapi.py --- a/lib_pypy/_pypy_testcapi.py +++ b/lib_pypy/_pypy_testcapi.py @@ -2,7 +2,7 @@ import tempfile, binascii -def get_hashed_dir(cfile): +def _get_hashed_filename(cfile): with open(cfile,'r') as fid: content = fid.read() # from cffi's Verifier() @@ -21,10 +21,28 @@ username = os.environ['USERNAME'] #windows except KeyError: username = os.getuid() - output_dir = tempfile.gettempdir() + os.path.sep + 'tmp_%s_%s%s' % ( + return tempfile.gettempdir() + os.path.sep + 'testcapi_%s_%s%s' % ( username, k1, k2) - if not os.path.exists(output_dir): + +def get_hashed_dir(cfile): + hashed_fn = _get_hashed_filename(cfile) + try: + with open(hashed_fn) as f: + dirname = f.read(1024) + except IOError: + dirname = '' + tmpdir = tempfile.gettempdir() + if (not dirname or '/' in dirname or '\\' in dirname or '\x00' in dirname + or not os.path.isdir(os.path.join(tmpdir, dirname))): + dirname = binascii.hexlify(os.urandom(8)) + if not isinstance(dirname, str): # Python 3 + dirname = dirname.decode('ascii') + dirname = 'testcapi_' + dirname + output_dir = os.path.join(tmpdir, dirname) + try: os.mkdir(output_dir) + except OSError: + pass return output_dir @@ -34,13 +52,12 @@ return ext -def compile_shared(csource, modulename, output_dir=None): +def compile_shared(csource, modulename, output_dir): """Compile '_testcapi.c' or '_ctypes_test.c' into an extension module, and import it. """ thisdir = os.path.dirname(__file__) - if output_dir is None: - output_dir = tempfile.mkdtemp() + assert output_dir is not None from distutils.ccompiler import new_compiler @@ -85,4 +102,16 @@ # Now import the newly created library, it will replace the original # module in sys.modules fp, filename, description = imp.find_module(modulename, path=[output_dir]) - imp.load_module(modulename, fp, filename, description) + with fp: + imp.load_module(modulename, fp, filename, description) + + # If everything went fine up to now, write the name of this new + # directory to 'hashed_fn', for future processes (and to avoid a + # growing number of temporary directories that are not completely + # obvious to clean up on Windows) + hashed_fn = _get_hashed_filename(os.path.join(thisdir, csource)) + try: + with open(hashed_fn, 'w') as f: + f.write(os.path.basename(output_dir)) + except IOError: + pass diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -153,9 +153,10 @@ factory = Connection if not factory else factory # an sqlite3 db seems to be around 100 KiB at least (doesn't matter if # backed by :memory: or a file) + res = factory(database, timeout, detect_types, isolation_level, + check_same_thread, factory, cached_statements) add_memory_pressure(100 * 1024) - return factory(database, timeout, detect_types, isolation_level, - check_same_thread, factory, cached_statements) + return res def _unicode_text_factory(x): diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.11.4 +Version: 1.11.5 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI from .error import CDefError, FFIError, VerificationError, VerificationMissing -__version__ = "1.11.4" -__version_info__ = (1, 11, 4) +__version__ = "1.11.5" +__version_info__ = (1, 11, 5) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h --- a/lib_pypy/cffi/_embedding.h +++ b/lib_pypy/cffi/_embedding.h @@ -146,32 +146,6 @@ PyGILState_STATE state; PyObject *pycode=NULL, *global_dict=NULL, *x; -#if PY_MAJOR_VERSION >= 3 - /* see comments in _cffi_carefully_make_gil() about the - Python2/Python3 difference - */ -#else - /* Acquire the GIL. We have no threadstate here. If Python is - already initialized, it is possible that there is already one - existing for this thread, but it is not made current now. - */ - PyEval_AcquireLock(); - - _cffi_py_initialize(); - - /* The Py_InitializeEx() sometimes made a threadstate for us, but - not always. Indeed Py_InitializeEx() could be called and do - nothing. So do we have a threadstate, or not? We don't know, - but we can replace it with NULL in all cases. - */ - (void)PyThreadState_Swap(NULL); - - /* Now we can release the GIL and re-acquire immediately using the - logic of PyGILState(), which handles making or installing the - correct threadstate. - */ - PyEval_ReleaseLock(); -#endif state = PyGILState_Ensure(); /* Call the initxxx() function from the present module. It will @@ -247,7 +221,7 @@ if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.11.4" + "\ncompiled with cffi version: 1.11.5" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); @@ -278,16 +252,14 @@ that we don't hold the GIL before (if it exists), and we don't hold it afterwards. - What it really does is completely different in Python 2 and - Python 3. + (What it really does used to be completely different in Python 2 + and Python 3, with the Python 2 solution avoiding the spin-lock + around the Py_InitializeEx() call. However, after recent changes + to CPython 2.7 (issue #358) it no longer works. So we use the + Python 3 solution everywhere.) - Python 2 - ======== - - Initialize the GIL, without initializing the rest of Python, - by calling PyEval_InitThreads(). - - PyEval_InitThreads() must not be called concurrently at all. + This initializes Python by calling Py_InitializeEx(). + Important: this must not be called concurrently at all. So we use a global variable as a simple spin lock. This global variable must be from 'libpythonX.Y.so', not from this cffi-based extension module, because it must be shared from @@ -297,18 +269,6 @@ string "ENDMARKER". We change it temporarily to point to the next character in that string. (Yes, I know it's REALLY obscure.) - - Python 3 - ======== - - In Python 3, PyEval_InitThreads() cannot be called before - Py_InitializeEx() any more. So this function calls - Py_InitializeEx() first. It uses the same obscure logic to - make sure we never call it concurrently. - - Arguably, this is less good on the spinlock, because - Py_InitializeEx() takes much longer to run than - PyEval_InitThreads(). But I didn't find a way around it. */ #ifdef WITH_THREAD @@ -332,8 +292,7 @@ } #endif -#if PY_MAJOR_VERSION >= 3 - /* Python 3: call Py_InitializeEx() */ + /* call Py_InitializeEx() */ { PyGILState_STATE state = PyGILState_UNLOCKED; if (!Py_IsInitialized()) @@ -344,17 +303,6 @@ PyEval_InitThreads(); PyGILState_Release(state); } -#else - /* Python 2: call PyEval_InitThreads() */ -# ifdef WITH_THREAD - if (!PyEval_ThreadsInitialized()) { - PyEval_InitThreads(); /* makes the GIL */ - PyEval_ReleaseLock(); /* then release it */ - } - /* else: there is already a GIL, but we still needed to do the - spinlock dance to make sure that we see it as fully ready */ -# endif -#endif #ifdef WITH_THREAD /* release the lock */ diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -143,6 +143,13 @@ self._libraries.append(lib) return lib + def dlclose(self, lib): + """Close a library obtained with ffi.dlopen(). After this call, + access to functions or variables from the library will fail + (possibly with a segmentation fault). + """ + type(lib).__cffi_close__(lib) + def _typeof_locked(self, cdecl): # call me with the lock! key = cdecl @@ -898,6 +905,9 @@ return addressof_var(name) raise AttributeError("cffi library has no function or " "global variable named '%s'" % (name,)) + def __cffi_close__(self): + backendlib.close_lib() + self.__dict__.clear() # if libname is not None: try: diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py --- a/lib_pypy/cffi/model.py +++ b/lib_pypy/cffi/model.py @@ -352,21 +352,20 @@ self.fldquals = fldquals self.build_c_name_with_marker() - def has_anonymous_struct_fields(self): - if self.fldtypes is None: - return False - for name, type in zip(self.fldnames, self.fldtypes): - if name == '' and isinstance(type, StructOrUnion): - return True - return False + def anonymous_struct_fields(self): + if self.fldtypes is not None: + for name, type in zip(self.fldnames, self.fldtypes): + if name == '' and isinstance(type, StructOrUnion): + yield type - def enumfields(self): + def enumfields(self, expand_anonymous_struct_union=True): fldquals = self.fldquals if fldquals is None: fldquals = (0,) * len(self.fldnames) for name, type, bitsize, quals in zip(self.fldnames, self.fldtypes, self.fldbitsize, fldquals): - if name == '' and isinstance(type, StructOrUnion): + if (name == '' and isinstance(type, StructOrUnion) + and expand_anonymous_struct_union): # nested anonymous struct/union for result in type.enumfields(): yield result diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py --- a/lib_pypy/cffi/recompiler.py +++ b/lib_pypy/cffi/recompiler.py @@ -836,6 +836,10 @@ def _struct_collecttype(self, tp): self._do_collect_type(tp) + if self.target_is_python: + # also requires nested anon struct/unions in ABI mode, recursively + for fldtype in tp.anonymous_struct_fields(): + self._struct_collecttype(fldtype) def _struct_decl(self, tp, cname, approxname): if tp.fldtypes is None: @@ -884,7 +888,7 @@ named_ptr not in self.ffi._parser._included_declarations)): if tp.fldtypes is None: pass # opaque - elif tp.partial or tp.has_anonymous_struct_fields(): + elif tp.partial or any(tp.anonymous_struct_fields()): pass # field layout obtained silently from the C compiler else: flags.append("_CFFI_F_CHECK_FIELDS") @@ -896,7 +900,8 @@ flags = '|'.join(flags) or '0' c_fields = [] if reason_for_not_expanding is None: - enumfields = list(tp.enumfields()) + expand_anonymous_struct_union = not self.target_is_python + enumfields = list(tp.enumfields(expand_anonymous_struct_union)) for fldname, fldtype, fbitsize, fqual in enumfields: fldtype = self._field_type(tp, fldname, fldtype) self._check_not_opaque(fldtype, diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py --- a/lib_pypy/cffi/setuptools_ext.py +++ b/lib_pypy/cffi/setuptools_ext.py @@ -81,8 +81,13 @@ it doesn't so far, creating troubles. That's why we check for "not hasattr(sys, 'gettotalrefcount')" (the 2.7 compatible equivalent of 'd' not in sys.abiflags). (http://bugs.python.org/issue28401) + + On Windows, it's better not to use py_limited_api until issue #355 + can be resolved (by having virtualenv copy PYTHON3.DLL). See also + the start of _cffi_include.h. """ - if 'py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount'): + if ('py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount') + and sys.platform != 'win32'): import setuptools try: setuptools_major_version = int(setuptools.__version__.partition('.')[0]) @@ -143,8 +148,8 @@ def _add_py_module(dist, ffi, module_name): from distutils.dir_util import mkpath - from distutils.command.build_py import build_py - from distutils.command.build_ext import build_ext + from setuptools.command.build_py import build_py + from setuptools.command.build_ext import build_ext from distutils import log from cffi import recompiler @@ -164,6 +169,17 @@ generate_mod(os.path.join(self.build_lib, *module_path)) dist.cmdclass['build_py'] = build_py_make_mod + # distutils and setuptools have no notion I could find of a + # generated python module. If we don't add module_name to + # dist.py_modules, then things mostly work but there are some + # combination of options (--root and --record) that will miss + # the module. So we add it here, which gives a few apparently + # harmless warnings about not finding the file outside the + # build directory. + if dist.py_modules is None: + dist.py_modules = [] + dist.py_modules.append(module_name) + # the following is only for "build_ext -i" base_class_2 = dist.cmdclass.get('build_ext', build_ext) class build_ext_make_mod(base_class_2): diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py --- a/lib_pypy/datetime.py +++ b/lib_pypy/datetime.py @@ -1415,9 +1415,14 @@ self.__setstate(year, month) self._hashcode = -1 return self - year, month, day = _check_date_fields(year, month, day) - hour, minute, second, microsecond = _check_time_fields( - hour, minute, second, microsecond) + elif isinstance(year, tuple) and len(year) == 7: + # Used by internal functions where the arguments are guaranteed to + # be valid. + year, month, day, hour, minute, second, microsecond = year + else: + year, month, day = _check_date_fields(year, month, day) + hour, minute, second, microsecond = _check_time_fields( + hour, minute, second, microsecond) _check_tzinfo_arg(tzinfo) self = dateinterop.__new__(cls) self._year = year @@ -1491,7 +1496,7 @@ us = 0 y, m, d, hh, mm, ss, weekday, jday, dst = converter(timestamp) ss = min(ss, 59) # clamp out leap seconds if the platform has them - return cls(y, m, d, hh, mm, ss, us, tzinfo) + return cls((y, m, d, hh, mm, ss, us), tzinfo=tzinfo) @classmethod def now(cls, tz=None): @@ -1800,7 +1805,7 @@ return diff and 1 or 0 def _add_timedelta(self, other, factor): - y, m, d, hh, mm, ss, us = _normalize_datetime( + result = _normalize_datetime( self._year, self._month, self._day + other.days * factor, @@ -1808,7 +1813,7 @@ self._minute, self._second + other.seconds * factor, self._microsecond + other.microseconds * factor) - return datetime(y, m, d, hh, mm, ss, us, tzinfo=self._tzinfo) + return datetime(result, tzinfo=self._tzinfo) def __add__(self, other): "Add a datetime and a timedelta." diff --git a/lib_pypy/dbm.py b/lib_pypy/dbm.py --- a/lib_pypy/dbm.py +++ b/lib_pypy/dbm.py @@ -157,7 +157,14 @@ def open(filename, flag='r', mode=0666): "open a DBM database" if not isinstance(filename, str): - raise TypeError("expected string") + if sys.version_info < (3,) and isinstance(filename, unicode): + # unlike CPython we'll encode 'filename' with filesystemencoding + # instead of defaultencoding, because that seems like a far + # better idea. But I'm also open for saying that we should + # rather go for bug-to-bug compatibility instead. + filename = filename.encode(sys.getfilesystemencoding()) + else: + raise TypeError("expected string") openflag = 0 diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info --- a/lib_pypy/greenlet.egg-info +++ b/lib_pypy/greenlet.egg-info @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: greenlet -Version: 0.4.12 +Version: 0.4.13 Summary: Lightweight in-process concurrent programming Home-page: https://github.com/python-greenlet/greenlet Author: Ralf Schmitt (for CPython), PyPy team diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py --- a/lib_pypy/greenlet.py +++ b/lib_pypy/greenlet.py @@ -1,7 +1,7 @@ import sys import _continuation -__version__ = "0.4.12" +__version__ = "0.4.13" # ____________________________________________________________ # Exceptions diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -56,13 +56,13 @@ Niko Matsakis Alexander Hesse Ludovic Aubry + stian Jacob Hallen Jason Creighton Mark Young Alex Martelli Spenser Bauman Michal Bendowski - stian Jan de Mooij Tyler Wade Vincent Legoll @@ -90,10 +90,10 @@ Wenzhu Man Konstantin Lopuhin John Witulski + Jeremy Thurgood Greg Price Ivan Sichmann Freitas Dario Bertini - Jeremy Thurgood Mark Pearse Simon Cross Tobias Pape @@ -112,18 +112,19 @@ Adrian Kuhn tav Georg Brandl + Joannah Nanjekye Bert Freudenberg Stian Andreassen Wanja Saatkamp Mike Blume - Joannah Nanjekye Gerald Klix Oscar Nierstrasz Rami Chowdhury Stefan H. Muller + Dodan Mihai Tim Felgentreff Eugene Oden - Dodan Mihai + Colin Valliant Jeff Terrace Henry Mason Vasily Kuznetsov @@ -192,12 +193,14 @@ Vaibhav Sood Reuben Cummings Attila Gobi + Floris Bruynooghe Christopher Pope Tristan Arthur Christian Tismer Dan Stromberg Carl Meyer Florin Papa + Arianna Avanzini Jens-Uwe Mager Valentina Mukhamedzhanova Stefano Parmesan @@ -221,6 +224,7 @@ Mark Williams Kunal Grover Nathan Taylor + Barry Hart Travis Francis Athougies Yasir Suhail Sergey Kishchenko @@ -228,6 +232,7 @@ Lutz Paelike Ian Foote Philipp Rustemeuer + Logan Chien Catalin Gabriel Manciu Jacob Oscarson Ryan Gonzalez @@ -263,7 +268,6 @@ Akira Li Gustavo Niemeyer Rafał Gałczyński - Logan Chien Lucas Stadler roberto at goyle Matt Bogosian @@ -276,6 +280,7 @@ Anna Katrina Dominguez Kim Jin Su Amber Brown + Miro Hrončok Anthony Sottile Nate Bragg Ben Darnell @@ -283,7 +288,6 @@ Godefroid Chappelle Julian Berman Michael Hudson-Doyle - Floris Bruynooghe Stephan Busemann Dan Colish timo @@ -325,6 +329,7 @@ Michael Chermside Anna Ravencroft remarkablerocket + Pauli Virtanen Petre Vijiac Berker Peksag Christian Muirhead @@ -349,6 +354,7 @@ Graham Markall Dan Loewenherz werat + Andrew Stepanov Niclas Olofsson Chris Pressey Tobias Diaz diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -10,89 +10,6 @@ PyPy. -.. _extension-modules: - -Extension modules ------------------ - -List of extension modules that we support: - -* Supported as built-in modules (in :source:`pypy/module/`): - - __builtin__ - :doc:`__pypy__ <__pypy__-module>` - _ast - _codecs - _collections - :doc:`_continuation ` - :doc:`_ffi ` - _hashlib - _io - _locale - _lsprof - _md5 - :doc:`_minimal_curses ` - _multiprocessing - _random - :doc:`_rawffi ` - _sha - _socket - _sre - _ssl - _warnings - _weakref - _winreg - array - binascii - bz2 - cStringIO - cmath - `cpyext`_ - crypt - errno - exceptions - fcntl - gc - imp - itertools - marshal - math - mmap - operator - parser - posix - pyexpat - select - signal - struct - symbol - sys - termios - thread - time - token - unicodedata - zipimport - zlib - - When translated on Windows, a few Unix-only modules are skipped, - and the following module is built instead: - - _winreg - -* Supported by being rewritten in pure Python (possibly using ``cffi``): - see the :source:`lib_pypy/` directory. Examples of modules that we - support this way: ``ctypes``, ``cPickle``, ``cmath``, ``dbm``, ``datetime``... - Note that some modules are both in there and in the list above; - by default, the built-in module is used (but can be disabled - at translation time). - -The extension modules (i.e. modules written in C, in the standard CPython) -that are neither mentioned above nor in :source:`lib_pypy/` are not available in PyPy. -(You may have a chance to use them anyway with `cpyext`_.) - -.. _cpyext: http://morepypy.blogspot.com/2010/04/using-cpython-extension-modules-with.html - Differences related to garbage collection strategies ---------------------------------------------------- @@ -554,7 +471,101 @@ versions of PyPy may have to rename the arguments if CPython starts accepting them too. +* PyPy3: ``distutils`` has been enhanced to allow finding ``VsDevCmd.bat`` in the + directory pointed to by the ``VS%0.f0COMNTOOLS`` (typically ``VS140COMNTOOLS``) + environment variable. CPython searches for ``vcvarsall.bat`` somewhere **above** + that value. + +* SyntaxError_ s try harder to give details about the cause of the failure, so + the error messages are not the same as in CPython + + +.. _extension-modules: + +Extension modules +----------------- + +List of extension modules that we support: + +* Supported as built-in modules (in :source:`pypy/module/`): + + __builtin__ + :doc:`__pypy__ <__pypy__-module>` + _ast + _codecs + _collections + :doc:`_continuation ` + :doc:`_ffi ` + _hashlib + _io + _locale + _lsprof + _md5 + :doc:`_minimal_curses ` + _multiprocessing + _random + :doc:`_rawffi ` + _sha + _socket + _sre + _ssl + _warnings + _weakref + _winreg + array + binascii + bz2 + cStringIO + cmath + `cpyext`_ + crypt + errno + exceptions + fcntl + gc + imp + itertools + marshal + math + mmap + operator + parser + posix + pyexpat + select + signal + struct + symbol + sys + termios + thread + time + token + unicodedata + zipimport + zlib + + When translated on Windows, a few Unix-only modules are skipped, + and the following module is built instead: + + _winreg + +* Supported by being rewritten in pure Python (possibly using ``cffi``): + see the :source:`lib_pypy/` directory. Examples of modules that we + support this way: ``ctypes``, ``cPickle``, ``cmath``, ``dbm``, ``datetime``... + Note that some modules are both in there and in the list above; + by default, the built-in module is used (but can be disabled + at translation time). + +The extension modules (i.e. modules written in C, in the standard CPython) +that are neither mentioned above nor in :source:`lib_pypy/` are not available in PyPy. +(You may have a chance to use them anyway with `cpyext`_.) + +.. _cpyext: http://morepypy.blogspot.com/2010/04/using-cpython-extension-modules-with.html + + .. _`is ignored in PyPy`: http://bugs.python.org/issue14621 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/ .. _`issue #2653`: https://bitbucket.org/pypy/pypy/issues/2653/ +.. _SyntaxError: https://morepypy.blogspot.co.il/2018/04/improving-syntaxerror-in-pypy.html diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst --- a/pypy/doc/gc_info.rst +++ b/pypy/doc/gc_info.rst @@ -1,17 +1,297 @@ -Garbage collector configuration -=============================== +Garbage collector documentation and configuration +================================================= + +Incminimark +----------- + +PyPy's default garbage collector is called incminimark - it's an incremental, +generational moving collector. Here we hope to explain a bit how it works +and how it can be tuned to suit the workload. + +Incminimark first allocates objects in so called *nursery* - place for young +objects, where allocation is very cheap, being just a pointer bump. The nursery +size is a very crucial variable - depending on your workload (one or many +processes) and cache sizes you might want to experiment with it via +*PYPY_GC_NURSERY* environment variable. When the nursery is full, there is +performed a minor collection. Freed objects are no longer referencable and +just die, just by not being referenced any more; on the other hand, objects +found to still be alive must survive and are copied from the nursery +to the old generation. Either to arenas, which are collections +of objects of the same size, or directly allocated with malloc if they're +larger. (A third category, the very large objects, are initially allocated +outside the nursery and never move.) + +Since Incminimark is an incremental GC, the major collection is incremental, +meaning there should not be any pauses longer than 1ms. + + +Fragmentation +------------- + +Before we discuss issues of "fragmentation", we need a bit of precision. +There are two kinds of related but distinct issues: + +* If the program allocates a lot of memory, and then frees it all by + dropping all references to it, then we might expect to see the RSS + to drop. (RSS = Resident Set Size on Linux, as seen by "top"; it is an + approximation of the actual memory usage from the OS's point of view.) + This might not occur: the RSS may remain at its highest value. This + issue is more precisely caused by the process not returning "free" + memory to the OS. We call this case "unreturned memory". + +* After doing the above, if the RSS didn't go down, then at least future + allocations should not cause the RSS to grow more. That is, the process + should reuse unreturned memory as long as it has got some left. If this + does not occur, the RSS grows even larger and we have real fragmentation + issues. + + +gc.get_stats +------------ + +There is a special function in the ``gc`` module called +``get_stats(memory_pressure=False)``. + +``memory_pressure`` controls whether or not to report memory pressure from +objects allocated outside of the GC, which requires walking the entire heap, +so it's disabled by default due to its cost. Enable it when debugging +mysterious memory disappearance. + +Example call looks like that:: + + >>> gc.get_stats(True) + Total memory consumed: + GC used: 4.2MB (peak: 4.2MB) + in arenas: 763.7kB + rawmalloced: 383.1kB + nursery: 3.1MB + raw assembler used: 0.0kB + memory pressure: 0.0kB + ----------------------------- + Total: 4.2MB + + Total memory allocated: + GC allocated: 4.5MB (peak: 4.5MB) + in arenas: 763.7kB + rawmalloced: 383.1kB + nursery: 3.1MB + raw assembler allocated: 0.0kB + memory pressure: 0.0kB + ----------------------------- + Total: 4.5MB + +In this particular case, which is just at startup, GC consumes relatively +little memory and there is even less unused, but allocated memory. In case +there is a lot of unreturned memory or actual fragmentation, the "allocated" +can be much higher than "used". Generally speaking, "peak" will more closely +resemble the actual memory consumed as reported by RSS. Indeed, returning +memory to the OS is a hard and not solved problem. In PyPy, it occurs only if +an arena is entirely free---a contiguous block of 64 pages of 4 or 8 KB each. +It is also rare for the "rawmalloced" category, at least for common system +implementations of ``malloc()``. + +The details of various fields: + +* GC in arenas - small old objects held in arenas. If the amount "allocated" + is much higher than the amount "used", we have unreturned memory. It is + possible but unlikely that we have internal fragmentation here. However, + this unreturned memory cannot be reused for any ``malloc()``, including the + memory from the "rawmalloced" section. + +* GC rawmalloced - large objects allocated with malloc. This is gives the + current (first block of text) and peak (second block of text) memory + allocated with ``malloc()``. The amount of unreturned memory or + fragmentation caused by ``malloc()`` cannot easily be reported. Usually + you can guess there is some if the RSS is much larger than the total + memory reported for "GC allocated", but do keep in mind that this total + does not include malloc'ed memory not known to PyPy's GC at all. If you + guess there is some, consider using `jemalloc`_ as opposed to system malloc. + +.. _`jemalloc`: http://jemalloc.net/ + +* nursery - amount of memory allocated for nursery, fixed at startup, + controlled via an environment variable + +* raw assembler allocated - amount of assembler memory that JIT feels + responsible for + +* memory pressure, if asked for - amount of memory we think got allocated + via external malloc (eg loading cert store in SSL contexts) that is kept + alive by GC objects, but not accounted in the GC + + +GC Hooks +-------- + +GC hooks are user-defined functions which are called whenever a specific GC +event occur, and can be used to monitor GC activity and pauses. You can +install the hooks by setting the following attributes: + +``gc.hook.on_gc_minor`` + Called whenever a minor collection occurs. It corresponds to + ``gc-minor`` sections inside ``PYPYLOG``. + +``gc.hook.on_gc_collect_step`` + Called whenever an incremental step of a major collection occurs. It + corresponds to ``gc-collect-step`` sections inside ``PYPYLOG``. + +``gc.hook.on_gc_collect`` + Called after the last incremental step, when a major collection is fully + done. It corresponds to ``gc-collect-done`` sections inside ``PYPYLOG``. + +To uninstall a hook, simply set the corresponding attribute to ``None``. To +install all hooks at once, you can call ``gc.hooks.set(obj)``, which will look +for methods ``on_gc_*`` on ``obj``. To uninstall all the hooks at once, you +can call ``gc.hooks.reset()``. + +The functions called by the hooks receive a single ``stats`` argument, which +contains various statistics about the event. + +Note that PyPy cannot call the hooks immediately after a GC event, but it has +to wait until it reaches a point in which the interpreter is in a known state +and calling user-defined code is harmless. It might happen that multiple +events occur before the hook is invoked: in this case, you can inspect the +value ``stats.count`` to know how many times the event occured since the last +time the hook was called. Similarly, ``stats.duration`` contains the +**total** time spent by the GC for this specific event since the last time the +hook was called. + +On the other hand, all the other fields of the ``stats`` object are relative +only to the **last** event of the series. + +The attributes for ``GcMinorStats`` are: + +``count`` + The number of minor collections occured since the last hook call. + +``duration`` + The total time spent inside minor collections since the last hook + call. See below for more information on the unit. + +``duration_min`` + The duration of the fastest minor collection since the last hook call. + +``duration_max`` + The duration of the slowest minor collection since the last hook call. + + ``total_memory_used`` + The amount of memory used at the end of the minor collection, in + bytes. This include the memory used in arenas (for GC-managed memory) and + raw-malloced memory (e.g., the content of numpy arrays). + +``pinned_objects`` + the number of pinned objects. + + +The attributes for ``GcCollectStepStats`` are: + +``count``, ``duration``, ``duration_min``, ``duration_max`` + See above. + +``oldstate``, ``newstate`` + Integers which indicate the state of the GC before and after the step. + +The value of ``oldstate`` and ``newstate`` is one of these constants, defined +inside ``gc.GcCollectStepStats``: ``STATE_SCANNING``, ``STATE_MARKING``, +``STATE_SWEEPING``, ``STATE_FINALIZING``. It is possible to get a string +representation of it by indexing the ``GC_STATS`` tuple. + + +The attributes for ``GcCollectStats`` are: + +``count`` + See above. + +``num_major_collects`` + The total number of major collections which have been done since the + start. Contrarily to ``count``, this is an always-growing counter and it's + not reset between invocations. + +``arenas_count_before``, ``arenas_count_after`` + Number of arenas used before and after the major collection. + +``arenas_bytes`` + Total number of bytes used by GC-managed objects. + +``rawmalloc_bytes_before``, ``rawmalloc_bytes_after`` + Total number of bytes used by raw-malloced objects, before and after the + major collection. + +Note that ``GcCollectStats`` has **not** got a ``duration`` field. This is +because all the GC work is done inside ``gc-collect-step``: +``gc-collect-done`` is used only to give additional stats, but doesn't do any +actual work. + +A note about the ``duration`` field: depending on the architecture and +operating system, PyPy uses different ways to read timestamps, so ``duration`` +is expressed in varying units. It is possible to know which by calling +``__pypy__.debug_get_timestamp_unit()``, which can be one of the following +values: + +``tsc`` + The default on ``x86`` machines: timestamps are expressed in CPU ticks, as + read by the `Time Stamp Counter`_. + +``ns`` + Timestamps are expressed in nanoseconds. + +``QueryPerformanceCounter`` + On Windows, in case for some reason ``tsc`` is not available: timestamps + are read using the win API ``QueryPerformanceCounter()``. + + +Unfortunately, there does not seem to be a reliable standard way for +converting ``tsc`` ticks into nanoseconds, although in practice on modern CPUs +it is enough to divide the ticks by the maximum nominal frequency of the CPU. +For this reason, PyPy gives the raw value, and leaves the job of doing the +conversion to external libraries. + +Here is an example of GC hooks in use:: + + import sys + import gc + + class MyHooks(object): + done = False + + def on_gc_minor(self, stats): + print 'gc-minor: count = %02d, duration = %d' % (stats.count, + stats.duration) + + def on_gc_collect_step(self, stats): + old = gc.GcCollectStepStats.GC_STATES[stats.oldstate] + new = gc.GcCollectStepStats.GC_STATES[stats.newstate] + print 'gc-collect-step: %s --> %s' % (old, new) + print ' count = %02d, duration = %d' % (stats.count, + stats.duration) + + def on_gc_collect(self, stats): + print 'gc-collect-done: count = %02d' % stats.count + self.done = True + + hooks = MyHooks() + gc.hooks.set(hooks) + + # simulate some GC activity + lst = [] + while not hooks.done: + lst = [lst, 1, 2, 3] + + +.. _`Time Stamp Counter`: https://en.wikipedia.org/wiki/Time_Stamp_Counter + .. _minimark-environment-variables: -Minimark --------- +Environment variables +--------------------- PyPy's default ``incminimark`` garbage collector is configurable through several environment variables: ``PYPY_GC_NURSERY`` The nursery size. - Defaults to 1/2 of your cache or ``4M``. + Defaults to 1/2 of your last-level cache, or ``4M`` if unknown. Small values (like 1 or 1KB) are useful for debugging. ``PYPY_GC_NURSERY_DEBUG`` diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst --- a/pypy/doc/how-to-release.rst +++ b/pypy/doc/how-to-release.rst @@ -40,6 +40,8 @@ sure things are ported back to the trunk and to the branch as necessary. +* Make sure the RPython builds on the buildbot pass with no failures + * Maybe bump the SOABI number in module/imp/importing. This has many implications, so make sure the PyPy community agrees to the change. diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-v6.0.0.rst release-v5.10.1.rst release-v5.10.0.rst release-v5.9.0.rst diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -7,6 +7,8 @@ .. toctree:: whatsnew-head.rst + whatsnew-pypy2-6.0.0.rst + whatsnew-pypy2-5.10.0.rst whatsnew-pypy2-5.10.0.rst whatsnew-pypy2-5.9.0.rst whatsnew-pypy2-5.8.0.rst diff --git a/pypy/doc/install.rst b/pypy/doc/install.rst --- a/pypy/doc/install.rst +++ b/pypy/doc/install.rst @@ -17,13 +17,18 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~ The quickest way to start using PyPy is to download a prebuilt binary for your -OS and architecture. You can either use the `most recent release`_ or one of -our `development nightly build`_. Please note that the nightly builds are not +OS and architecture. You may be able to use either use the +`most recent release`_ or one of our `development nightly build`_. These +builds depend on dynamically linked libraries that may not be available on your +OS. See the section about `Linux binaries` for more info and alternatives that +may work on your system. + +Please note that the nightly builds are not guaranteed to be as stable as official releases, use them at your own risk. .. _most recent release: http://pypy.org/download.html .. _development nightly build: http://buildbot.pypy.org/nightly/trunk/ - +.. _Linux binaries: http://pypy.org/download.html#linux-binaries-and-common-distributions Installing PyPy ~~~~~~~~~~~~~~~ @@ -69,9 +74,9 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~ It is often convenient to run pypy inside a virtualenv. To do this -you need a recent version of virtualenv -- 1.6.1 or greater. You can +you need a version of virtualenv -- 1.6.1 or greater. You can then install PyPy both from a precompiled tarball or from a mercurial -checkout:: +checkout after translation:: # from a tarball $ virtualenv -p /opt/pypy-xxx/bin/pypy my-pypy-env diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -1,26 +1,41 @@ Potential Project List ====================== -Google Summer of Code 2017 --------------------------- +Getting involved +---------------- -PyPy is generally open to new ideas for Google Summer of Code. We are happy to accept good ideas around the PyPy ecosystem. If you need more information about the ideas we propose for this year please join us on irc, channel #pypy (freenode). If you are unsure, but still think that you can make a valuable contribution to PyPy, dont hesitate to contact us on #pypy or on our mailing list. - +We are happy to discuss ideas around the PyPy ecosystem. +If you are interested in palying with RPython or PyPy, or have a new idea not +mentioned here please join us on irc, channel #pypy (freenode). If you are unsure, +but still think that you can make a valuable contribution to PyPy, dont +hesitate to contact us on #pypy or on our mailing list. Here are some ideas +to get you thinking: * **Optimize PyPy Memory Usage**: Sometimes PyPy consumes more memory than CPython. - Two examples: 1) PyPy seems to allocate and keep alive more strings when importing a big Python modules. - 2) The base interpreter size (cold VM started from a console) of PyPy is bigger than the one of CPython. - The general procedure of this project is: Run both CPython and PyPy of the same Python version and - compare the memory usage (using Massif or other tools). + Two examples: 1) PyPy seems to allocate and keep alive more strings when + importing a big Python modules. 2) The base interpreter size (cold VM started + from a console) of PyPy is bigger than the one of CPython. The general + procedure of this project is: Run both CPython and PyPy of the same Python + version and compare the memory usage (using Massif or other tools). If PyPy consumes a lot more memory then find and resolve the issue. -* **VMProf + memory profiler**: vmprof by now has a memory profiler that can be used already. We want extend it with more features and resolve some current limitations. +* **VMProf + memory profiler**: vmprof is a statistical memory profiler. We + want extend it with new features and resolve some current limitations. -* **VMProf visualisations**: vmprof just shows a flame graph of the statistical profile and some more information about specific call sites. It would be very interesting to experiment with different information (such as memory, or even information generated by our jit compiler). +* **VMProf visualisations**: vmprof shows a flame graph of the statistical + profile and some more information about specific call sites. It would be + very interesting to experiment with different information (such as memory, + or even information generated by our jit compiler). -* **Explicit typing in RPython**: PyPy wants to have better ways to specify the signature and class attribute types in RPython. See more information about this topic below on this page. +* **Explicit typing in RPython**: PyPy wants to have better ways to specify + the signature and class attribute types in RPython. See more information + about this topic below on this page. -* **Virtual Reality (VR) visualisations for vmprof**: This is a very open topic with lots of freedom to explore data visualisation for profiles. No VR hardware would be needed for this project. Either universities provide such hardware or in any other case we potentially can lend the VR hardware setup. +* **Virtual Reality (VR) visualisations for vmprof**: This is a very open + topic with lots of freedom to explore data visualisation for profiles. No + VR hardware would be needed for this project. Either universities provide + such hardware or in any other case we potentially can lend the VR hardware + setup. Simple tasks for newcomers -------------------------- @@ -34,6 +49,11 @@ * Implement AF_XXX packet types of sockets: https://bitbucket.org/pypy/pypy/issue/1942/support-for-af_xxx-sockets +* Help with documentation. One task would be to document rpython configuration + options currently listed only on :doc:`this site ` also on the + RPython_ documentation site. + +.. _RPython: http://rpython.readthedocs.io Mid-to-large tasks ------------------ @@ -201,7 +221,9 @@ Introduce new benchmarks ------------------------ -We're usually happy to introduce new benchmarks. Please consult us +Our benchmark runner_ is showing its age. We should merge with the `CPython site`_ + +Additionally, we're usually happy to introduce new benchmarks. Please consult us before, but in general something that's real-world python code and is not already represented is welcome. We need at least a standalone script that can run without parameters. Example ideas (benchmarks need @@ -209,6 +231,8 @@ * `hg` +.. _runner: http://speed.pypy.org +.. _`CPython site`: https://speed.python.org/ ====================================== Make more python modules pypy-friendly @@ -238,15 +262,6 @@ using more pypy-friendly technologies, e.g. cffi. Here is a partial list of good work that needs to be finished: -**matplotlib** https://github.com/matplotlib/matplotlib - - Status: using the matplotlib branch of PyPy and the tkagg-cffi branch of - matplotlib from https://github.com/mattip/matplotlib/tree/tkagg-cffi, the - tkagg backend can function. - - TODO: the matplotlib branch passes numpy arrays by value (copying all the - data), this proof-of-concept needs help to become completely compliant - **wxPython** https://bitbucket.org/amauryfa/wxpython-cffi Status: A project by a PyPy developer to adapt the Phoenix sip build system to cffi diff --git a/pypy/doc/release-v6.0.0.rst b/pypy/doc/release-v6.0.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-v6.0.0.rst @@ -0,0 +1,109 @@ +====================================== +PyPy2.7 and PyPy3.5 v6.0 dual release +====================================== + +The PyPy team is proud to release both PyPy2.7 v6.0 (an interpreter supporting +Python 2.7 syntax), and a PyPy3.5 v6.0 (an interpreter supporting Python +3.5 syntax). The two releases are both based on much the same codebase, thus +the dual release. + +This release is a feature release following our previous 5.10 incremental +release in late December 2017. Our C-API compatability layer ``cpyext`` is +now much faster (see the `blog post`_) as well as more complete. We have made +many other improvements in speed and CPython compatibility. Since the changes +affect the included python development header files, all c-extension modules must +be recompiled for this version. + +First-time python users are often stumped by silly typos and emissions when +getting started writing code. We have improved our parser to emit more friendly +`syntax errors`_, making PyPy not only faster but more friendly. + +The Windows PyPy3.5 release is still considered beta-quality. There are open +issues with unicode handling especially around system calls and c-extensions. + +The Matplotlib TkAgg backend now works with PyPy, as do pygame and pygobject_. + +As always, this release is 100% compatible with the previous one and fixed +several issues and bugs raised by the growing community of PyPy users. +We strongly recommend updating. + +We updated the cffi module included in PyPy to version 1.11.5 + +The utf8 branch that changes internal representation of unicode to utf8 did not +make it into the release, so there is still more goodness coming. We also +began working on a Python3.6 implementation, help is welcome. + +You can download the v6.0 releases here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project. If PyPy is not quite good enough for your needs, we are available for +direct consulting work. + +We would also like to thank our contributors and encourage new people to join +the project. PyPy has many layers and we need help with all of them: `PyPy`_ +and `RPython`_ documentation improvements, tweaking popular `modules`_ to run +on pypy, or general `help`_ with making RPython's JIT even better. + +.. _`PyPy`: index.html +.. _`RPython`: https://rpython.readthedocs.org +.. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly +.. _`help`: project-ideas.html +.. _`blog post`: https://morepypy.blogspot.it/2017/10/cape-of-good-hope-for-pypy-hello-from.html +.. _pygobject: https://lazka.github.io/posts/2018-04_pypy-pygobject/index.html +.. _`syntax errors`: https://morepypy.blogspot.com/2018/04/improving-syntaxerror-in-pypy.html + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7 and CPython 3.5. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +The PyPy release supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html + +Changelog +========= + +* Speed up C-API method calls, and make most Py*_Check calls C macros +* Speed up C-API slot method calls +* Enable TkAgg backend support for matplotlib +* support ``hastzinfo`` and ``tzinfo`` in the C-API ``PyDateTime*`` structures +* datetime.h is now more similar to CPython +* We now support ``PyUnicode_AsUTF{16,32}String``, ``_PyLong_AsByteArray``, + ``_PyLong_AsByteArrayO``, +* PyPy3.5 on Windows is compiled with the Microsoft Visual Compiler v14, like + CPython +* Fix performance of attribute lookup when more than 80 attributes are used +* Improve performance on passing built-in types to C-API C code +* Improve the performance of datetime and timedelta by skipping the consistency + checks of the datetime values (they are correct by construction) +* Improve handling of ``bigint`` s, including fixing ``int_divmod`` +* Improve reporting of GC statistics +* Accept unicode filenames in ``dbm.open()`` +* Improve RPython support for half-floats +* Added missing attributes to C-API ``instancemethod`` on pypy3 +* Store error state in thread-local storage for C-API. +* Fix JIT bugs exposed in the sre module +* Improve speed of Python parser, improve ParseError messages slightly +* Handle JIT hooks more efficiently + +We also refactored many parts of the JIT bridge optimizations, as well as cpyext +internals, and together with new contributors fixed issues, added new +documentation, and cleaned up the codebase. diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -1,16 +1,20 @@ -=========================== -What's new in PyPy2.7 5.10+ -=========================== +========================== +What's new in PyPy2.7 6.0+ +========================== -.. this is a revision shortly after release-pypy2.7-v5.10.0 -.. startrev: 6b024edd9d12 +.. this is a revision shortly after release-pypy-6.0.0 +.. startrev: f22145c34985 -.. branch: cpyext-avoid-roundtrip -Big refactoring of some cpyext code, which avoids a lot of nonsense when -calling C from Python and vice-versa: the result is a big speedup in -function/method calls, up to 6 times faster. +.. branch: issue2752 -.. branch: cpyext-datetime2 +Fix a rare GC bug that was introduced more than one year ago, but was +not diagnosed before issue #2752. -Support ``tzinfo`` field on C-API datetime objects, fixes latest pandas HEAD +.. branch: gc-hooks + +Introduce GC hooks, as documented in doc/gc_info.rst + +.. branch: gc-hook-better-timestamp + +Improve GC hooks diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-pypy2-6.0.0.rst copy from pypy/doc/whatsnew-head.rst copy to pypy/doc/whatsnew-pypy2-6.0.0.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-pypy2-6.0.0.rst @@ -14,3 +14,98 @@ .. branch: cpyext-datetime2 Support ``tzinfo`` field on C-API datetime objects, fixes latest pandas HEAD + + +.. branch: mapdict-size-limit + +Fix a corner case of mapdict: When an instance is used like a dict (using +``setattr`` and ``getattr``, or ``.__dict__``) and a lot of attributes are +added, then the performance using mapdict is linear in the number of +attributes. This is now fixed (by switching to a regular dict after 80 +attributes). + + +.. branch: cpyext-faster-arg-passing + +When using cpyext, improve the speed of passing certain objects from PyPy to C +code, most notably None, True, False, types, all instances of C-defined types. +Before, a dict lookup was needed every time such an object crossed over, now it +is just a field read. + + +.. branch: 2634_datetime_timedelta_performance + +Improve datetime + timedelta performance. + +.. branch: memory-accounting + +Improve way to describe memory + +.. branch: msvc14 + +Allow compilaiton with Visual Studio 2017 compiler suite on windows + +.. branch: refactor-slots + +Refactor cpyext slots. + + +.. branch: call-loopinvariant-into-bridges + +Speed up branchy code that does a lot of function inlining by saving one call +to read the TLS in most bridges. + From pypy.commits at gmail.com Fri Apr 20 20:33:18 2018 From: pypy.commits at gmail.com (wlav) Date: Fri, 20 Apr 2018 17:33:18 -0700 (PDT) Subject: [pypy-commit] pypy cppyy-packaging: translator fix Message-ID: <5ada86ce.16451c0a.1b4a5.35ee@mx.google.com> Author: Wim Lavrijsen Branch: cppyy-packaging Changeset: r94405:55cdebafb2de Date: 2018-04-20 17:15 -0700 http://bitbucket.org/pypy/pypy/changeset/55cdebafb2de/ Log: translator fix diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py --- a/pypy/module/_cppyy/interp_cppyy.py +++ b/pypy/module/_cppyy/interp_cppyy.py @@ -848,7 +848,7 @@ alldir = capi.c_get_all_cpp_names(self.space, self) w_alldir = self.space.newlist([]) for name in alldir: - w_alldir.append(self.space.wrap(name)) + w_alldir.append(self.space.newtext(name)) return w_alldir def missing_attribute_error(self, name): From pypy.commits at gmail.com Sat Apr 21 10:20:59 2018 From: pypy.commits at gmail.com (mwjackson) Date: Sat, 21 Apr 2018 07:20:59 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: (ronan, mwjackson) added kwargs-only for sorted() Message-ID: <5adb48cb.41711c0a.65e68.06cc@mx.google.com> Author: Matt Jackson Branch: py3.6 Changeset: r94406:5781b52fc50f Date: 2018-04-21 14:20 +0100 http://bitbucket.org/pypy/pypy/changeset/5781b52fc50f/ Log: (ronan, mwjackson) added kwargs-only for sorted() diff --git a/lib-python/3/test/test_builtin.py b/lib-python/3/test/test_builtin.py --- a/lib-python/3/test/test_builtin.py +++ b/lib-python/3/test/test_builtin.py @@ -1635,8 +1635,12 @@ def test_bad_arguments(self): # Issue #29327: The first argument is positional-only. sorted([]) - with self.assertRaises(TypeError): - sorted(iterable=[]) + + # pypy doesn't support positional only arguments + if check_impl_detail(): + with self.assertRaises(TypeError): + sorted(iterable=[]) + # Other arguments are keyword-only sorted([], key=None) with self.assertRaises(TypeError): diff --git a/pypy/module/__builtin__/app_functional.py b/pypy/module/__builtin__/app_functional.py --- a/pypy/module/__builtin__/app_functional.py +++ b/pypy/module/__builtin__/app_functional.py @@ -4,8 +4,7 @@ """ # ____________________________________________________________ - -def sorted(iterable, key=None, reverse=False): +def sorted(iterable, *, key=None, reverse=False): "sorted(iterable, key=None, reverse=False) --> new sorted list" sorted_lst = list(iterable) sorted_lst.sort(key=key, reverse=reverse) diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py --- a/pypy/module/__builtin__/test/test_builtin.py +++ b/pypy/module/__builtin__/test/test_builtin.py @@ -420,6 +420,7 @@ assert sorted_l is not l assert sorted_l == ['C', 'b', 'a'] raises(TypeError, sorted, [], reverse=None) + raises(TypeError, sorted, [], None) def test_reversed_simple_sequences(self): l = range(5) From pypy.commits at gmail.com Sat Apr 21 11:32:17 2018 From: pypy.commits at gmail.com (mwjackson) Date: Sat, 21 Apr 2018 08:32:17 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: (ronan mwjackson) fixed platform tests that utilise sys._git Message-ID: <5adb5981.4c3e1c0a.add5c.66e6@mx.google.com> Author: Matt Jackson Branch: py3.6 Changeset: r94407:86478b4c1d71 Date: 2018-04-21 16:17 +0100 http://bitbucket.org/pypy/pypy/changeset/86478b4c1d71/ Log: (ronan mwjackson) fixed platform tests that utilise sys._git we are not reverting the entire migration to _git in the sys module (#27593), just patching the tests so they pass diff --git a/lib-python/3/test/test_platform.py b/lib-python/3/test/test_platform.py --- a/lib-python/3/test/test_platform.py +++ b/lib-python/3/test/test_platform.py @@ -67,12 +67,12 @@ def setUp(self): self.save_version = sys.version - self.save_git = sys._git + self.save_mercurial = sys._mercurial self.save_platform = sys.platform def tearDown(self): sys.version = self.save_version - sys._git = self.save_git + sys._mercurial = self.save_mercurial sys.platform = self.save_platform def test_sys_version(self): @@ -149,10 +149,10 @@ sys_versions.items(): sys.version = version_tag if subversion is None: - if hasattr(sys, "_git"): - del sys._git + if hasattr(sys, "_mercurial"): + del sys._mercurial else: - sys._git = subversion + sys._mercurial = subversion if sys_platform is not None: sys.platform = sys_platform self.assertEqual(platform.python_implementation(), info[0]) From pypy.commits at gmail.com Sat Apr 21 20:18:04 2018 From: pypy.commits at gmail.com (tdziopa) Date: Sat, 21 Apr 2018 17:18:04 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: (tomek.dziopa ronan) add deprecationwarning for incorrectly escaped strings Message-ID: <5adbd4bc.1c69fb81.36a46.300c@mx.google.com> Author: Tomasz Dziopa Branch: py3.6 Changeset: r94408:bce4e51f4d12 Date: 2018-04-21 22:48 +0100 http://bitbucket.org/pypy/pypy/changeset/bce4e51f4d12/ Log: (tomek.dziopa ronan) add deprecationwarning for incorrectly escaped strings https://hg.python.org/cpython/rev/ee82266ad35b added verbose warnings on incorrectly escaped string literals. This PR unifies pypy behaviour in this matter. diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py --- a/pypy/interpreter/pyparser/parsestring.py +++ b/pypy/interpreter/pyparser/parsestring.py @@ -114,7 +114,13 @@ v = unicodehelper.decode_utf8(space, substr) return space.newunicode(v) - v = PyString_DecodeEscape(space, substr, 'strict', encoding) + v, first_escape_error_position = PyString_DecodeEscape( + space, substr, 'strict', encoding) + + if first_escape_error_position is not None: + space.warn("invalid excape sequence '\\%c'" % ch, + space.w_DeprecationWarning) + return space.newbytes(v) def decode_unicode_utf8(space, s, ps, q): @@ -158,6 +164,7 @@ builder = StringBuilder(len(s)) ps = 0 end = len(s) + first_escape_error_position = None while ps < end: if s[ps] != '\\': # note that the C code has a label here. @@ -237,11 +244,13 @@ builder.append('\\') ps -= 1 assert ps >= 0 + if first_escape_error_position is None: + first_escape_error_position = ps continue # an arbitry number of unescaped UTF-8 bytes may follow. buf = builder.build() - return buf + return buf, first_escape_error_position def isxdigit(ch): diff --git a/pypy/module/__builtin__/test/test_compile.py b/pypy/module/__builtin__/test/test_compile.py --- a/pypy/module/__builtin__/test/test_compile.py +++ b/pypy/module/__builtin__/test/test_compile.py @@ -39,6 +39,13 @@ except SyntaxError as e: assert e.lineno == 1 + def test_incorrect_escape_deprecation(self): + import warnings + with warnings.catch_warnings(record=True) as l: + warnings.simplefilter('always', category=DeprecationWarning) + compile(r"'\%c'" % 125, '', 'exec') + assert l == None + def test_unicode_encoding(self): code = "# -*- coding: utf-8 -*-\npass\n" compile(code, "tmp", "exec") diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -1027,5 +1027,6 @@ def escape_decode(space, w_data, errors='strict'): data = space.getarg_w('s#', w_data) from pypy.interpreter.pyparser.parsestring import PyString_DecodeEscape - result = PyString_DecodeEscape(space, data, errors, None) + result, _ = PyString_DecodeEscape(space, data, errors, None) + return space.newtuple([space.newbytes(result), space.newint(len(data))]) From pypy.commits at gmail.com Sun Apr 22 07:08:42 2018 From: pypy.commits at gmail.com (tdziopa) Date: Sun, 22 Apr 2018 04:08:42 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: (tdziopa) fix warnings for PyString_DecodeEscape Message-ID: <5adc6d3a.1c69fb81.975da.72fa@mx.google.com> Author: Tomasz Dziopa Branch: py3.6 Changeset: r94409:d3ffc4376fe9 Date: 2018-04-22 09:41 +0100 http://bitbucket.org/pypy/pypy/changeset/d3ffc4376fe9/ Log: (tdziopa) fix warnings for PyString_DecodeEscape Recent PR introduced a bug with undefined variable ch being accessed when raising DeprecationWarning on errorneous escaping. This PR fixes that issue and changes the names of variables to better reflect their contents (first_escape_error_position -> first_escape_error_char) diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py --- a/pypy/interpreter/pyparser/parsestring.py +++ b/pypy/interpreter/pyparser/parsestring.py @@ -114,11 +114,11 @@ v = unicodehelper.decode_utf8(space, substr) return space.newunicode(v) - v, first_escape_error_position = PyString_DecodeEscape( + v, first_escape_error_char = PyString_DecodeEscape( space, substr, 'strict', encoding) - if first_escape_error_position is not None: - space.warn("invalid excape sequence '\\%c'" % ch, + if first_escape_error_char is not None: + space.warn("invalid excape sequence '\\%c'" % first_escape_error_char, space.w_DeprecationWarning) return space.newbytes(v) @@ -164,7 +164,7 @@ builder = StringBuilder(len(s)) ps = 0 end = len(s) - first_escape_error_position = None + first_escape_error_char = None while ps < end: if s[ps] != '\\': # note that the C code has a label here. @@ -244,13 +244,13 @@ builder.append('\\') ps -= 1 assert ps >= 0 - if first_escape_error_position is None: - first_escape_error_position = ps + if first_escape_error_char is None: + first_escape_error_char = ch continue # an arbitry number of unescaped UTF-8 bytes may follow. buf = builder.build() - return buf, first_escape_error_position + return buf, first_escape_error_char def isxdigit(ch): From pypy.commits at gmail.com Sun Apr 22 07:08:44 2018 From: pypy.commits at gmail.com (tdziopa) Date: Sun, 22 Apr 2018 04:08:44 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: (tdziopa) fix default value for first_escape_error_char Message-ID: <5adc6d3c.0e941c0a.69d4f.1709@mx.google.com> Author: Tomasz Dziopa Branch: py3.6 Changeset: r94410:bc678f7e35f0 Date: 2018-04-22 11:23 +0100 http://bitbucket.org/pypy/pypy/changeset/bc678f7e35f0/ Log: (tdziopa) fix default value for first_escape_error_char As RPython doesn't allow having int and None in the same variable, the initial value of first_escape_error_char should be fixed to -1. diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py --- a/pypy/interpreter/pyparser/parsestring.py +++ b/pypy/interpreter/pyparser/parsestring.py @@ -117,7 +117,7 @@ v, first_escape_error_char = PyString_DecodeEscape( space, substr, 'strict', encoding) - if first_escape_error_char is not None: + if first_escape_error_char != -1: space.warn("invalid excape sequence '\\%c'" % first_escape_error_char, space.w_DeprecationWarning) @@ -164,7 +164,7 @@ builder = StringBuilder(len(s)) ps = 0 end = len(s) - first_escape_error_char = None + first_escape_error_char = -1 while ps < end: if s[ps] != '\\': # note that the C code has a label here. @@ -244,7 +244,7 @@ builder.append('\\') ps -= 1 assert ps >= 0 - if first_escape_error_char is None: + if first_escape_error_char == -1: first_escape_error_char = ch continue # an arbitry number of unescaped UTF-8 bytes may follow. From pypy.commits at gmail.com Sun Apr 22 16:58:40 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 22 Apr 2018 13:58:40 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: replace 7ac5e33a8260, applevel datetime inherit from a cpyext-friendly class Message-ID: <5adcf780.1c69fb81.a2536.544b@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r94411:42681276e562 Date: 2018-04-22 23:57 +0300 http://bitbucket.org/pypy/pypy/changeset/42681276e562/ Log: replace 7ac5e33a8260, applevel datetime inherit from a cpyext- friendly class diff --git a/lib-python/3/datetime.py b/lib-python/3/datetime.py --- a/lib-python/3/datetime.py +++ b/lib-python/3/datetime.py @@ -7,6 +7,9 @@ import time as _time import math as _math +# for cpyext, use these as base classes +from __pypy__._pypydatetime import dateinterop, deltainterop, timeinterop + def _cmp(x, y): return 0 if x == y else 1 if x > y else -1 @@ -316,7 +319,7 @@ return q -class timedelta: +class timedelta(deltainterop): """Represent the difference between two datetime objects. Supported operators: @@ -429,7 +432,7 @@ if abs(d) > 999999999: raise OverflowError("timedelta # of days is too large: %d" % d) - self = object.__new__(cls) + self = deltainterop.__new__(cls) self._days = d self._seconds = s self._microseconds = us @@ -638,7 +641,7 @@ microseconds=999999) timedelta.resolution = timedelta(microseconds=1) -class date: +class date(dateinterop): """Concrete date type. Constructors: @@ -678,12 +681,12 @@ if month is None and isinstance(year, bytes) and len(year) == 4 and \ 1 <= year[2] <= 12: # Pickle support - self = object.__new__(cls) + self = dateinterop.__new__(cls) self.__setstate(year) self._hashcode = -1 return self year, month, day = _check_date_fields(year, month, day) - self = object.__new__(cls) + self = dateinterop.__new__(cls) self._year = year self._month = month self._day = day @@ -1008,7 +1011,7 @@ _tzinfo_class = tzinfo -class time: +class time(timeinterop): """Time with time zone. Constructors: @@ -1044,14 +1047,14 @@ """ if isinstance(hour, bytes) and len(hour) == 6 and hour[0] < 24: # Pickle support - self = object.__new__(cls) + self = timeinterop.__new__(cls) self.__setstate(hour, minute or None) self._hashcode = -1 return self hour, minute, second, microsecond = _check_time_fields( hour, minute, second, microsecond) _check_tzinfo_arg(tzinfo) - self = object.__new__(cls) + self = timeinterop.__new__(cls) self._hour = hour self._minute = minute self._second = second @@ -1329,7 +1332,7 @@ microsecond=0, tzinfo=None): if isinstance(year, bytes) and len(year) == 10 and 1 <= year[2] <= 12: # Pickle support - self = object.__new__(cls) + self = dateinterop.__new__(cls) self.__setstate(year, month) self._hashcode = -1 return self @@ -1337,7 +1340,7 @@ hour, minute, second, microsecond = _check_time_fields( hour, minute, second, microsecond) _check_tzinfo_arg(tzinfo) - self = object.__new__(cls) + self = dateinterop.__new__(cls) self._year = year self._month = month self._day = day diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py --- a/pypy/module/cpyext/cdatetime.py +++ b/pypy/module/cpyext/cdatetime.py @@ -40,26 +40,18 @@ w_type = space.getattr(w_datetime, space.newtext("datetime")) datetimeAPI.c_DateTimeType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) - datetimeAPI.c_DateTimeType.c_tp_basicsize = rffi.sizeof( - cts.gettype('PyDateTime_DateTime')) w_type = space.getattr(w_datetime, space.newtext("time")) datetimeAPI.c_TimeType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) - datetimeAPI.c_TimeType.c_tp_basicsize = rffi.sizeof( - cts.gettype('PyDateTime_Time')) w_type = space.getattr(w_datetime, space.newtext("timedelta")) datetimeAPI.c_DeltaType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) - datetimeAPI.c_DeltaType.c_tp_basicsize = rffi.sizeof( - cts.gettype('PyDateTime_Delta')) w_type = space.getattr(w_datetime, space.newtext("tzinfo")) datetimeAPI.c_TZInfoType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) - datetimeAPI.c_TZInfoType.c_tp_basicsize = rffi.sizeof( - cts.gettype('PyDateTime_TZInfo')) datetimeAPI.c_Date_FromDate = llhelper( _PyDate_FromDate.api_func.functype, From pypy.commits at gmail.com Sun Apr 22 17:26:38 2018 From: pypy.commits at gmail.com (amauryfa) Date: Sun, 22 Apr 2018 14:26:38 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: Add support for PROTOCOL_TLS_CLIENT and PROTOCOL_TLS_SERVER Message-ID: <5adcfe0e.1c69fb81.41e1b.214a@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.6 Changeset: r94412:e82fec399c34 Date: 2018-04-22 19:37 +0200 http://bitbucket.org/pypy/pypy/changeset/e82fec399c34/ Log: Add support for PROTOCOL_TLS_CLIENT and PROTOCOL_TLS_SERVER (CPython Issue #28085) diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py --- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py +++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py @@ -83,6 +83,8 @@ PROTOCOL_TLSv1 = 3 PROTOCOL_TLSv1_1 = 4 PROTOCOL_TLSv1_2 = 5 +PROTOCOL_TLS_CLIENT = 0x10 +PROTOCOL_TLS_SERVER = 0x11 _PROTOCOL_NAMES = (name for name in dir(lib) if name.startswith('PROTOCOL_')) @@ -746,6 +748,10 @@ method = lib.SSLv2_method() elif protocol == PROTOCOL_SSLv23: method = lib.SSLv23_method() + elif protocol == PROTOCOL_TLS_CLIENT: + method = lib.SSLv23_client_method() + elif protocol == PROTOCOL_TLS_SERVER: + method = lib.SSLv23_server_method() else: raise ValueError("invalid protocol version") From pypy.commits at gmail.com Sun Apr 22 17:26:44 2018 From: pypy.commits at gmail.com (amauryfa) Date: Sun, 22 Apr 2018 14:26:44 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: Fix default socket options, added by CPython Issue 28043. Message-ID: <5adcfe14.4c3e1c0a.add5c.80ed@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.6 Changeset: r94415:26bff7c256af Date: 2018-04-22 21:04 +0200 http://bitbucket.org/pypy/pypy/changeset/26bff7c256af/ Log: Fix default socket options, added by CPython Issue 28043. diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py --- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py +++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py @@ -64,7 +64,9 @@ for name in dir(lib): if name.startswith('SSL_OP'): - globals()[name[4:]] = getattr(lib, name) + value = getattr(lib, name) + if value != 0: + globals()[name[4:]] = getattr(lib, name) OP_ALL = lib.SSL_OP_ALL & ~lib.SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS @@ -829,6 +831,12 @@ options |= lib.SSL_OP_NO_SSLv2 if protocol != PROTOCOL_SSLv3: options |= lib.SSL_OP_NO_SSLv3 + # Minimal security flags for server and client side context. + # Client sockets ignore server-side parameters. + options |= lib.SSL_OP_NO_COMPRESSION; + options |= lib.SSL_OP_CIPHER_SERVER_PREFERENCE; + options |= lib.SSL_OP_SINGLE_DH_USE; + options |= lib.SSL_OP_SINGLE_ECDH_USE; lib.SSL_CTX_set_options(self.ctx, options) lib.SSL_CTX_set_session_id_context(self.ctx, b"Python", len(b"Python")) From pypy.commits at gmail.com Sun Apr 22 17:26:40 2018 From: pypy.commits at gmail.com (amauryfa) Date: Sun, 22 Apr 2018 14:26:40 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: Return a dummy .session to let more SSL tests pass. Message-ID: <5adcfe10.1c69fb81.d4a11.bb71@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.6 Changeset: r94413:2271eaab22b2 Date: 2018-04-22 19:42 +0200 http://bitbucket.org/pypy/pypy/changeset/2271eaab22b2/ Log: Return a dummy .session to let more SSL tests pass. diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py --- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py +++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py @@ -690,6 +690,16 @@ return None return _str_with_len(out[0], outlen[0]) + @property + def session(self): + "Get / set SSLSession." + return None + + @property + def session_reused(self): + "Was the client session reused during handshake?" + return bool(lib.SSL_session_reused(self.ssl)) + def _fs_decode(name): return name.decode(sys.getfilesystemencoding()) From pypy.commits at gmail.com Sun Apr 22 17:26:42 2018 From: pypy.commits at gmail.com (amauryfa) Date: Sun, 22 Apr 2018 14:26:42 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: Implement SSLSession. Message-ID: <5adcfe12.c21b1c0a.7d2e4.0650@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.6 Changeset: r94414:6dc2dbf20397 Date: 2018-04-22 20:41 +0200 http://bitbucket.org/pypy/pypy/changeset/6dc2dbf20397/ Log: Implement SSLSession. diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py --- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py +++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py @@ -198,6 +198,8 @@ const char *SSL_get_cipher_list(const SSL *, int); Cryptography_STACK_OF_SSL_CIPHER *SSL_get_ciphers(const SSL *); +int SSL_is_init_finished(const SSL*); + /* context */ void SSL_CTX_free(SSL_CTX *); long SSL_CTX_set_timeout(SSL_CTX *, long); @@ -265,6 +267,10 @@ SSL_SESSION *SSL_get_session(const SSL *); const unsigned char *SSL_SESSION_get_id(const SSL_SESSION *, unsigned int *); +long SSL_SESSION_get_time(const SSL_SESSION *); +long SSL_SESSION_get_timeout(const SSL_SESSION *); +int SSL_SESSION_has_ticket(const SSL_SESSION *); +long SSL_SESSION_get_ticket_lifetime_hint(const SSL_SESSION *); /* not a macro, but older OpenSSLs don't pass the args as const */ char *SSL_CIPHER_description(const SSL_CIPHER *, char *, int); @@ -509,6 +515,14 @@ memcpy(out, session->master_key, outlen); return outlen; } + +int SSL_SESSION_has_ticket(const SSL_SESSION *s) { + return (s->tlsext_ticklen > 0) ? 1 : 0; +} +unsigned long SSL_SESSION_get_ticket_lifetime_hint(const SSL_SESSION *s) +{ + return s->tlsext_tick_lifetime_hint; +} #endif static const long Cryptography_HAS_SECURE_RENEGOTIATION = 1; diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py --- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py +++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py @@ -693,7 +693,20 @@ @property def session(self): "Get / set SSLSession." - return None + return Session(self) + + @session.setter + def session(self, value): + if not isinstance(value, Session): + raise TypeError("Value is not a SSLSession.") + if self.ctx.ctx != value._ctx.ctx: + raise ValueError("Session refers to a different SSLContext.") + if self.socket_type != SSL_CLIENT: + raise ValueError("Cannot set session for server-side SSLSocket.") + if lib.SSL_is_init_finished(self.ssl): + raise ValueError("Cannot set session after handshake.") + if not lib.SSL_set_session(self.ssl, value._session): + raise pyssl_error(self, 0) @property def session_reused(self): @@ -727,6 +740,43 @@ return (cipher_name, cipher_protocol, bits) +class Session(object): + def __new__(cls, ssl): + self = object.__new__(cls) + session = lib.SSL_get1_session(ssl.ssl) + if not session: + return None + self._session = ffi.gc(session, lib.SSL_SESSION_free) + self._ctx = ssl.ctx + return self + + def __eq__(self, other): + if not isinstance(other, Session): + return NotImplemented; + return self.id == other.id + + @property + def id(self): + lenp = ffi.new("unsigned int*") + id = lib.SSL_SESSION_get_id(self._session, lenp) + return ffi.string(id, lenp[0]) + + @property + def time(self): + return lib.SSL_SESSION_get_time(self._session) + + @property + def timeout(self): + return lib.SSL_SESSION_get_timeout(self._session) + + @property + def has_ticket(self): + return bool(lib.SSL_SESSION_has_ticket(self._session)) + + @property + def ticket_lifetime_hint(self): + return lib.SSL_SESSION_get_ticket_lifetime_hint(self._session) + SSL_CTX_STATS_NAMES = """ number connect connect_good connect_renegotiate accept accept_good From pypy.commits at gmail.com Sun Apr 22 17:26:50 2018 From: pypy.commits at gmail.com (amauryfa) Date: Sun, 22 Apr 2018 14:26:50 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: Is it only on my machine? 10% of the time, the test fails because the error is: Message-ID: <5adcfe1a.0b371c0a.12306.f8c3@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.6 Changeset: r94418:2360f76c2d3c Date: 2018-04-22 21:39 +0200 http://bitbucket.org/pypy/pypy/changeset/2360f76c2d3c/ Log: Is it only on my machine? 10% of the time, the test fails because the error is: '[SSL: NO_SHARED_CIPHER] error:14077410:SSL routines:SSL23_GET_SERVER_HELLO:sslv3 alert handshake failure' diff --git a/lib-python/3/test/test_ssl.py b/lib-python/3/test/test_ssl.py --- a/lib-python/3/test/test_ssl.py +++ b/lib-python/3/test/test_ssl.py @@ -3087,7 +3087,7 @@ with context.wrap_socket(socket.socket()) as s: with self.assertRaises(OSError): s.connect((HOST, server.port)) - self.assertIn("no shared cipher", str(server.conn_errors[0])) + self.assertEqual("NO_SHARED_CIPHER", server.conn_errors[0].reason) def test_version_basic(self): """ From pypy.commits at gmail.com Sun Apr 22 17:26:46 2018 From: pypy.commits at gmail.com (amauryfa) Date: Sun, 22 Apr 2018 14:26:46 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: Apply the rest of CPython Issue 28043. This part is not well tested... Message-ID: <5adcfe16.530b1c0a.aa7d2.7261@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.6 Changeset: r94416:fc92860d2c30 Date: 2018-04-22 21:11 +0200 http://bitbucket.org/pypy/pypy/changeset/fc92860d2c30/ Log: Apply the rest of CPython Issue 28043. This part is not well tested... diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py --- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py +++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py @@ -833,13 +833,24 @@ options |= lib.SSL_OP_NO_SSLv3 # Minimal security flags for server and client side context. # Client sockets ignore server-side parameters. - options |= lib.SSL_OP_NO_COMPRESSION; - options |= lib.SSL_OP_CIPHER_SERVER_PREFERENCE; - options |= lib.SSL_OP_SINGLE_DH_USE; - options |= lib.SSL_OP_SINGLE_ECDH_USE; + options |= lib.SSL_OP_NO_COMPRESSION + options |= lib.SSL_OP_CIPHER_SERVER_PREFERENCE + options |= lib.SSL_OP_SINGLE_DH_USE + options |= lib.SSL_OP_SINGLE_ECDH_USE lib.SSL_CTX_set_options(self.ctx, options) lib.SSL_CTX_set_session_id_context(self.ctx, b"Python", len(b"Python")) + # A bare minimum cipher list without completely broken cipher suites. + # It's far from perfect but gives users a better head start. + if lib.Cryptography_HAS_SSL2 and protocol == PROTOCOL_SSLv2: + # SSLv2 needs MD5 + default_ciphers = b"HIGH:!aNULL:!eNULL" + else: + default_ciphers = b"HIGH:!aNULL:!eNULL:!MD5" + if not lib.SSL_CTX_set_cipher_list(ctx, default_ciphers): + lib.ERR_clear_error() + raise SSLError("No cipher can be selected.") + if HAS_ECDH: # Allow automatic ECDH curve selection (on # OpenSSL 1.0.2+), or use prime256v1 by default. From pypy.commits at gmail.com Sun Apr 22 17:26:52 2018 From: pypy.commits at gmail.com (amauryfa) Date: Sun, 22 Apr 2018 14:26:52 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: The session.id is a bytes string with NULs, use ffi.unpack() instead of ffi.string() Message-ID: <5adcfe1c.1c69fb81.d4d4d.b5ee@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.6 Changeset: r94419:3f675393a946 Date: 2018-04-22 21:40 +0200 http://bitbucket.org/pypy/pypy/changeset/3f675393a946/ Log: The session.id is a bytes string with NULs, use ffi.unpack() instead of ffi.string() diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py --- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py +++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py @@ -761,7 +761,7 @@ def id(self): lenp = ffi.new("unsigned int*") id = lib.SSL_SESSION_get_id(self._session, lenp) - return ffi.string(id, lenp[0]) + return ffi.unpack(id, lenp[0]) @property def time(self): From pypy.commits at gmail.com Sun Apr 22 17:26:48 2018 From: pypy.commits at gmail.com (amauryfa) Date: Sun, 22 Apr 2018 14:26:48 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: Complete the implementation of PROTOCOL_TLS_CLIENT: host names are checked. Message-ID: <5adcfe18.1c69fb81.f0c89.8116@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.6 Changeset: r94417:869382060075 Date: 2018-04-22 21:31 +0200 http://bitbucket.org/pypy/pypy/changeset/869382060075/ Log: Complete the implementation of PROTOCOL_TLS_CLIENT: host names are checked. diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py --- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py +++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py @@ -822,10 +822,16 @@ raise ssl_error("failed to allocate SSL context") self.ctx = ffi.gc(lib.SSL_CTX_new(method), lib.SSL_CTX_free) + # Don't check host name by default self._check_hostname = False + if protocol == PROTOCOL_TLS_CLIENT: + self._check_hostname = True + self.verify_mode = CERT_REQUIRED + else: + self._check_hostname = False + self.verify_mode = CERT_NONE # Defaults - lib.SSL_CTX_set_verify(self.ctx, lib.SSL_VERIFY_NONE, ffi.NULL) options = lib.SSL_OP_ALL & ~lib.SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS if not lib.Cryptography_HAS_SSL2 or protocol != PROTOCOL_SSLv2: options |= lib.SSL_OP_NO_SSLv2 @@ -910,7 +916,9 @@ if mode == lib.SSL_VERIFY_NONE and self.check_hostname: raise ValueError("Cannot set verify_mode to CERT_NONE when " \ "check_hostname is enabled.") - lib.SSL_CTX_set_verify(self.ctx, mode, ffi.NULL); + # Keep current verify cb + verify_cb = lib.SSL_CTX_get_verify_callback(self.ctx) + lib.SSL_CTX_set_verify(self.ctx, mode, verify_cb) @property def verify_flags(self): From pypy.commits at gmail.com Sun Apr 22 18:02:47 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 22 Apr 2018 15:02:47 -0700 (PDT) Subject: [pypy-commit] pypy issue2806_tp_init: issue 2806 - object.__init__ is called not CALL.__init__ Message-ID: <5add0687.1c69fb81.c7d7c.c3cd@mx.google.com> Author: Matti Picus Branch: issue2806_tp_init Changeset: r94420:de6e457c5576 Date: 2018-04-23 01:01 +0300 http://bitbucket.org/pypy/pypy/changeset/de6e457c5576/ Log: issue 2806 - object.__init__ is called not CALL.__init__ diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -833,6 +833,14 @@ x = LL.__new__(LL) assert module.tp_init(list, x, ("hi",)) is None assert x == ["h", "i"] + init_called = [] + class CALL(object): + def __init__(self): + init_called.append(42) + x = object.__new__(CALL) + x.__init__() + module.tp_init(CALL, x, ()) + assert len(init_called) == 2 def test_mp_subscript(self): module = self.import_extension('foo', [ From pypy.commits at gmail.com Sun Apr 22 18:39:22 2018 From: pypy.commits at gmail.com (amauryfa) Date: Sun, 22 Apr 2018 15:39:22 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: We don't need SSL_is_init_finished after all Message-ID: <5add0f1a.1c69fb81.12801.a3a9@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.6 Changeset: r94421:a29a0980fd16 Date: 2018-04-23 00:34 +0200 http://bitbucket.org/pypy/pypy/changeset/a29a0980fd16/ Log: We don't need SSL_is_init_finished after all diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py --- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py +++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py @@ -198,8 +198,6 @@ const char *SSL_get_cipher_list(const SSL *, int); Cryptography_STACK_OF_SSL_CIPHER *SSL_get_ciphers(const SSL *); -int SSL_is_init_finished(const SSL*); - /* context */ void SSL_CTX_free(SSL_CTX *); long SSL_CTX_set_timeout(SSL_CTX *, long); diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py --- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py +++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py @@ -705,7 +705,7 @@ raise ValueError("Session refers to a different SSLContext.") if self.socket_type != SSL_CLIENT: raise ValueError("Cannot set session for server-side SSLSocket.") - if lib.SSL_is_init_finished(self.ssl): + if self.handshake_done: raise ValueError("Cannot set session after handshake.") if not lib.SSL_set_session(self.ssl, value._session): raise pyssl_error(self, 0) From pypy.commits at gmail.com Sun Apr 22 18:39:24 2018 From: pypy.commits at gmail.com (amauryfa) Date: Sun, 22 Apr 2018 15:39:24 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: The class is named ssl.SSLSession. Restore the import in ssl.py Message-ID: <5add0f1c.1c69fb81.5be47.5e30@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.6 Changeset: r94422:85c682c5ec33 Date: 2018-04-23 00:38 +0200 http://bitbucket.org/pypy/pypy/changeset/85c682c5ec33/ Log: The class is named ssl.SSLSession. Restore the import in ssl.py diff --git a/lib-python/3/ssl.py b/lib-python/3/ssl.py --- a/lib-python/3/ssl.py +++ b/lib-python/3/ssl.py @@ -100,7 +100,7 @@ import _ssl # if we can't import it, let the error propagate from _ssl import OPENSSL_VERSION_NUMBER, OPENSSL_VERSION_INFO, OPENSSL_VERSION -from _ssl import _SSLContext, MemoryBIO#, SSLSession # XXX: PyPy hack +from _ssl import _SSLContext, MemoryBIO, SSLSession from _ssl import ( SSLError, SSLZeroReturnError, SSLWantReadError, SSLWantWriteError, SSLSyscallError, SSLEOFError, diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py --- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py +++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py @@ -695,11 +695,11 @@ @property def session(self): "Get / set SSLSession." - return Session(self) + return SSLSession(self) @session.setter def session(self, value): - if not isinstance(value, Session): + if not isinstance(value, SSLSession): raise TypeError("Value is not a SSLSession.") if self.ctx.ctx != value._ctx.ctx: raise ValueError("Session refers to a different SSLContext.") @@ -742,7 +742,7 @@ return (cipher_name, cipher_protocol, bits) -class Session(object): +class SSLSession(object): def __new__(cls, ssl): self = object.__new__(cls) session = lib.SSL_get1_session(ssl.ssl) @@ -753,7 +753,7 @@ return self def __eq__(self, other): - if not isinstance(other, Session): + if not isinstance(other, SSLSession): return NotImplemented; return self.id == other.id From pypy.commits at gmail.com Sun Apr 22 20:30:05 2018 From: pypy.commits at gmail.com (tdziopa) Date: Sun, 22 Apr 2018 17:30:05 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: Fix default values for first_escape_error_char Message-ID: <5add290d.1c69fb81.5be47.9b31@mx.google.com> Author: Tomasz Dziopa Branch: py3.6 Changeset: r94423:cd4db20fa471 Date: 2018-04-22 23:41 +0100 http://bitbucket.org/pypy/pypy/changeset/cd4db20fa471/ Log: Fix default values for first_escape_error_char This PR fixes the RPython annotation errors for PyString_DecodeEscape. diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py --- a/pypy/interpreter/pyparser/parsestring.py +++ b/pypy/interpreter/pyparser/parsestring.py @@ -117,8 +117,10 @@ v, first_escape_error_char = PyString_DecodeEscape( space, substr, 'strict', encoding) - if first_escape_error_char != -1: - space.warn("invalid excape sequence '\\%c'" % first_escape_error_char, + if first_escape_error_char != '': + space.warn( + space.newtext("invalid excape sequence '\\%c'" + % first_escape_error_char), space.w_DeprecationWarning) return space.newbytes(v) @@ -164,7 +166,7 @@ builder = StringBuilder(len(s)) ps = 0 end = len(s) - first_escape_error_char = -1 + first_escape_error_char = '' while ps < end: if s[ps] != '\\': # note that the C code has a label here. @@ -244,7 +246,7 @@ builder.append('\\') ps -= 1 assert ps >= 0 - if first_escape_error_char == -1: + if first_escape_error_char == '': first_escape_error_char = ch continue # an arbitry number of unescaped UTF-8 bytes may follow. From pypy.commits at gmail.com Sun Apr 22 20:34:30 2018 From: pypy.commits at gmail.com (rlamy) Date: Sun, 22 Apr 2018 17:34:30 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: fix test and a typo Message-ID: <5add2a16.1c69fb81.a2536.9e66@mx.google.com> Author: Ronan Lamy Branch: py3.6 Changeset: r94424:0bfc5e908da5 Date: 2018-04-23 01:33 +0100 http://bitbucket.org/pypy/pypy/changeset/0bfc5e908da5/ Log: fix test and a typo diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py --- a/pypy/interpreter/pyparser/parsestring.py +++ b/pypy/interpreter/pyparser/parsestring.py @@ -119,7 +119,7 @@ if first_escape_error_char != '': space.warn( - space.newtext("invalid excape sequence '\\%c'" + space.newtext("invalid escape sequence '\\%c'" % first_escape_error_char), space.w_DeprecationWarning) diff --git a/pypy/module/__builtin__/test/test_compile.py b/pypy/module/__builtin__/test/test_compile.py --- a/pypy/module/__builtin__/test/test_compile.py +++ b/pypy/module/__builtin__/test/test_compile.py @@ -39,12 +39,12 @@ except SyntaxError as e: assert e.lineno == 1 - def test_incorrect_escape_deprecation(self): + def test_incorrect_escape_deprecation_bytes(self): import warnings with warnings.catch_warnings(record=True) as l: warnings.simplefilter('always', category=DeprecationWarning) - compile(r"'\%c'" % 125, '', 'exec') - assert l == None + compile(r"b'\}'", '', 'exec') + assert len(l) == 1 def test_unicode_encoding(self): code = "# -*- coding: utf-8 -*-\npass\n" From pypy.commits at gmail.com Mon Apr 23 08:39:33 2018 From: pypy.commits at gmail.com (arigo) Date: Mon, 23 Apr 2018 05:39:33 -0700 (PDT) Subject: [pypy-commit] pypy default: Fix a case where the list strategy leaks to the user Message-ID: <5addd405.54d91c0a.1ad69.a24b@mx.google.com> Author: Armin Rigo Branch: Changeset: r94425:bae7d61d060c Date: 2018-04-23 14:38 +0200 http://bitbucket.org/pypy/pypy/changeset/bae7d61d060c/ Log: Fix a case where the list strategy leaks to the user diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py --- a/pypy/objspace/std/listobject.py +++ b/pypy/objspace/std/listobject.py @@ -991,6 +991,14 @@ def setslice(self, w_list, start, step, slicelength, w_other): strategy = w_other.strategy + if step != 1: + len2 = strategy.length(w_other) + if len2 == 0: + return + else: + raise oefmt(self.space.w_ValueError, + "attempt to assign sequence of size %d to extended " + "slice of size %d", len2, 0) storage = strategy.getstorage_copy(w_other) w_list.strategy = strategy w_list.lstorage = storage diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py --- a/pypy/objspace/std/test/test_listobject.py +++ b/pypy/objspace/std/test/test_listobject.py @@ -1080,6 +1080,15 @@ l[::3] = ('a', 'b') assert l == ['a', 1.1, 2.2, 'b', 4.4, 5.5] + l_int = [5]; l_int.pop() # IntListStrategy + l_empty = [] # EmptyListStrategy + raises(ValueError, "l_int[::-1] = [42]") + raises(ValueError, "l_int[::7] = [42]") + raises(ValueError, "l_empty[::-1] = [42]") + raises(ValueError, "l_empty[::7] = [42]") + l_int[::1] = [42]; assert l_int == [42] + l_empty[::1] = [42]; assert l_empty == [42] + def test_setslice_with_self(self): l = [1,2,3,4] l[:] = l From pypy.commits at gmail.com Mon Apr 23 09:04:57 2018 From: pypy.commits at gmail.com (arigo) Date: Mon, 23 Apr 2018 06:04:57 -0700 (PDT) Subject: [pypy-commit] pypy default: Issue #2805 Attempted fix Message-ID: <5addd9f9.1c69fb81.c1b88.6b2a@mx.google.com> Author: Armin Rigo Branch: Changeset: r94426:321aff47be74 Date: 2018-04-23 15:04 +0200 http://bitbucket.org/pypy/pypy/changeset/321aff47be74/ Log: Issue #2805 Attempted fix diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -491,11 +491,17 @@ # 'action.fire()' happens to be called any time before # the corresponding perform(), the fire() has no # effect---which is the effect we want, because - # perform() will be called anyway. + # perform() will be called anyway. All such pending + # actions with _fired == True are still inside the old + # chained list. As soon as we reset _fired to False, + # we also reset _next to None and we are ready for + # another fire(). while action is not None: + next_action = action._next + action._next = None action._fired = False action.perform(ec, frame) - action._next, action = None, action._next + action = next_action self.action_dispatcher = action_dispatcher From pypy.commits at gmail.com Mon Apr 23 09:53:04 2018 From: pypy.commits at gmail.com (antocuni) Date: Mon, 23 Apr 2018 06:53:04 -0700 (PDT) Subject: [pypy-commit] pypy default: add a test for 321aff47be74 Message-ID: <5adde540.1c69fb81.fc257.6c8d@mx.google.com> Author: Antonio Cuni Branch: Changeset: r94427:05d4bb2624e8 Date: 2018-04-23 15:52 +0200 http://bitbucket.org/pypy/pypy/changeset/05d4bb2624e8/ Log: add a test for 321aff47be74 diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py --- a/pypy/interpreter/test/test_executioncontext.py +++ b/pypy/interpreter/test/test_executioncontext.py @@ -67,6 +67,47 @@ """) assert events == ['one'] + def test_fire_inside_perform(self): + # test what happens if we call AsyncAction.fire() while we are in the + # middle of an AsyncAction.perform(). In particular, this happens when + # PyObjectDeallocAction.fire() is called by rawrefcount: see issue + # 2805 + events = [] + + class Action1(executioncontext.AsyncAction): + _count = 0 + + def perform(self, ec, frame): + events.append('one') + if self._count == 0: + # a1 is no longer in the queue, so it will be enqueued + a1.fire() + # + # a2 is still in the queue, so the fire() is ignored and + # it's performed in its normal order, i.e. BEFORE a3 + a2.fire() + self._count += 1 + + class Action2(executioncontext.AsyncAction): + def perform(self, ec, frame): + events.append('two') + + class Action3(executioncontext.AsyncAction): + def perform(self, ec, frame): + events.append('three') + + space = self.space + a1 = Action1(space) + a2 = Action2(space) + a3 = Action3(space) + a1.fire() + a2.fire() + a3.fire() + space.appexec([], """(): + pass + """) + assert events == ['one', 'two', 'three', 'one'] + def test_periodic_action(self): from pypy.interpreter.executioncontext import ActionFlag From pypy.commits at gmail.com Mon Apr 23 12:15:22 2018 From: pypy.commits at gmail.com (arigo) Date: Mon, 23 Apr 2018 09:15:22 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Implement the win32 function ssl.enum_certificates() Message-ID: <5ade069a.d6b11c0a.59683.dc1e@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r94428:b129a34163ac Date: 2018-04-23 17:47 +0200 http://bitbucket.org/pypy/pypy/changeset/b129a34163ac/ Log: Implement the win32 function ssl.enum_certificates() diff --git a/lib_pypy/_cffi_ssl/README.md b/lib_pypy/_cffi_ssl/README.md --- a/lib_pypy/_cffi_ssl/README.md +++ b/lib_pypy/_cffi_ssl/README.md @@ -14,6 +14,8 @@ * ``_cffi_src/openssl/x509_vfy.py`` for issue #2605 (ca4d0c90f5a1) +* ``_cffi_src/openssl/pypy_win32_extra.py`` for Win32-only functionality like ssl.enum_certificates() + # Tests? diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/pypy_win32_extra.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/pypy_win32_extra.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/pypy_win32_extra.py @@ -0,0 +1,68 @@ +# +# An extra bit of logic for the Win32-only functionality that is missing from the +# version from cryptography. +# + +import sys + +INCLUDES = """ +#include +""" + +TYPES = """ +typedef ... *HCERTSTORE; +typedef ... *HCRYPTPROV_LEGACY; +typedef struct { + DWORD dwCertEncodingType; + BYTE *pbCertEncoded; + DWORD cbCertEncoded; + ...; +} CERT_CONTEXT, *PCCERT_CONTEXT; +typedef struct { + DWORD cUsageIdentifier; + LPSTR *rgpszUsageIdentifier; + ...; +} CERT_ENHKEY_USAGE, *PCERT_ENHKEY_USAGE; +""" + +FUNCTIONS = """ +HCERTSTORE WINAPI CertOpenStore( + LPCSTR lpszStoreProvider, + DWORD dwMsgAndCertEncodingType, + HCRYPTPROV_LEGACY hCryptProv, + DWORD dwFlags, + const char *pvPara +); +PCCERT_CONTEXT WINAPI CertEnumCertificatesInStore( + HCERTSTORE hCertStore, + PCCERT_CONTEXT pPrevCertContext +); +BOOL WINAPI CertFreeCertificateContext( + PCCERT_CONTEXT pCertContext +); +BOOL WINAPI CertCloseStore( + HCERTSTORE hCertStore, + DWORD dwFlags +); +BOOL WINAPI CertGetEnhancedKeyUsage( + PCCERT_CONTEXT pCertContext, + DWORD dwFlags, + PCERT_ENHKEY_USAGE pUsage, + DWORD *pcbUsage +); +""" + +MACROS = """ +#define CERT_STORE_READONLY_FLAG ... +#define CERT_SYSTEM_STORE_LOCAL_MACHINE ... +#define CRYPT_E_NOT_FOUND ... +#define CERT_FIND_PROP_ONLY_ENHKEY_USAGE_FLAG ... +#define CERT_FIND_EXT_ONLY_ENHKEY_USAGE_FLAG ... +#define X509_ASN_ENCODING ... +#define PKCS_7_ASN_ENCODING ... + +static const LPCSTR CERT_STORE_PROV_SYSTEM_A; +""" + +CUSTOMIZATIONS = """ +""" diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py --- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py +++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py @@ -24,6 +24,7 @@ from enum import IntEnum as _IntEnum if sys.platform == 'win32': + from _cffi_ssl._stdssl.win32_extra import enum_certificates HAVE_POLL = False else: from select import poll, POLLIN, POLLOUT diff --git a/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py b/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py @@ -0,0 +1,66 @@ +from _pypy_openssl import lib, ffi + + +def enum_certificates(store_name): + """Retrieve certificates from Windows' cert store. + +store_name may be one of 'CA', 'ROOT' or 'MY'. The system may provide +more cert storages, too. The function returns a list of (bytes, +encoding_type, trust) tuples. The encoding_type flag can be interpreted +with X509_ASN_ENCODING or PKCS_7_ASN_ENCODING. The trust setting is either +a set of OIDs or the boolean True. + """ + hStore = lib.CertOpenStore(lib.CERT_STORE_PROV_SYSTEM_A, 0, ffi.NULL, + lib.CERT_STORE_READONLY_FLAG | lib.CERT_SYSTEM_STORE_LOCAL_MACHINE, + bytes(store_name, "ascii")) + if hStore == ffi.NULL: + raise WindowsError(*ffi.getwinerror()) + + result = [] + pCertCtx = ffi.NULL + while True: + pCertCtx = lib.CertEnumCertificatesInStore(hStore, pCertCtx) + if pCertCtx == ffi.NULL: + break + cert = ffi.buffer(pCertCtx.pbCertEncoded, pCertCtx.cbCertEncoded)[:] + enc = certEncodingType(pCertCtx.dwCertEncodingType) + keyusage = parseKeyUsage(pCertCtx, lib.CERT_FIND_PROP_ONLY_ENHKEY_USAGE_FLAG) + if keyusage is True: + keyusage = parseKeyUsage(pCertCtx, lib.CERT_FIND_EXT_ONLY_ENHKEY_USAGE_FLAG) + result.append((cert, enc, keyusage)) + + if pCertCtx != ffi.NULL: + lib.CertFreeCertificateContext(pCertCtx) + lib.CertCloseStore(hStore, 0) + return result + + +def certEncodingType(encodingType): + if encodingType == lib.X509_ASN_ENCODING: + return "x509_asn" + if encodingType == lib.PKCS_7_ASN_ENCODING: + return "pkcs_7_asn" + return encodingType + +def parseKeyUsage(pCertCtx, flags): + pSize = ffi.new("DWORD *") + if not lib.CertGetEnhancedKeyUsage(pCertCtx, flags, ffi.NULL, pSize): + error_with_message = ffi.getwinerror() + if error_with_message[0] == lib.CRYPT_E_NOT_FOUND: + return True + raise WindowsError(*error_with_message) + + pUsageMem = ffi.new("char[]", pSize[0]) + pUsage = ffi.cast("PCERT_ENHKEY_USAGE", pUsageMem) + if not lib.CertGetEnhancedKeyUsage(pCertCtx, flags, pUsage, pSize): + error_with_message = ffi.getwinerror() + if error_with_message[0] == lib.CRYPT_E_NOT_FOUND: + return True + raise WindowsError(*error_with_message) + + retval = set() + for i in range(pUsage.cUsageIdentifier): + if pUsage.rgpszUsageIdentifier[i]: + oid = ffi.string(pUsage.rgpszUsageIdentifier[i]).decode('ascii') + retval.add(oid) + return retval diff --git a/lib_pypy/_ssl/__init__.py b/lib_pypy/_ssl/__init__.py --- a/lib_pypy/_ssl/__init__.py +++ b/lib_pypy/_ssl/__init__.py @@ -16,12 +16,14 @@ RAND_egd = builtinify(RAND_egd) import sys -if sys.platform == "win32" and 'enum_certificates' not in globals(): - def enum_certificates(*args, **kwds): - import warnings - warnings.warn("ssl.enum_certificates() is not implemented") - return [] - def enum_crls(*args, **kwds): - import warnings - warnings.warn("ssl.enum_crls() is not implemented") - return [] +if sys.platform == "win32": + if 'enum_certificates' not in globals(): + def enum_certificates(*args, **kwds): + import warnings + warnings.warn("ssl.enum_certificates() is not implemented") + return [] + if 'enum_crls' not in globals(): + def enum_crls(*args, **kwds): + import warnings + warnings.warn("ssl.enum_crls() is not implemented") + return [] diff --git a/lib_pypy/_ssl_build.py b/lib_pypy/_ssl_build.py --- a/lib_pypy/_ssl_build.py +++ b/lib_pypy/_ssl_build.py @@ -5,6 +5,11 @@ from _cffi_ssl._cffi_src.build_openssl import (build_ffi_for_binding, _get_openssl_libraries, extra_link_args, compiler_type) +if sys.platform == "win32": + pypy_win32_extra = ["pypy_win32_extra"] +else: + pypy_win32_extra = [] + ffi = build_ffi_for_binding( module_name="_pypy_openssl", module_prefix="_cffi_src.openssl.", @@ -44,10 +49,10 @@ "x509_vfy", "pkcs7", "callbacks", - ], + ] + pypy_win32_extra, libraries=_get_openssl_libraries(sys.platform), extra_link_args=extra_link_args(compiler_type()), ) if __name__ == '__main__': - ffi.compile() + ffi.compile(verbose=True) From pypy.commits at gmail.com Mon Apr 23 12:15:24 2018 From: pypy.commits at gmail.com (arigo) Date: Mon, 23 Apr 2018 09:15:24 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Implement ssl.enum_crls() for win32 Message-ID: <5ade069c.902e1c0a.cb947.1cca@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r94429:2a637a86bcba Date: 2018-04-23 18:02 +0200 http://bitbucket.org/pypy/pypy/changeset/2a637a86bcba/ Log: Implement ssl.enum_crls() for win32 diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/pypy_win32_extra.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/pypy_win32_extra.py --- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/pypy_win32_extra.py +++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/pypy_win32_extra.py @@ -12,12 +12,21 @@ TYPES = """ typedef ... *HCERTSTORE; typedef ... *HCRYPTPROV_LEGACY; + typedef struct { DWORD dwCertEncodingType; BYTE *pbCertEncoded; DWORD cbCertEncoded; ...; } CERT_CONTEXT, *PCCERT_CONTEXT; + +typedef struct { + DWORD dwCertEncodingType; + BYTE *pbCrlEncoded; + DWORD cbCrlEncoded; + ...; +} CRL_CONTEXT, *PCCRL_CONTEXT; + typedef struct { DWORD cUsageIdentifier; LPSTR *rgpszUsageIdentifier; @@ -40,6 +49,9 @@ BOOL WINAPI CertFreeCertificateContext( PCCERT_CONTEXT pCertContext ); +BOOL WINAPI CertFreeCRLContext( + PCCRL_CONTEXT pCrlContext +); BOOL WINAPI CertCloseStore( HCERTSTORE hCertStore, DWORD dwFlags @@ -50,6 +62,10 @@ PCERT_ENHKEY_USAGE pUsage, DWORD *pcbUsage ); +PCCRL_CONTEXT WINAPI CertEnumCRLsInStore( + HCERTSTORE hCertStore, + PCCRL_CONTEXT pPrevCrlContext +); """ MACROS = """ diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py --- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py +++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py @@ -24,7 +24,7 @@ from enum import IntEnum as _IntEnum if sys.platform == 'win32': - from _cffi_ssl._stdssl.win32_extra import enum_certificates + from _cffi_ssl._stdssl.win32_extra import enum_certificates, enum_crls HAVE_POLL = False else: from select import poll, POLLIN, POLLOUT diff --git a/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py b/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py --- a/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py +++ b/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py @@ -18,20 +18,55 @@ result = [] pCertCtx = ffi.NULL - while True: - pCertCtx = lib.CertEnumCertificatesInStore(hStore, pCertCtx) - if pCertCtx == ffi.NULL: - break - cert = ffi.buffer(pCertCtx.pbCertEncoded, pCertCtx.cbCertEncoded)[:] - enc = certEncodingType(pCertCtx.dwCertEncodingType) - keyusage = parseKeyUsage(pCertCtx, lib.CERT_FIND_PROP_ONLY_ENHKEY_USAGE_FLAG) - if keyusage is True: - keyusage = parseKeyUsage(pCertCtx, lib.CERT_FIND_EXT_ONLY_ENHKEY_USAGE_FLAG) - result.append((cert, enc, keyusage)) + try: + while True: + pCertCtx = lib.CertEnumCertificatesInStore(hStore, pCertCtx) + if pCertCtx == ffi.NULL: + break + cert = ffi.buffer(pCertCtx.pbCertEncoded, pCertCtx.cbCertEncoded)[:] + enc = certEncodingType(pCertCtx.dwCertEncodingType) + keyusage = parseKeyUsage(pCertCtx, lib.CERT_FIND_PROP_ONLY_ENHKEY_USAGE_FLAG) + if keyusage is True: + keyusage = parseKeyUsage(pCertCtx, lib.CERT_FIND_EXT_ONLY_ENHKEY_USAGE_FLAG) + result.append((cert, enc, keyusage)) + finally: + if pCertCtx != ffi.NULL: + lib.CertFreeCertificateContext(pCertCtx) + if not lib.CertCloseStore(hStore, 0): + # This error case might shadow another exception. + raise WindowsError(*ffi.getwinerror()) + return result - if pCertCtx != ffi.NULL: - lib.CertFreeCertificateContext(pCertCtx) - lib.CertCloseStore(hStore, 0) + +def enum_crls(store_name): + """Retrieve CRLs from Windows' cert store. + +store_name may be one of 'CA', 'ROOT' or 'MY'. The system may provide +more cert storages, too. The function returns a list of (bytes, +encoding_type) tuples. The encoding_type flag can be interpreted with +X509_ASN_ENCODING or PKCS_7_ASN_ENCODING.""" + hStore = lib.CertOpenStore(lib.CERT_STORE_PROV_SYSTEM_A, 0, ffi.NULL, + lib.CERT_STORE_READONLY_FLAG | lib.CERT_SYSTEM_STORE_LOCAL_MACHINE, + bytes(store_name, "ascii")) + if hStore == ffi.NULL: + raise WindowsError(*ffi.getwinerror()) + + result = [] + pCrlCtx = ffi.NULL + try: + while True: + pCrlCtx = lib.CertEnumCRLsInStore(hStore, pCrlCtx) + if pCrlCtx == ffi.NULL: + break + crl = ffi.buffer(pCrlCtx.pbCrlEncoded, pCrlCtx.cbCrlEncoded)[:] + enc = certEncodingType(pCrlCtx.dwCertEncodingType) + result.append((crl, enc)) + finally: + if pCrlCtx != ffi.NULL: + lib.CertFreeCRLContext(pCrlCtx) + if not lib.CertCloseStore(hStore, 0): + # This error case might shadow another exception. + raise WindowsError(*ffi.getwinerror()) return result From pypy.commits at gmail.com Mon Apr 23 23:11:07 2018 From: pypy.commits at gmail.com (wlav) Date: Mon, 23 Apr 2018 20:11:07 -0700 (PDT) Subject: [pypy-commit] pypy cppyy-packaging: update enum handling Message-ID: <5adea04b.1c69fb81.1c5f7.b126@mx.google.com> Author: Wim Lavrijsen Branch: cppyy-packaging Changeset: r94433:3f9d98258560 Date: 2018-04-23 19:46 -0700 http://bitbucket.org/pypy/pypy/changeset/3f9d98258560/ Log: update enum handling diff --git a/pypy/module/_cppyy/src/dummy_backend.cxx b/pypy/module/_cppyy/src/dummy_backend.cxx --- a/pypy/module/_cppyy/src/dummy_backend.cxx +++ b/pypy/module/_cppyy/src/dummy_backend.cxx @@ -408,6 +408,8 @@ /* name to opaque C++ scope representation -------------------------------- */ char* cppyy_resolve_name(const char* cppitem_name) { + if (cppyy_is_enum(cppitem_name)) + return cppstring_to_cstring("internal_enum_type_t"); return cppstring_to_cstring(cppitem_name); } From pypy.commits at gmail.com Mon Apr 23 23:11:02 2018 From: pypy.commits at gmail.com (wlav) Date: Mon, 23 Apr 2018 20:11:02 -0700 (PDT) Subject: [pypy-commit] pypy cppyy-packaging: initial support for exception handling from wrapped functions Message-ID: <5adea046.08da1c0a.bf586.9bab@mx.google.com> Author: Wim Lavrijsen Branch: cppyy-packaging Changeset: r94431:dd967ce1da92 Date: 2018-04-23 15:52 -0700 http://bitbucket.org/pypy/pypy/changeset/dd967ce1da92/ Log: initial support for exception handling from wrapped functions diff --git a/pypy/module/_cppyy/include/capi.h b/pypy/module/_cppyy/include/capi.h --- a/pypy/module/_cppyy/include/capi.h +++ b/pypy/module/_cppyy/include/capi.h @@ -16,6 +16,8 @@ typedef long cppyy_index_t; typedef void* cppyy_funcaddr_t; + typedef unsigned long cppyy_exctype_t; + /* name to opaque C++ scope representation -------------------------------- */ RPY_EXTERN char* cppyy_resolve_name(const char* cppitem_name); diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py --- a/pypy/module/_cppyy/interp_cppyy.py +++ b/pypy/module/_cppyy/interp_cppyy.py @@ -19,6 +19,9 @@ INSTANCE_FLAGS_IS_REF = 0x0002 INSTANCE_FLAGS_IS_R_VALUE = 0x0004 +OVERLOAD_FLAGS_USE_FFI = 0x0001 + + class FastCallNotPossible(Exception): pass @@ -186,7 +189,7 @@ return rffi.cast(rffi.VOIDP, loc_idx) @jit.unroll_safe - def call(self, cppthis, args_w): + def call(self, cppthis, args_w, useffi): jit.promote(self) assert lltype.typeOf(cppthis) == capi.C_OBJECT @@ -218,16 +221,25 @@ try: # attempt to call directly through ffi chain - if self._funcaddr: + if useffi and self._funcaddr: try: return self.do_fast_call(cppthis, args_w, call_local) except FastCallNotPossible: pass # can happen if converters or executor does not implement ffi # ffi chain must have failed; using stub functions instead - args = self.prepare_arguments(args_w, call_local) + args, stat = self.prepare_arguments(args_w, call_local) try: - return self.executor.execute(self.space, self.cppmethod, cppthis, len(args_w), args) + result = self.executor.execute( + self.space, self.cppmethod, cppthis, len(args_w), args) + if stat[0] != rffi.cast(rffi.ULONG, 0): + what = rffi.cast(rffi.CCHARP, stat[1]) + pywhat = rffi.charp2str(what) + capi.c_free(self.space, rffi.cast(rffi.VOIDP, what)) + if hasattr(self.space, "fake"): + raise OperationError(self.space.w_Exception, self.space.newtext("C++ exception")) + raise oefmt(self.space.w_Exception, pywhat) + return result finally: self.finalize_call(args, args_w, call_local) finally: @@ -373,7 +385,10 @@ conv.free_argument(self.space, rffi.cast(capi.C_OBJECT, arg_j), loc_j) capi.c_deallocate_function_args(self.space, args) raise - return args + stat = rffi.cast(rffi.ULONGP, + lltype.direct_ptradd(rffi.cast(rffi.CCHARP, args), int(len(args_w))*stride)) + stat[0] = rffi.cast(rffi.ULONG, 0) + return args, stat @jit.unroll_safe def finalize_call(self, args, args_w, call_local): @@ -435,7 +450,7 @@ # TODO: might have to specialize for CPPTemplatedCall on CPPMethod/CPPFunction here CPPMethod.__init__(self, space, declaring_scope, method_index, arg_defs, args_required) - def call(self, cppthis, args_w): + def call(self, cppthis, args_w, useffi): assert lltype.typeOf(cppthis) == capi.C_OBJECT for i in range(len(args_w)): try: @@ -447,10 +462,10 @@ raise oefmt(self.space.w_TypeError, "non-matching template (got %s where %s expected)", s, self.templ_args[i]) - return W_CPPBoundMethod(cppthis, self) + return W_CPPBoundMethod(cppthis, self, useffi) - def bound_call(self, cppthis, args_w): - return CPPMethod.call(self, cppthis, args_w) + def bound_call(self, cppthis, args_w, useffi): + return CPPMethod.call(self, cppthis, args_w, useffi) def __repr__(self): return "CPPTemplatedCall: %s" % self.prototype() @@ -468,11 +483,11 @@ def unpack_cppthis(space, w_cppinstance, declaring_scope): return rffi.cast(capi.C_OBJECT, declaring_scope.handle) - def call(self, cppthis, args_w): + def call(self, cppthis, args_w, useffi): # Note: this does not return a wrapped instance, just a pointer to the # new instance; the overload must still wrap it before returning. Also, # cppthis is declaring_scope.handle (as per unpack_cppthis(), above). - return CPPMethod.call(self, cppthis, args_w) + return CPPMethod.call(self, cppthis, args_w, useffi) def __repr__(self): return "CPPConstructor: %s" % self.prototype() @@ -485,7 +500,7 @@ _immutable_ = True - def call(self, cppthis, args_w): + def call(self, cppthis, args_w, useffi): end = len(args_w)-1 if 0 <= end: w_item = args_w[end] @@ -493,7 +508,7 @@ if self.converters is None: self._setup(cppthis) self.executor.set_item(self.space, w_item) # TODO: what about threads? - CPPMethod.call(self, cppthis, args_w) + CPPMethod.call(self, cppthis, args_w, useffi) class W_CPPOverload(W_Root): @@ -501,7 +516,7 @@ collection of (possibly) overloaded methods or functions. It calls these in order and deals with error handling and reporting.""" - _attrs_ = ['space', 'scope', 'functions'] + _attrs_ = ['space', 'scope', 'functions', 'flags'] _immutable_fields_ = ['scope', 'functions[*]'] def __init__(self, space, declaring_scope, functions): @@ -510,6 +525,19 @@ assert len(functions) from rpython.rlib import debug self.functions = debug.make_sure_not_resized(functions) + self.flags = 0 + self.flags |= OVERLOAD_FLAGS_USE_FFI + + # allow user to determine ffi use rules per overload + def fget_useffi(self, space): + return space.newbool(bool(self.flags & OVERLOAD_FLAGS_USE_FFI)) + + @unwrap_spec(value=bool) + def fset_useffi(self, space, value): + if space.is_true(value): + self.flags |= OVERLOAD_FLAGS_USE_FFI + else: + self.flags &= ~OVERLOAD_FLAGS_USE_FFI @jit.elidable_promote() def is_static(self): @@ -540,7 +568,7 @@ for i in range(len(self.functions)): cppyyfunc = self.functions[i] try: - return cppyyfunc.call(cppthis, args_w) + return cppyyfunc.call(cppthis, args_w, self.flags & OVERLOAD_FLAGS_USE_FFI) except Exception: pass @@ -553,7 +581,7 @@ for i in range(len(self.functions)): cppyyfunc = self.functions[i] try: - return cppyyfunc.call(cppthis, args_w) + return cppyyfunc.call(cppthis, args_w, self.flags & OVERLOAD_FLAGS_USE_FFI) except OperationError as e: # special case if there's just one function, to prevent clogging the error message if len(self.functions) == 1: @@ -588,6 +616,7 @@ 'CPPOverload', is_static = interp2app(W_CPPOverload.is_static), call = interp2app(W_CPPOverload.call), + __useffi__ = GetSetProperty(W_CPPOverload.fget_useffi, W_CPPOverload.fset_useffi), prototype = interp2app(W_CPPOverload.prototype), ) @@ -642,17 +671,18 @@ class W_CPPBoundMethod(W_Root): - _attrs_ = ['cppthis', 'method'] + _attrs_ = ['cppthis', 'method', 'useffi'] - def __init__(self, cppthis, method): + def __init__(self, cppthis, method, useffi): self.cppthis = cppthis self.method = method + self.useffi = useffi def __call__(self, args_w): - return self.method.bound_call(self.cppthis, args_w) + return self.method.bound_call(self.cppthis, args_w, self.useffi) def __repr__(self): - return "W_CPPBoundMethod(%s)" % [f.prototype() for f in self.functions] + return "W_CPPBoundMethod(%s)" % self.method.prototype() W_CPPBoundMethod.typedef = TypeDef( 'CPPBoundMethod', diff --git a/pypy/module/_cppyy/test/advancedcpp.cxx b/pypy/module/_cppyy/test/advancedcpp.cxx --- a/pypy/module/_cppyy/test/advancedcpp.cxx +++ b/pypy/module/_cppyy/test/advancedcpp.cxx @@ -1,5 +1,7 @@ #include "advancedcpp.h" +#include + // for testing of default arguments #define IMPLEMENT_DEFAULTER_CLASS(type, tname) \ @@ -112,3 +114,13 @@ std::string overload_the_other_way::gime() { return "aap"; } int overload_the_other_way::gime() const { return 1; } + + +// exception handling testing +void Thrower::throw_anything() { + throw 1; +} + +void Thrower::throw_exception() { + throw std::runtime_error("C++ function failed"); +} diff --git a/pypy/module/_cppyy/test/advancedcpp.h b/pypy/module/_cppyy/test/advancedcpp.h --- a/pypy/module/_cppyy/test/advancedcpp.h +++ b/pypy/module/_cppyy/test/advancedcpp.h @@ -400,3 +400,11 @@ std::string gime(); int gime() const; }; + + +//=========================================================================== +class Thrower { // exception handling testing +public: + void throw_anything(); + void throw_exception(); +}; diff --git a/pypy/module/_cppyy/test/advancedcpp.xml b/pypy/module/_cppyy/test/advancedcpp.xml --- a/pypy/module/_cppyy/test/advancedcpp.xml +++ b/pypy/module/_cppyy/test/advancedcpp.xml @@ -57,4 +57,6 @@ + + diff --git a/pypy/module/_cppyy/test/test_advancedcpp.py b/pypy/module/_cppyy/test/test_advancedcpp.py --- a/pypy/module/_cppyy/test/test_advancedcpp.py +++ b/pypy/module/_cppyy/test/test_advancedcpp.py @@ -656,3 +656,22 @@ # TODO: currently fails b/c double** not understood as &double* #assert cppyy.gbl.my_global_ptr[0] == 1234. + def test22_exceptions(self): + """Catching of C++ exceptions""" + + import _cppyy as cppyy + Thrower = cppyy.gbl.Thrower + + # TODO: clean up this interface: + Thrower.__cppdecl__.get_overload('throw_anything').__useffi__ = False + Thrower.__cppdecl__.get_overload('throw_exception').__useffi__ = False + + t = Thrower() + + assert raises(Exception, t.throw_anything) + assert raises(Exception, t.throw_exception) + + try: + t.throw_exception() + except Exception, e: + "C++ function failed" in str(e) diff --git a/pypy/module/_cppyy/test/test_overloads.py b/pypy/module/_cppyy/test/test_overloads.py --- a/pypy/module/_cppyy/test/test_overloads.py +++ b/pypy/module/_cppyy/test/test_overloads.py @@ -15,7 +15,6 @@ spaceconfig = dict(usemodules=['_cppyy', '_rawffi', 'itertools']) def setup_class(cls): - env = os.environ cls.w_test_dct = cls.space.newtext(test_dct) cls.w_overloads = cls.space.appexec([], """(): import ctypes diff --git a/pypy/module/_cppyy/test/test_zjit.py b/pypy/module/_cppyy/test/test_zjit.py --- a/pypy/module/_cppyy/test/test_zjit.py +++ b/pypy/module/_cppyy/test/test_zjit.py @@ -137,6 +137,7 @@ executor.get_executor(self, 'int').__class__.c_stubcall = staticmethod(c_call_i) self.w_AttributeError = FakeException(self, "AttributeError") + self.w_Exception = FakeException(self, "Exception") self.w_KeyError = FakeException(self, "KeyError") self.w_NotImplementedError = FakeException(self, "NotImplementedError") self.w_ReferenceError = FakeException(self, "ReferenceError") From pypy.commits at gmail.com Mon Apr 23 23:11:09 2018 From: pypy.commits at gmail.com (wlav) Date: Mon, 23 Apr 2018 20:11:09 -0700 (PDT) Subject: [pypy-commit] pypy cppyy-packaging: fix translation Message-ID: <5adea04d.1c69fb81.2f69b.552a@mx.google.com> Author: Wim Lavrijsen Branch: cppyy-packaging Changeset: r94434:43f7036815b1 Date: 2018-04-23 19:46 -0700 http://bitbucket.org/pypy/pypy/changeset/43f7036815b1/ Log: fix translation diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py --- a/pypy/module/_cppyy/interp_cppyy.py +++ b/pypy/module/_cppyy/interp_cppyy.py @@ -236,9 +236,7 @@ what = rffi.cast(rffi.CCHARP, stat[1]) pywhat = rffi.charp2str(what) capi.c_free(self.space, rffi.cast(rffi.VOIDP, what)) - if hasattr(self.space, "fake"): - raise OperationError(self.space.w_Exception, self.space.newtext("C++ exception")) - raise oefmt(self.space.w_Exception, pywhat) + raise OperationError(self.space.w_Exception, self.space.newtext(pywhat)) return result finally: self.finalize_call(args, args_w, call_local) From pypy.commits at gmail.com Mon Apr 23 23:11:05 2018 From: pypy.commits at gmail.com (wlav) Date: Mon, 23 Apr 2018 20:11:05 -0700 (PDT) Subject: [pypy-commit] pypy cppyy-packaging: merge default into branch Message-ID: <5adea049.4a061c0a.b4d55.c706@mx.google.com> Author: Wim Lavrijsen Branch: cppyy-packaging Changeset: r94432:4f2df82b0b9f Date: 2018-04-23 15:52 -0700 http://bitbucket.org/pypy/pypy/changeset/4f2df82b0b9f/ Log: merge default into branch diff --git a/pypy/doc/release-v6.0.0.rst b/pypy/doc/release-v6.0.0.rst --- a/pypy/doc/release-v6.0.0.rst +++ b/pypy/doc/release-v6.0.0.rst @@ -18,6 +18,8 @@ getting started writing code. We have improved our parser to emit more friendly `syntax errors`_, making PyPy not only faster but more friendly. +The GC now has `hooks`_ to gain more insights into its performance + The Windows PyPy3.5 release is still considered beta-quality. There are open issues with unicode handling especially around system calls and c-extensions. @@ -53,6 +55,7 @@ .. _`blog post`: https://morepypy.blogspot.it/2017/10/cape-of-good-hope-for-pypy-hello-from.html .. _pygobject: https://lazka.github.io/posts/2018-04_pypy-pygobject/index.html .. _`syntax errors`: https://morepypy.blogspot.com/2018/04/improving-syntaxerror-in-pypy.html +.. _`hooks`: gc_info.html#gc-hooks What is PyPy? ============= @@ -101,8 +104,9 @@ * Added missing attributes to C-API ``instancemethod`` on pypy3 * Store error state in thread-local storage for C-API. * Fix JIT bugs exposed in the sre module -* Improve speed of Python parser, improve ParseError messages slightly +* Improve speed of Python parser, improve ParseError messages and SyntaxError * Handle JIT hooks more efficiently +* Fix a rare GC bug exposed by intensive use of cpyext `Buffer` s We also refactored many parts of the JIT bridge optimizations, as well as cpyext internals, and together with new contributors fixed issues, added new diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -3,18 +3,7 @@ ========================== .. this is a revision shortly after release-pypy-6.0.0 -.. startrev: f22145c34985 +.. startrev: ad79cc0ce9a8 -.. branch: issue2752 -Fix a rare GC bug that was introduced more than one year ago, but was -not diagnosed before issue #2752. - -.. branch: gc-hooks - -Introduce GC hooks, as documented in doc/gc_info.rst - -.. branch: gc-hook-better-timestamp - -Improve GC hooks diff --git a/pypy/doc/whatsnew-pypy2-6.0.0.rst b/pypy/doc/whatsnew-pypy2-6.0.0.rst --- a/pypy/doc/whatsnew-pypy2-6.0.0.rst +++ b/pypy/doc/whatsnew-pypy2-6.0.0.rst @@ -109,3 +109,16 @@ Improve line offsets that are reported by SyntaxError. Improve error messages for a few situations, including mismatched parenthesis. + +.. branch: issue2752 + +Fix a rare GC bug that was introduced more than one year ago, but was +not diagnosed before issue #2752. + +.. branch: gc-hooks + +Introduce GC hooks, as documented in doc/gc_info.rst + +.. branch: gc-hook-better-timestamp + +Improve GC hooksd diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -491,11 +491,17 @@ # 'action.fire()' happens to be called any time before # the corresponding perform(), the fire() has no # effect---which is the effect we want, because - # perform() will be called anyway. + # perform() will be called anyway. All such pending + # actions with _fired == True are still inside the old + # chained list. As soon as we reset _fired to False, + # we also reset _next to None and we are ready for + # another fire(). while action is not None: + next_action = action._next + action._next = None action._fired = False action.perform(ec, frame) - action._next, action = None, action._next + action = next_action self.action_dispatcher = action_dispatcher diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py --- a/pypy/interpreter/test/test_executioncontext.py +++ b/pypy/interpreter/test/test_executioncontext.py @@ -67,6 +67,47 @@ """) assert events == ['one'] + def test_fire_inside_perform(self): + # test what happens if we call AsyncAction.fire() while we are in the + # middle of an AsyncAction.perform(). In particular, this happens when + # PyObjectDeallocAction.fire() is called by rawrefcount: see issue + # 2805 + events = [] + + class Action1(executioncontext.AsyncAction): + _count = 0 + + def perform(self, ec, frame): + events.append('one') + if self._count == 0: + # a1 is no longer in the queue, so it will be enqueued + a1.fire() + # + # a2 is still in the queue, so the fire() is ignored and + # it's performed in its normal order, i.e. BEFORE a3 + a2.fire() + self._count += 1 + + class Action2(executioncontext.AsyncAction): + def perform(self, ec, frame): + events.append('two') + + class Action3(executioncontext.AsyncAction): + def perform(self, ec, frame): + events.append('three') + + space = self.space + a1 = Action1(space) + a2 = Action2(space) + a3 = Action3(space) + a1.fire() + a2.fire() + a3.fire() + space.appexec([], """(): + pass + """) + assert events == ['one', 'two', 'three', 'one'] + def test_periodic_action(self): from pypy.interpreter.executioncontext import ActionFlag diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py --- a/pypy/module/cpyext/test/test_arraymodule.py +++ b/pypy/module/cpyext/test/test_arraymodule.py @@ -194,7 +194,4 @@ self.attrib = True import gc module.subclass_with_attribute(Sub, "addattrib", "attrib", gc.collect) - if self.runappdirect: - assert Sub.__module__ == 'pypy.module.cpyext.test.test_arraymodule' - assert str(Sub) == "" - + assert Sub.__module__ == __name__ diff --git a/pypy/module/cpyext/test/test_datetime.py b/pypy/module/cpyext/test/test_datetime.py --- a/pypy/module/cpyext/test/test_datetime.py +++ b/pypy/module/cpyext/test/test_datetime.py @@ -1,3 +1,5 @@ +import pytest + from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.cdatetime import * @@ -82,6 +84,14 @@ date = datetime.datetime.fromtimestamp(0) assert space.unwrap(space.str(w_date)) == str(date) + @pytest.mark.parametrize('name', ['Time', 'DateTime', 'Date', 'Delta']) + def test_basicsize(self, space, name): + datetime = _PyDateTime_Import(space) + py_size = getattr(datetime, "c_%sType" % name).c_tp_basicsize + c_size = rffi.sizeof(cts.gettype("PyDateTime_%s" % name)) + assert py_size == c_size + + class AppTestDatetime(AppTestCpythonExtensionBase): def test_CAPI(self): module = self.import_extension('foo', [ @@ -271,9 +281,9 @@ 6, 6, 6, 6, args, PyDateTimeAPI->TimeType); """), ("datetime_with_tzinfo", "METH_O", - """ + """ PyObject * obj; - int tzrefcnt = args->ob_refcnt; + int tzrefcnt = args->ob_refcnt; PyDateTime_IMPORT; obj = PyDateTimeAPI->DateTime_FromDateAndTime( 2000, 6, 6, 6, 6, 6, 6, args, @@ -291,7 +301,7 @@ return NULL; } return obj; - + """), ], prologue='#include "datetime.h"\n') from datetime import tzinfo, datetime, timedelta, time @@ -339,4 +349,4 @@ assert module.checks(o) == (True,) * 3 + (False,) * 7 # isinstance(datetime, date) o = tzinfo() assert module.checks(o) == (False,) * 8 + (True,) * 2 - + diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py --- a/pypy/module/cpyext/test/test_unicodeobject.py +++ b/pypy/module/cpyext/test/test_unicodeobject.py @@ -26,11 +26,8 @@ if(PyUnicode_GetSize(s) != 11) { result = -PyUnicode_GetSize(s); } -#ifdef PYPY_VERSION - // Slightly silly test that tp_basicsize is reasonable. - if(s->ob_type->tp_basicsize != sizeof(void*)*7) + if(s->ob_type->tp_basicsize != sizeof(PyUnicodeObject)) result = s->ob_type->tp_basicsize; -#endif // PYPY_VERSION Py_DECREF(s); return PyLong_FromLong(result); """), diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py --- a/pypy/objspace/std/listobject.py +++ b/pypy/objspace/std/listobject.py @@ -991,6 +991,14 @@ def setslice(self, w_list, start, step, slicelength, w_other): strategy = w_other.strategy + if step != 1: + len2 = strategy.length(w_other) + if len2 == 0: + return + else: + raise oefmt(self.space.w_ValueError, + "attempt to assign sequence of size %d to extended " + "slice of size %d", len2, 0) storage = strategy.getstorage_copy(w_other) w_list.strategy = strategy w_list.lstorage = storage diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py --- a/pypy/objspace/std/test/test_listobject.py +++ b/pypy/objspace/std/test/test_listobject.py @@ -1080,6 +1080,15 @@ l[::3] = ('a', 'b') assert l == ['a', 1.1, 2.2, 'b', 4.4, 5.5] + l_int = [5]; l_int.pop() # IntListStrategy + l_empty = [] # EmptyListStrategy + raises(ValueError, "l_int[::-1] = [42]") + raises(ValueError, "l_int[::7] = [42]") + raises(ValueError, "l_empty[::-1] = [42]") + raises(ValueError, "l_empty[::7] = [42]") + l_int[::1] = [42]; assert l_int == [42] + l_empty[::1] = [42]; assert l_empty == [42] + def test_setslice_with_self(self): l = [1,2,3,4] l[:] = l From pypy.commits at gmail.com Mon Apr 23 23:17:17 2018 From: pypy.commits at gmail.com (wlav) Date: Mon, 23 Apr 2018 20:17:17 -0700 (PDT) Subject: [pypy-commit] pypy default: merge cppyy-packaging: move to latest backend (0.6.0) and support exceptions through wrappers Message-ID: <5adea1bd.8e6f1c0a.d6887.d494@mx.google.com> Author: Wim Lavrijsen Branch: Changeset: r94435:e50e11af23f1 Date: 2018-04-23 19:57 -0700 http://bitbucket.org/pypy/pypy/changeset/e50e11af23f1/ Log: merge cppyy-packaging: move to latest backend (0.6.0) and support exceptions through wrappers diff --git a/pypy/module/_cppyy/__init__.py b/pypy/module/_cppyy/__init__.py --- a/pypy/module/_cppyy/__init__.py +++ b/pypy/module/_cppyy/__init__.py @@ -13,7 +13,7 @@ '_set_function_generator': 'interp_cppyy.set_function_generator', '_register_class' : 'interp_cppyy.register_class', '_get_nullptr' : 'interp_cppyy.get_nullptr', - 'CPPClassBase' : 'interp_cppyy.W_CPPClass', + 'CPPInstanceBase' : 'interp_cppyy.W_CPPInstance', 'addressof' : 'interp_cppyy.addressof', '_bind_object' : 'interp_cppyy._bind_object', 'bind_object' : 'interp_cppyy.bind_object', diff --git a/pypy/module/_cppyy/capi/loadable_capi.py b/pypy/module/_cppyy/capi/loadable_capi.py --- a/pypy/module/_cppyy/capi/loadable_capi.py +++ b/pypy/module/_cppyy/capi/loadable_capi.py @@ -121,11 +121,11 @@ # TODO: the following need to match up with the globally defined C_XYZ low-level # types (see capi/__init__.py), but by using strings here, that isn't guaranteed - c_opaque_ptr = state.c_ulong + c_opaque_ptr = state.c_ulong # not ptrdiff_t (which is signed) c_scope = c_opaque_ptr c_type = c_scope - c_object = c_opaque_ptr + c_object = c_opaque_ptr # not voidp (to stick with one handle type) c_method = c_opaque_ptr c_index = state.c_long c_index_array = state.c_voidp @@ -150,16 +150,17 @@ self.capi_call_ifaces = { # name to opaque C++ scope representation - 'num_scopes' : ([c_scope], c_int), - 'scope_name' : ([c_scope, c_int], c_ccharp), - 'resolve_name' : ([c_ccharp], c_ccharp), + 'resolve_enum' : ([c_ccharp], c_ccharp), 'get_scope' : ([c_ccharp], c_scope), 'actual_class' : ([c_type, c_object], c_type), + 'size_of_klass' : ([c_type], c_size_t), + 'size_of_type' : ([c_ccharp], c_size_t), # memory management 'allocate' : ([c_type], c_object), 'deallocate' : ([c_type, c_object], c_void), + 'construct' : ([c_type], c_object), 'destruct' : ([c_type, c_object], c_void), # method/function dispatching @@ -182,7 +183,8 @@ 'constructor' : ([c_method, c_object, c_int, c_voidp], c_object), 'call_o' : ([c_method, c_object, c_int, c_voidp, c_type], c_object), - 'get_function_address' : ([c_scope, c_index], c_voidp), # TODO: verify + 'function_address_from_index' : ([c_scope, c_index], c_voidp), # TODO: verify + 'function_address_from_method' : ([c_method], c_voidp), # id. # handling of function argument buffer 'allocate_function_args' : ([c_int], c_voidp), @@ -196,6 +198,8 @@ 'is_abstract' : ([c_type], c_int), 'is_enum' : ([c_ccharp], c_int), + 'get_all_cpp_names' : ([c_scope, c_voidp], c_voidp), # const char** + # type/class reflection information 'final_name' : ([c_type], c_ccharp), 'scoped_final_name' : ([c_type], c_ccharp), @@ -208,10 +212,10 @@ # method/function reflection information 'num_methods' : ([c_scope], c_int), - 'method_index_at' : ([c_scope, c_int], c_index), 'method_indices_from_name' : ([c_scope, c_ccharp], c_index_array), 'method_name' : ([c_scope, c_index], c_ccharp), + 'method_mangled_name' : ([c_scope, c_index], c_ccharp), 'method_result_type' : ([c_scope, c_index], c_ccharp), 'method_num_args' : ([c_scope, c_index], c_int), 'method_req_args' : ([c_scope, c_index], c_int), @@ -219,7 +223,9 @@ 'method_arg_default' : ([c_scope, c_index, c_int], c_ccharp), 'method_signature' : ([c_scope, c_index, c_int], c_ccharp), 'method_prototype' : ([c_scope, c_index, c_int], c_ccharp), + 'is_const_method' : ([c_method], c_int), + 'exists_method_template' : ([c_scope, c_ccharp], c_int), 'method_is_template' : ([c_scope, c_index], c_int), 'method_num_template_args' : ([c_scope, c_index], c_int), 'method_template_arg_name' : ([c_scope, c_index, c_index], c_ccharp), @@ -228,7 +234,9 @@ 'get_global_operator' : ([c_scope, c_scope, c_scope, c_ccharp], c_index), # method properties + 'is_public_method' : ([c_type, c_index], c_int), 'is_constructor' : ([c_type, c_index], c_int), + 'is_destructor' : ([c_type, c_index], c_int), 'is_staticmethod' : ([c_type, c_index], c_int), # data member reflection information @@ -236,12 +244,14 @@ 'datamember_name' : ([c_scope, c_int], c_ccharp), 'datamember_type' : ([c_scope, c_int], c_ccharp), 'datamember_offset' : ([c_scope, c_int], c_ptrdiff_t), - 'datamember_index' : ([c_scope, c_ccharp], c_int), # data member properties 'is_publicdata' : ([c_scope, c_int], c_int), 'is_staticdata' : ([c_scope, c_int], c_int), + 'is_const_data' : ([c_scope, c_int], c_int), + 'is_enum_data' : ([c_scope, c_int], c_int), + 'get_dimension_size' : ([c_scope, c_int, c_int], c_int), # misc helpers 'strtoll' : ([c_ccharp], c_llong), @@ -328,25 +338,27 @@ return rffi.cast(rffi.CCHARP, ptr) # name to opaque C++ scope representation ------------------------------------ -def c_num_scopes(space, cppscope): - return space.int_w(call_capi(space, 'num_scopes', [_ArgH(cppscope.handle)])) -def c_scope_name(space, cppscope, iscope): - args = [_ArgH(cppscope.handle), _ArgL(iscope)] - return charp2str_free(space, call_capi(space, 'scope_name', args)) - def c_resolve_name(space, name): return charp2str_free(space, call_capi(space, 'resolve_name', [_ArgS(name)])) +def c_resolve_enum(space, name): + return charp2str_free(space, call_capi(space, 'resolve_enum', [_ArgS(name)])) def c_get_scope_opaque(space, name): return rffi.cast(C_SCOPE, space.uint_w(call_capi(space, 'get_scope', [_ArgS(name)]))) def c_actual_class(space, cppclass, cppobj): args = [_ArgH(cppclass.handle), _ArgH(cppobj)] return rffi.cast(C_TYPE, space.uint_w(call_capi(space, 'actual_class', args))) +def c_size_of_klass(space, cppclass): + return _cdata_to_size_t(space, call_capi(space, 'size_of_klass', [_ArgH(cppclass.handle)])) +def c_size_of_type(space, name): + return _cdata_to_size_t(space, call_capi(space, 'size_of_type', [_ArgS(name)])) # memory management ---------------------------------------------------------- def c_allocate(space, cppclass): return _cdata_to_cobject(space, call_capi(space, 'allocate', [_ArgH(cppclass.handle)])) def c_deallocate(space, cppclass, cppobject): call_capi(space, 'deallocate', [_ArgH(cppclass.handle), _ArgH(cppobject)]) +def c_construct(space, cppclass): + return _cdata_to_cobject(space, call_capi(space, 'construct', [_ArgH(cppclass.handle)])) def c_destruct(space, cppclass, cppobject): call_capi(space, 'destruct', [_ArgH(cppclass.handle), _ArgH(cppobject)]) @@ -391,7 +403,7 @@ w_cstr = call_capi(space, 'call_s', [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs), _ArgP(rffi.cast(rffi.VOIDP, length))]) - cstr_len = intmask(length[0]) + cstr_len = int(intmask(length[0])) finally: lltype.free(length, flavor='raw') return _cdata_to_ccharp(space, w_cstr), cstr_len @@ -403,10 +415,13 @@ args = [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs), _ArgH(cppclass.handle)] return _cdata_to_cobject(space, call_capi(space, 'call_o', args)) -def c_get_function_address(space, cppscope, index): +def c_function_address_from_index(space, cppscope, index): args = [_ArgH(cppscope.handle), _ArgL(index)] return rffi.cast(C_FUNC_PTR, - _cdata_to_ptr(space, call_capi(space, 'get_function_address', args))) + _cdata_to_ptr(space, call_capi(space, 'function_address_from_index', args))) +def c_function_address_from_method(space, cppmethod): + return rffi.cast(C_FUNC_PTR, + _cdata_to_ptr(space, call_capi(space, 'function_address_from_method', _ArgH(cppmethod)))) # handling of function argument buffer --------------------------------------- def c_allocate_function_args(space, size): @@ -430,6 +445,23 @@ def c_is_enum(space, name): return space.bool_w(call_capi(space, 'is_enum', [_ArgS(name)])) +def c_get_all_cpp_names(space, scope): + sz = lltype.malloc(rffi.SIZE_TP.TO, 1, flavor='raw', zero=True) + try: + args = [_ArgH(scope.handle), _ArgP(rffi.cast(rffi.VOIDP, sz))] + rawnames = rffi.cast(rffi.CCHARPP, + _cdata_to_ptr(space, call_capi(space, 'get_all_cpp_names', args))) + count = int(intmask(sz[0])) + finally: + lltype.free(sz, flavor='raw') + allnames = [] + for i in range(count): + pystr = rffi.charp2str(rawnames[i]) + c_free(space, rffi.cast(rffi.VOIDP, rawnames[i])) # c_free defined below + allnames.append(pystr) + c_free(space, rffi.cast(rffi.VOIDP, rawnames)) # id. + return allnames + # type/class reflection information ------------------------------------------ def c_final_name(space, cpptype): return charp2str_free(space, call_capi(space, 'final_name', [_ArgH(cpptype)])) @@ -462,13 +494,10 @@ def c_num_methods(space, cppscope): args = [_ArgH(cppscope.handle)] return space.int_w(call_capi(space, 'num_methods', args)) -def c_method_index_at(space, cppscope, imethod): - args = [_ArgH(cppscope.handle), _ArgL(imethod)] - return space.int_w(call_capi(space, 'method_index_at', args)) def c_method_indices_from_name(space, cppscope, name): args = [_ArgH(cppscope.handle), _ArgS(name)] indices = rffi.cast(C_INDEX_ARRAY, - _cdata_to_ptr(space, call_capi(space, 'method_indices_from_name', args))) + _cdata_to_ptr(space, call_capi(space, 'method_indices_from_name', args))) if not indices: return [] py_indices = [] @@ -506,6 +535,9 @@ args = [_ArgH(cppscope.handle), _ArgL(index), _ArgL(show_formalargs)] return charp2str_free(space, call_capi(space, 'method_prototype', args)) +def c_exists_method_template(space, cppscope, name): + args = [_ArgH(cppscope.handle), _ArgS(name)] + return space.bool_w(call_capi(space, 'exists_method_template', args)) def c_method_is_template(space, cppscope, index): args = [_ArgH(cppscope.handle), _ArgL(index)] return space.bool_w(call_capi(space, 'method_is_template', args)) @@ -531,9 +563,15 @@ return rffi.cast(WLAVC_INDEX, -1) # method properties ---------------------------------------------------------- +def c_is_public_method(space, cppclass, index): + args = [_ArgH(cppclass.handle), _ArgL(index)] + return space.bool_w(call_capi(space, 'is_public_method', args)) def c_is_constructor(space, cppclass, index): args = [_ArgH(cppclass.handle), _ArgL(index)] return space.bool_w(call_capi(space, 'is_constructor', args)) +def c_is_destructor(space, cppclass, index): + args = [_ArgH(cppclass.handle), _ArgL(index)] + return space.bool_w(call_capi(space, 'is_destructor', args)) def c_is_staticmethod(space, cppclass, index): args = [_ArgH(cppclass.handle), _ArgL(index)] return space.bool_w(call_capi(space, 'is_staticmethod', args)) @@ -562,6 +600,15 @@ def c_is_staticdata(space, cppscope, datamember_index): args = [_ArgH(cppscope.handle), _ArgL(datamember_index)] return space.bool_w(call_capi(space, 'is_staticdata', args)) +def c_is_const_data(space, cppscope, datamember_index): + args = [_ArgH(cppscope.handle), _ArgL(datamember_index)] + return space.bool_w(call_capi(space, 'is_const_data', args)) +def c_is_enum_data(space, cppscope, datamember_index): + args = [_ArgH(cppscope.handle), _ArgL(datamember_index)] + return space.bool_w(call_capi(space, 'is_enum_data', args)) +def c_get_dimension_size(space, cppscope, datamember_index, dim_idx): + args = [_ArgH(cppscope.handle), _ArgL(datamember_index), _ArgL(dim_idx)] + return space.bool_w(call_capi(space, 'get_dimension_size', args)) # misc helpers --------------------------------------------------------------- def c_strtoll(space, svalue): @@ -585,7 +632,7 @@ try: w_cstr = call_capi(space, 'stdstring2charp', [_ArgH(cppstr), _ArgP(rffi.cast(rffi.VOIDP, sz))]) - cstr_len = intmask(sz[0]) + cstr_len = int(intmask(sz[0])) finally: lltype.free(sz, flavor='raw') return rffi.charpsize2str(_cdata_to_ccharp(space, w_cstr), cstr_len) @@ -607,7 +654,7 @@ """Return a python string taking into account \0""" from pypy.module._cppyy import interp_cppyy - cppstr = space.interp_w(interp_cppyy.W_CPPClass, w_self, can_be_None=False) + cppstr = space.interp_w(interp_cppyy.W_CPPInstance, w_self, can_be_None=False) return space.newtext(c_stdstring2charp(space, cppstr._rawobject)) # setup pythonizations for later use at run-time diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py --- a/pypy/module/_cppyy/converter.py +++ b/pypy/module/_cppyy/converter.py @@ -22,8 +22,8 @@ def get_rawobject(space, w_obj, can_be_None=True): - from pypy.module._cppyy.interp_cppyy import W_CPPClass - cppinstance = space.interp_w(W_CPPClass, w_obj, can_be_None=can_be_None) + from pypy.module._cppyy.interp_cppyy import W_CPPInstance + cppinstance = space.interp_w(W_CPPInstance, w_obj, can_be_None=can_be_None) if cppinstance: rawobject = cppinstance.get_rawobject() assert lltype.typeOf(rawobject) == capi.C_OBJECT @@ -31,15 +31,15 @@ return capi.C_NULL_OBJECT def set_rawobject(space, w_obj, address): - from pypy.module._cppyy.interp_cppyy import W_CPPClass - cppinstance = space.interp_w(W_CPPClass, w_obj, can_be_None=True) + from pypy.module._cppyy.interp_cppyy import W_CPPInstance + cppinstance = space.interp_w(W_CPPInstance, w_obj, can_be_None=True) if cppinstance: assert lltype.typeOf(cppinstance._rawobject) == capi.C_OBJECT cppinstance._rawobject = rffi.cast(capi.C_OBJECT, address) def get_rawobject_nonnull(space, w_obj): - from pypy.module._cppyy.interp_cppyy import W_CPPClass - cppinstance = space.interp_w(W_CPPClass, w_obj, can_be_None=True) + from pypy.module._cppyy.interp_cppyy import W_CPPInstance + cppinstance = space.interp_w(W_CPPInstance, w_obj, can_be_None=True) if cppinstance: cppinstance._nullcheck() rawobject = cppinstance.get_rawobject() @@ -502,8 +502,8 @@ self.clsdecl = clsdecl def _unwrap_object(self, space, w_obj): - from pypy.module._cppyy.interp_cppyy import W_CPPClass - if isinstance(w_obj, W_CPPClass): + from pypy.module._cppyy.interp_cppyy import W_CPPInstance + if isinstance(w_obj, W_CPPInstance): from pypy.module._cppyy.interp_cppyy import INSTANCE_FLAGS_IS_R_VALUE if w_obj.flags & INSTANCE_FLAGS_IS_R_VALUE: # reject moves as all are explicit @@ -534,8 +534,8 @@ class InstanceMoveConverter(InstanceRefConverter): def _unwrap_object(self, space, w_obj): # moving is same as by-ref, but have to check that move is allowed - from pypy.module._cppyy.interp_cppyy import W_CPPClass, INSTANCE_FLAGS_IS_R_VALUE - if isinstance(w_obj, W_CPPClass): + from pypy.module._cppyy.interp_cppyy import W_CPPInstance, INSTANCE_FLAGS_IS_R_VALUE + if isinstance(w_obj, W_CPPInstance): if w_obj.flags & INSTANCE_FLAGS_IS_R_VALUE: w_obj.flags &= ~INSTANCE_FLAGS_IS_R_VALUE return InstanceRefConverter._unwrap_object(self, space, w_obj) @@ -598,8 +598,8 @@ raise FastCallNotPossible def finalize_call(self, space, w_obj, call_local): - from pypy.module._cppyy.interp_cppyy import W_CPPClass - assert isinstance(w_obj, W_CPPClass) + from pypy.module._cppyy.interp_cppyy import W_CPPInstance + assert isinstance(w_obj, W_CPPInstance) r = rffi.cast(rffi.VOIDPP, call_local) w_obj._rawobject = rffi.cast(capi.C_OBJECT, r[0]) @@ -617,8 +617,8 @@ InstanceConverter.__init__(self, space, cppclass) def _unwrap_object(self, space, w_obj): - from pypy.module._cppyy.interp_cppyy import W_CPPClass - if isinstance(w_obj, W_CPPClass): + from pypy.module._cppyy.interp_cppyy import W_CPPInstance + if isinstance(w_obj, W_CPPInstance): arg = InstanceConverter._unwrap_object(self, space, w_obj) return capi.c_stdstring2stdstring(space, arg) else: @@ -749,8 +749,6 @@ return InstancePtrPtrConverter(space, clsdecl) elif compound == "": return InstanceConverter(space, clsdecl) - elif capi.c_is_enum(space, clean_name): - return _converters['unsigned'](space, default) # 5) void* or void converter (which fails on use) if 0 <= compound.find('*'): diff --git a/pypy/module/_cppyy/executor.py b/pypy/module/_cppyy/executor.py --- a/pypy/module/_cppyy/executor.py +++ b/pypy/module/_cppyy/executor.py @@ -289,8 +289,6 @@ return InstancePtrExecutor(space, cppclass) elif compound == '**' or compound == '*&': return InstancePtrPtrExecutor(space, cppclass) - elif capi.c_is_enum(space, clean_name): - return _executors['internal_enum_type_t'](space, None) # 4) additional special cases if compound == '*': diff --git a/pypy/module/_cppyy/include/capi.h b/pypy/module/_cppyy/include/capi.h --- a/pypy/module/_cppyy/include/capi.h +++ b/pypy/module/_cppyy/include/capi.h @@ -8,26 +8,29 @@ extern "C" { #endif // ifdef __cplusplus - typedef unsigned long cppyy_scope_t; + typedef ptrdiff_t cppyy_scope_t; typedef cppyy_scope_t cppyy_type_t; - typedef unsigned long cppyy_object_t; - typedef unsigned long cppyy_method_t; + typedef void* cppyy_object_t; + typedef ptrdiff_t cppyy_method_t; + typedef long cppyy_index_t; typedef void* cppyy_funcaddr_t; + typedef unsigned long cppyy_exctype_t; + /* name to opaque C++ scope representation -------------------------------- */ RPY_EXTERN - int cppyy_num_scopes(cppyy_scope_t parent); + char* cppyy_resolve_name(const char* cppitem_name); RPY_EXTERN - char* cppyy_scope_name(cppyy_scope_t parent, cppyy_index_t iscope); - RPY_EXTERN - char* cppyy_resolve_name(const char* cppitem_name); + char* cppyy_resolve_enum(const char* enum_type); RPY_EXTERN cppyy_scope_t cppyy_get_scope(const char* scope_name); RPY_EXTERN cppyy_type_t cppyy_actual_class(cppyy_type_t klass, cppyy_object_t obj); RPY_EXTERN - size_t cppyy_size_of(cppyy_type_t klass); + size_t cppyy_size_of_klass(cppyy_type_t klass); + RPY_EXTERN + size_t cppyy_size_of_type(const char* type_name); /* memory management ------------------------------------------------------ */ RPY_EXTERN @@ -35,48 +38,53 @@ RPY_EXTERN void cppyy_deallocate(cppyy_type_t type, cppyy_object_t self); RPY_EXTERN + cppyy_object_t cppyy_construct(cppyy_type_t type); + RPY_EXTERN void cppyy_destruct(cppyy_type_t type, cppyy_object_t self); /* method/function dispatching -------------------------------------------- */ RPY_EXTERN - void cppyy_call_v(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); + void cppyy_call_v(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN unsigned char cppyy_call_b(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN - char cppyy_call_c(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); + char cppyy_call_c(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN - short cppyy_call_h(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); + short cppyy_call_h(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN - int cppyy_call_i(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); + int cppyy_call_i(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN - long cppyy_call_l(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); + long cppyy_call_l(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN long long cppyy_call_ll(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN - float cppyy_call_f(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); + float cppyy_call_f(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN double cppyy_call_d(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN long double cppyy_call_ld(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN - void* cppyy_call_r(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); + void* cppyy_call_r(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN - char* cppyy_call_s(cppyy_method_t method, cppyy_object_t self, int nargs, void* args, size_t* length); - + char* cppyy_call_s(cppyy_method_t method, cppyy_object_t self, int nargs, void* args, size_t* length); RPY_EXTERN cppyy_object_t cppyy_constructor(cppyy_method_t method, cppyy_type_t klass, int nargs, void* args); RPY_EXTERN + void cppyy_destructor(cppyy_type_t type, cppyy_object_t self); + RPY_EXTERN cppyy_object_t cppyy_call_o(cppyy_method_t method, cppyy_object_t self, int nargs, void* args, cppyy_type_t result_type); RPY_EXTERN - cppyy_funcaddr_t cppyy_get_function_address(cppyy_scope_t scope, cppyy_index_t idx); + cppyy_funcaddr_t cppyy_function_address_from_index(cppyy_scope_t scope, cppyy_index_t idx); + RPY_EXTERN + cppyy_funcaddr_t cppyy_function_address_from_method(cppyy_method_t method); /* handling of function argument buffer ----------------------------------- */ RPY_EXTERN - void* cppyy_allocate_function_args(int nargs); + void* cppyy_allocate_function_args(int nargs); RPY_EXTERN - void cppyy_deallocate_function_args(void* args); + void cppyy_deallocate_function_args(void* args); RPY_EXTERN size_t cppyy_function_arg_sizeof(); RPY_EXTERN @@ -92,6 +100,9 @@ RPY_EXTERN int cppyy_is_enum(const char* type_name); + RPY_EXTERN + const char** cppyy_get_all_cpp_names(cppyy_scope_t scope, size_t* count); + /* class reflection information ------------------------------------------- */ RPY_EXTERN char* cppyy_final_name(cppyy_type_t type); @@ -105,6 +116,10 @@ char* cppyy_base_name(cppyy_type_t type, int base_index); RPY_EXTERN int cppyy_is_subtype(cppyy_type_t derived, cppyy_type_t base); + RPY_EXTERN + int cppyy_smartptr_info(const char* name, cppyy_type_t* raw, cppyy_method_t* deref); + RPY_EXTERN + void cppyy_add_smartptr_type(const char* type_name); /* calculate offsets between declared and actual type, up-cast: direction > 0; down-cast: direction < 0 */ RPY_EXTERN @@ -114,8 +129,6 @@ RPY_EXTERN int cppyy_num_methods(cppyy_scope_t scope); RPY_EXTERN - cppyy_index_t cppyy_method_index_at(cppyy_scope_t scope, int imeth); - RPY_EXTERN cppyy_index_t* cppyy_method_indices_from_name(cppyy_scope_t scope, const char* name); RPY_EXTERN @@ -136,8 +149,12 @@ char* cppyy_method_signature(cppyy_scope_t scope, cppyy_index_t idx, int show_formalargs); RPY_EXTERN char* cppyy_method_prototype(cppyy_scope_t scope, cppyy_index_t idx, int show_formalargs); + RPY_EXTERN + int cppyy_is_const_method(cppyy_method_t); RPY_EXTERN + int cppyy_exists_method_template(cppyy_scope_t scope, const char* name); + RPY_EXTERN int cppyy_method_is_template(cppyy_scope_t scope, cppyy_index_t idx); RPY_EXTERN int cppyy_method_num_template_args(cppyy_scope_t scope, cppyy_index_t idx); @@ -169,15 +186,20 @@ char* cppyy_datamember_type(cppyy_scope_t scope, int datamember_index); RPY_EXTERN ptrdiff_t cppyy_datamember_offset(cppyy_scope_t scope, int datamember_index); - RPY_EXTERN int cppyy_datamember_index(cppyy_scope_t scope, const char* name); /* data member properties ------------------------------------------------- */ RPY_EXTERN - int cppyy_is_publicdata(cppyy_type_t type, int datamember_index); + int cppyy_is_publicdata(cppyy_type_t type, cppyy_index_t datamember_index); RPY_EXTERN - int cppyy_is_staticdata(cppyy_type_t type, int datamember_index); + int cppyy_is_staticdata(cppyy_type_t type, cppyy_index_t datamember_index); + RPY_EXTERN + int cppyy_is_const_data(cppyy_scope_t scope, cppyy_index_t idata); + RPY_EXTERN + int cppyy_is_enum_data(cppyy_scope_t scope, cppyy_index_t idata); + RPY_EXTERN + int cppyy_get_dimension_size(cppyy_scope_t scope, cppyy_index_t idata, int dimension); /* misc helpers ----------------------------------------------------------- */ RPY_EXTERN @@ -197,7 +219,7 @@ RPY_EXTERN const char* cppyy_stdvector_valuetype(const char* clname); RPY_EXTERN - size_t cppyy_stdvector_valuesize(const char* clname); + size_t cppyy_stdvector_valuesize(const char* clname); #ifdef __cplusplus } diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py --- a/pypy/module/_cppyy/interp_cppyy.py +++ b/pypy/module/_cppyy/interp_cppyy.py @@ -19,6 +19,9 @@ INSTANCE_FLAGS_IS_REF = 0x0002 INSTANCE_FLAGS_IS_R_VALUE = 0x0004 +OVERLOAD_FLAGS_USE_FFI = 0x0001 + + class FastCallNotPossible(Exception): pass @@ -174,7 +177,7 @@ @staticmethod def unpack_cppthis(space, w_cppinstance, declaring_scope): - cppinstance = space.interp_w(W_CPPClass, w_cppinstance, can_be_None=False) + cppinstance = space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=False) cppinstance._nullcheck() return cppinstance.get_cppthis(declaring_scope) @@ -186,7 +189,7 @@ return rffi.cast(rffi.VOIDP, loc_idx) @jit.unroll_safe - def call(self, cppthis, args_w): + def call(self, cppthis, args_w, useffi): jit.promote(self) assert lltype.typeOf(cppthis) == capi.C_OBJECT @@ -218,16 +221,23 @@ try: # attempt to call directly through ffi chain - if self._funcaddr: + if useffi and self._funcaddr: try: return self.do_fast_call(cppthis, args_w, call_local) except FastCallNotPossible: pass # can happen if converters or executor does not implement ffi # ffi chain must have failed; using stub functions instead - args = self.prepare_arguments(args_w, call_local) + args, stat = self.prepare_arguments(args_w, call_local) try: - return self.executor.execute(self.space, self.cppmethod, cppthis, len(args_w), args) + result = self.executor.execute( + self.space, self.cppmethod, cppthis, len(args_w), args) + if stat[0] != rffi.cast(rffi.ULONG, 0): + what = rffi.cast(rffi.CCHARP, stat[1]) + pywhat = rffi.charp2str(what) + capi.c_free(self.space, rffi.cast(rffi.VOIDP, what)) + raise OperationError(self.space.w_Exception, self.space.newtext(pywhat)) + return result finally: self.finalize_call(args, args_w, call_local) finally: @@ -322,7 +332,7 @@ # Each CPPMethod corresponds one-to-one to a C++ equivalent and cppthis # has been offset to the matching class. Hence, the libffi pointer is # uniquely defined and needs to be setup only once. - funcaddr = capi.c_get_function_address(self.space, self.scope, self.index) + funcaddr = capi.c_function_address_from_index(self.space, self.scope, self.index) if funcaddr and cppthis: # methods only for now state = self.space.fromcache(ffitypes.State) @@ -373,7 +383,10 @@ conv.free_argument(self.space, rffi.cast(capi.C_OBJECT, arg_j), loc_j) capi.c_deallocate_function_args(self.space, args) raise - return args + stat = rffi.cast(rffi.ULONGP, + lltype.direct_ptradd(rffi.cast(rffi.CCHARP, args), int(len(args_w))*stride)) + stat[0] = rffi.cast(rffi.ULONG, 0) + return args, stat @jit.unroll_safe def finalize_call(self, args, args_w, call_local): @@ -435,7 +448,7 @@ # TODO: might have to specialize for CPPTemplatedCall on CPPMethod/CPPFunction here CPPMethod.__init__(self, space, declaring_scope, method_index, arg_defs, args_required) - def call(self, cppthis, args_w): + def call(self, cppthis, args_w, useffi): assert lltype.typeOf(cppthis) == capi.C_OBJECT for i in range(len(args_w)): try: @@ -447,10 +460,10 @@ raise oefmt(self.space.w_TypeError, "non-matching template (got %s where %s expected)", s, self.templ_args[i]) - return W_CPPBoundMethod(cppthis, self) + return W_CPPBoundMethod(cppthis, self, useffi) - def bound_call(self, cppthis, args_w): - return CPPMethod.call(self, cppthis, args_w) + def bound_call(self, cppthis, args_w, useffi): + return CPPMethod.call(self, cppthis, args_w, useffi) def __repr__(self): return "CPPTemplatedCall: %s" % self.prototype() @@ -468,11 +481,11 @@ def unpack_cppthis(space, w_cppinstance, declaring_scope): return rffi.cast(capi.C_OBJECT, declaring_scope.handle) - def call(self, cppthis, args_w): + def call(self, cppthis, args_w, useffi): # Note: this does not return a wrapped instance, just a pointer to the # new instance; the overload must still wrap it before returning. Also, # cppthis is declaring_scope.handle (as per unpack_cppthis(), above). - return CPPMethod.call(self, cppthis, args_w) + return CPPMethod.call(self, cppthis, args_w, useffi) def __repr__(self): return "CPPConstructor: %s" % self.prototype() @@ -485,7 +498,7 @@ _immutable_ = True - def call(self, cppthis, args_w): + def call(self, cppthis, args_w, useffi): end = len(args_w)-1 if 0 <= end: w_item = args_w[end] @@ -493,7 +506,7 @@ if self.converters is None: self._setup(cppthis) self.executor.set_item(self.space, w_item) # TODO: what about threads? - CPPMethod.call(self, cppthis, args_w) + CPPMethod.call(self, cppthis, args_w, useffi) class W_CPPOverload(W_Root): @@ -501,7 +514,7 @@ collection of (possibly) overloaded methods or functions. It calls these in order and deals with error handling and reporting.""" - _attrs_ = ['space', 'scope', 'functions'] + _attrs_ = ['space', 'scope', 'functions', 'flags'] _immutable_fields_ = ['scope', 'functions[*]'] def __init__(self, space, declaring_scope, functions): @@ -510,6 +523,19 @@ assert len(functions) from rpython.rlib import debug self.functions = debug.make_sure_not_resized(functions) + self.flags = 0 + self.flags |= OVERLOAD_FLAGS_USE_FFI + + # allow user to determine ffi use rules per overload + def fget_useffi(self, space): + return space.newbool(bool(self.flags & OVERLOAD_FLAGS_USE_FFI)) + + @unwrap_spec(value=bool) + def fset_useffi(self, space, value): + if space.is_true(value): + self.flags |= OVERLOAD_FLAGS_USE_FFI + else: + self.flags &= ~OVERLOAD_FLAGS_USE_FFI @jit.elidable_promote() def is_static(self): @@ -540,7 +566,7 @@ for i in range(len(self.functions)): cppyyfunc = self.functions[i] try: - return cppyyfunc.call(cppthis, args_w) + return cppyyfunc.call(cppthis, args_w, self.flags & OVERLOAD_FLAGS_USE_FFI) except Exception: pass @@ -553,7 +579,7 @@ for i in range(len(self.functions)): cppyyfunc = self.functions[i] try: - return cppyyfunc.call(cppthis, args_w) + return cppyyfunc.call(cppthis, args_w, self.flags & OVERLOAD_FLAGS_USE_FFI) except OperationError as e: # special case if there's just one function, to prevent clogging the error message if len(self.functions) == 1: @@ -588,6 +614,7 @@ 'CPPOverload', is_static = interp2app(W_CPPOverload.is_static), call = interp2app(W_CPPOverload.call), + __useffi__ = GetSetProperty(W_CPPOverload.fget_useffi, W_CPPOverload.fset_useffi), prototype = interp2app(W_CPPOverload.prototype), ) @@ -611,7 +638,7 @@ self.scope.name) w_result = W_CPPOverload.call(self, w_cppinstance, args_w) newthis = rffi.cast(capi.C_OBJECT, self.space.uint_w(w_result)) - cppinstance = self.space.interp_w(W_CPPClass, w_cppinstance, can_be_None=True) + cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=True) if cppinstance is not None: cppinstance._rawobject = newthis memory_regulator.register(cppinstance) @@ -642,17 +669,18 @@ class W_CPPBoundMethod(W_Root): - _attrs_ = ['cppthis', 'method'] + _attrs_ = ['cppthis', 'method', 'useffi'] - def __init__(self, cppthis, method): + def __init__(self, cppthis, method, useffi): self.cppthis = cppthis self.method = method + self.useffi = useffi def __call__(self, args_w): - return self.method.bound_call(self.cppthis, args_w) + return self.method.bound_call(self.cppthis, args_w, self.useffi) def __repr__(self): - return "W_CPPBoundMethod(%s)" % [f.prototype() for f in self.functions] + return "W_CPPBoundMethod(%s)" % self.method.prototype() W_CPPBoundMethod.typedef = TypeDef( 'CPPBoundMethod', @@ -682,7 +710,7 @@ return offset def get(self, w_cppinstance, w_pycppclass): - cppinstance = self.space.interp_w(W_CPPClass, w_cppinstance, can_be_None=True) + cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=True) if not cppinstance: raise oefmt(self.space.w_AttributeError, "attribute access requires an instance") @@ -690,7 +718,7 @@ return self.converter.from_memory(self.space, w_cppinstance, w_pycppclass, offset) def set(self, w_cppinstance, w_value): - cppinstance = self.space.interp_w(W_CPPClass, w_cppinstance, can_be_None=True) + cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=True) if not cppinstance: raise oefmt(self.space.w_AttributeError, "attribute access requires an instance") @@ -845,24 +873,11 @@ return self.space.w_True def ns__dir__(self): - # Collect a list of everything (currently) available in the namespace. - # The backend can filter by returning empty strings. Special care is - # taken for functions, which need not be unique (overloading). - alldir = [] - for i in range(capi.c_num_scopes(self.space, self)): - sname = capi.c_scope_name(self.space, self, i) - if sname: alldir.append(self.space.newtext(sname)) - allmeth = {} - for i in range(capi.c_num_methods(self.space, self)): - idx = capi.c_method_index_at(self.space, self, i) - mname = capi.c_method_name(self.space, self, idx) - if mname: allmeth.setdefault(mname, 0) - for m in allmeth.keys(): - alldir.append(self.space.newtext(m)) - for i in range(capi.c_num_datamembers(self.space, self)): - dname = capi.c_datamember_name(self.space, self, i) - if dname: alldir.append(self.space.newtext(dname)) - return self.space.newlist(alldir) + alldir = capi.c_get_all_cpp_names(self.space, self) + w_alldir = self.space.newlist([]) + for name in alldir: + w_alldir.append(self.space.newtext(name)) + return w_alldir def missing_attribute_error(self, name): return oefmt(self.space.w_AttributeError, @@ -890,8 +905,7 @@ def _build_methods(self): assert len(self.methods) == 0 methods_temp = {} - for i in range(capi.c_num_methods(self.space, self)): - idx = capi.c_method_index_at(self.space, self, i) + for idx in range(capi.c_num_methods(self.space, self)): if capi.c_is_constructor(self.space, self, idx): pyname = '__init__' else: @@ -1029,7 +1043,7 @@ W_CPPComplexClassDecl.typedef.acceptable_as_base_class = False -class W_CPPClass(W_Root): +class W_CPPInstance(W_Root): _attrs_ = ['space', 'clsdecl', '_rawobject', 'flags', 'finalizer_registered'] _immutable_fields_ = ['clsdecl'] @@ -1109,8 +1123,8 @@ # find a global overload in gbl, in __gnu_cxx (for iterators), or in the # scopes of the argument classes (TODO: implement that last option) try: - # TODO: expecting w_other to be an W_CPPClass is too limiting - other = self.space.interp_w(W_CPPClass, w_other, can_be_None=False) + # TODO: expecting w_other to be an W_CPPInstance is too limiting + other = self.space.interp_w(W_CPPInstance, w_other, can_be_None=False) for name in ["", "__gnu_cxx", "__1"]: nss = scope_byname(self.space, name) meth_idx = capi.c_get_global_operator( @@ -1132,7 +1146,7 @@ # fallback 2: direct pointer comparison (the class comparison is needed since # the first data member in a struct and the struct have the same address) - other = self.space.interp_w(W_CPPClass, w_other, can_be_None=False) # TODO: factor out + other = self.space.interp_w(W_CPPInstance, w_other, can_be_None=False) # TODO: factor out iseq = (self._rawobject == other._rawobject) and (self.clsdecl == other.clsdecl) return self.space.newbool(iseq) @@ -1176,19 +1190,19 @@ if self.flags & INSTANCE_FLAGS_PYTHON_OWNS: self.destruct() -W_CPPClass.typedef = TypeDef( - 'CPPClass', - __python_owns__ = GetSetProperty(W_CPPClass.fget_python_owns, W_CPPClass.fset_python_owns), - __init__ = interp2app(W_CPPClass.instance__init__), - __eq__ = interp2app(W_CPPClass.instance__eq__), - __ne__ = interp2app(W_CPPClass.instance__ne__), - __nonzero__ = interp2app(W_CPPClass.instance__nonzero__), - __len__ = interp2app(W_CPPClass.instance__len__), - __cmp__ = interp2app(W_CPPClass.instance__cmp__), - __repr__ = interp2app(W_CPPClass.instance__repr__), - __destruct__ = interp2app(W_CPPClass.destruct), +W_CPPInstance.typedef = TypeDef( + 'CPPInstance', + __python_owns__ = GetSetProperty(W_CPPInstance.fget_python_owns, W_CPPInstance.fset_python_owns), + __init__ = interp2app(W_CPPInstance.instance__init__), + __eq__ = interp2app(W_CPPInstance.instance__eq__), + __ne__ = interp2app(W_CPPInstance.instance__ne__), + __nonzero__ = interp2app(W_CPPInstance.instance__nonzero__), + __len__ = interp2app(W_CPPInstance.instance__len__), + __cmp__ = interp2app(W_CPPInstance.instance__cmp__), + __repr__ = interp2app(W_CPPInstance.instance__repr__), + __destruct__ = interp2app(W_CPPInstance.destruct), ) -W_CPPClass.typedef.acceptable_as_base_class = True +W_CPPInstance.typedef.acceptable_as_base_class = True class MemoryRegulator: @@ -1200,7 +1214,7 @@ # Note that for now, the associated test carries an m_padding to make # a difference in the addresses. def __init__(self): - self.objects = rweakref.RWeakValueDictionary(int, W_CPPClass) + self.objects = rweakref.RWeakValueDictionary(int, W_CPPInstance) def register(self, obj): if not obj._rawobject: @@ -1266,8 +1280,8 @@ return obj # fresh creation - w_cppinstance = space.allocate_instance(W_CPPClass, w_pycppclass) - cppinstance = space.interp_w(W_CPPClass, w_cppinstance, can_be_None=False) + w_cppinstance = space.allocate_instance(W_CPPInstance, w_pycppclass) + cppinstance = space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=False) cppinstance.__init__(space, clsdecl, rawobject, is_ref, python_owns) memory_regulator.register(cppinstance) return w_cppinstance @@ -1311,7 +1325,7 @@ def move(space, w_obj): """Casts the given instance into an C++-style rvalue.""" - obj = space.interp_w(W_CPPClass, w_obj, can_be_None=True) + obj = space.interp_w(W_CPPInstance, w_obj, can_be_None=True) if obj: obj.flags |= INSTANCE_FLAGS_IS_R_VALUE return w_obj diff --git a/pypy/module/_cppyy/pythonify.py b/pypy/module/_cppyy/pythonify.py --- a/pypy/module/_cppyy/pythonify.py +++ b/pypy/module/_cppyy/pythonify.py @@ -7,7 +7,7 @@ # Metaclasses are needed to store C++ static data members as properties. Since # the interp-level does not support metaclasses, they are created at app-level. # These are the metaclass base classes: -class CPPMetaScope(type): +class CPPScope(type): def __getattr__(self, name): try: return get_scoped_pycppitem(self, name) # will cache on self @@ -15,11 +15,11 @@ raise AttributeError("%s object has no attribute '%s' (details: %s)" % (self, name, str(e))) -class CPPMetaNamespace(CPPMetaScope): +class CPPMetaNamespace(CPPScope): def __dir__(self): return self.__cppdecl__.__dir__() -class CPPMetaClass(CPPMetaScope): +class CPPClass(CPPScope): pass # namespace base class (class base class defined in _init_pythonify) @@ -173,7 +173,7 @@ # get a list of base classes for class creation bases = [get_pycppclass(base) for base in decl.get_base_names()] if not bases: - bases = [CPPClass,] + bases = [CPPInstance,] else: # it's possible that the required class now has been built if one of # the base classes uses it in e.g. a function interface @@ -214,7 +214,7 @@ # create a metaclass to allow properties (for static data write access) metabases = [type(base) for base in bases] - metacpp = type(CPPMetaScope)(cl_name+'_meta', _drop_cycles(metabases), d_meta) + metacpp = type(CPPScope)(cl_name+'_meta', _drop_cycles(metabases), d_meta) # create the python-side C++ class pycls = metacpp(cl_name, _drop_cycles(bases), d_class) @@ -412,11 +412,11 @@ # at pypy-c startup, rather than on the "import _cppyy" statement import _cppyy - # root of all proxy classes: CPPClass in pythonify exists to combine the - # CPPMetaScope metaclass with the interp-level CPPClassBase - global CPPClass - class CPPClass(_cppyy.CPPClassBase): - __metaclass__ = CPPMetaScope + # root of all proxy classes: CPPInstance in pythonify exists to combine + # the CPPScope metaclass with the interp-level CPPInstanceBase + global CPPInstance + class CPPInstance(_cppyy.CPPInstanceBase): + __metaclass__ = CPPScope pass # class generator callback @@ -438,9 +438,8 @@ gbl.std.move = _cppyy.move # install a type for enums to refer to - # TODO: this is correct for C++98, not for C++11 and in general there will - # be the same issue for all typedef'd builtin types setattr(gbl, 'internal_enum_type_t', int) + setattr(gbl, 'unsigned int', int) # if resolved # install for user access _cppyy.gbl = gbl diff --git a/pypy/module/_cppyy/src/dummy_backend.cxx b/pypy/module/_cppyy/src/dummy_backend.cxx --- a/pypy/module/_cppyy/src/dummy_backend.cxx +++ b/pypy/module/_cppyy/src/dummy_backend.cxx @@ -407,11 +407,9 @@ /* name to opaque C++ scope representation -------------------------------- */ -int cppyy_num_scopes(cppyy_scope_t handle) { - return 0; -} - char* cppyy_resolve_name(const char* cppitem_name) { + if (cppyy_is_enum(cppitem_name)) + return cppstring_to_cstring("internal_enum_type_t"); return cppstring_to_cstring(cppitem_name); } @@ -851,10 +849,13 @@ return (cppyy_object_t)result; } -cppyy_funcaddr_t cppyy_get_function_address(cppyy_scope_t /* scope */, cppyy_index_t /* idx */) { +cppyy_funcaddr_t cppyy_function_address_from_index(cppyy_scope_t /* scope */, cppyy_index_t /* idx */) { return (cppyy_funcaddr_t)0; } +cppyy_funcaddr_t cppyy_function_address_from_method(cppyy_method_t /* method */) { + return (cppyy_funcaddr_t)0; +} /* handling of function argument buffer ----------------------------------- */ void* cppyy_allocate_function_args(int nargs) { @@ -926,10 +927,6 @@ return s_scopes[handle].m_methods.size(); } -cppyy_index_t cppyy_method_index_at(cppyy_scope_t /* scope */, int imeth) { - return (cppyy_index_t)imeth; -} - char* cppyy_method_name(cppyy_scope_t handle, cppyy_index_t method_index) { return cppstring_to_cstring(s_scopes[handle].m_methods[(int)method_index].m_name); } @@ -978,8 +975,17 @@ return (cppyy_method_t)0; } +cppyy_index_t cppyy_get_global_operator(cppyy_scope_t /* scope */, + cppyy_scope_t /* lc */, cppyy_scope_t /* rc */, const char* /* op */) { + return (cppyy_index_t)-1; +} + /* method properties ----------------------------------------------------- */ +int cppyy_is_publicmethod(cppyy_type_t /* handle */, cppyy_index_t /* method_index */) { + return 1; +} + int cppyy_is_constructor(cppyy_type_t handle, cppyy_index_t method_index) { if (s_scopes.find(handle) != s_scopes.end()) return s_scopes[handle].m_methods[method_index].m_type == kConstructor; @@ -987,6 +993,10 @@ return 0; } +int cppyy_is_destructor(cppyy_type_t /* handle */, cppyy_index_t /* method_index */) { + return 0; +} + int cppyy_is_staticmethod(cppyy_type_t handle, cppyy_index_t method_index) { if (s_scopes.find(handle) != s_scopes.end()) return s_scopes[handle].m_methods[method_index].m_type == kStatic; @@ -1014,14 +1024,22 @@ /* data member properties ------------------------------------------------ */ -int cppyy_is_publicdata(cppyy_scope_t handle, int idatambr) { +int cppyy_is_publicdata(cppyy_scope_t /* handle */, cppyy_index_t /* idatambr */) { return 1; } -int cppyy_is_staticdata(cppyy_scope_t handle, int idatambr) { +int cppyy_is_staticdata(cppyy_scope_t handle, cppyy_index_t idatambr) { return s_scopes[handle].m_datambrs[idatambr].m_isstatic; } +int cppyy_is_const_data(cppyy_scope_t /* handle */, cppyy_index_t /* idatambr */) { + return 0; +} + +int cppyy_is_enum_data(cppyy_scope_t /* handle */, cppyy_index_t /* idatambr */) { + return 0; +} + /* misc helpers ----------------------------------------------------------- */ #if defined(_MSC_VER) diff --git a/pypy/module/_cppyy/test/Makefile b/pypy/module/_cppyy/test/Makefile --- a/pypy/module/_cppyy/test/Makefile +++ b/pypy/module/_cppyy/test/Makefile @@ -14,7 +14,7 @@ HASGENREFLEX:=$(shell command -v genreflex 2> /dev/null) -cppflags=-std=c++11 -O3 -m64 -fPIC -rdynamic +cppflags=-std=c++14 -O3 -m64 -fPIC -rdynamic ifdef HASGENREFLEX genreflex_flags:=$(shell genreflex --cppflags) cppflags+=$(genreflex_flags) diff --git a/pypy/module/_cppyy/test/advancedcpp.cxx b/pypy/module/_cppyy/test/advancedcpp.cxx --- a/pypy/module/_cppyy/test/advancedcpp.cxx +++ b/pypy/module/_cppyy/test/advancedcpp.cxx @@ -1,5 +1,7 @@ #include "advancedcpp.h" +#include + // for testing of default arguments #define IMPLEMENT_DEFAULTER_CLASS(type, tname) \ @@ -112,3 +114,13 @@ std::string overload_the_other_way::gime() { return "aap"; } int overload_the_other_way::gime() const { return 1; } + + +// exception handling testing +void Thrower::throw_anything() { + throw 1; +} + +void Thrower::throw_exception() { + throw std::runtime_error("C++ function failed"); +} diff --git a/pypy/module/_cppyy/test/advancedcpp.h b/pypy/module/_cppyy/test/advancedcpp.h --- a/pypy/module/_cppyy/test/advancedcpp.h +++ b/pypy/module/_cppyy/test/advancedcpp.h @@ -59,7 +59,7 @@ class a_class { // for esoteric inheritance testing public: a_class() { m_a = 1; m_da = 1.1; } - ~a_class() {} + virtual ~a_class() {} virtual int get_value() = 0; public: @@ -221,6 +221,7 @@ //=========================================================================== class some_abstract_class { // to test abstract class handling public: + virtual ~some_abstract_class() {} virtual void a_virtual_method() = 0; }; @@ -399,3 +400,11 @@ std::string gime(); int gime() const; }; + + +//=========================================================================== +class Thrower { // exception handling testing +public: + void throw_anything(); + void throw_exception(); +}; diff --git a/pypy/module/_cppyy/test/advancedcpp.xml b/pypy/module/_cppyy/test/advancedcpp.xml --- a/pypy/module/_cppyy/test/advancedcpp.xml +++ b/pypy/module/_cppyy/test/advancedcpp.xml @@ -57,4 +57,6 @@ + + diff --git a/pypy/module/_cppyy/test/conftest.py b/pypy/module/_cppyy/test/conftest.py --- a/pypy/module/_cppyy/test/conftest.py +++ b/pypy/module/_cppyy/test/conftest.py @@ -55,7 +55,7 @@ separate_module_files=[srcpath.join('dummy_backend.cxx')], include_dirs=[incpath, tstpath, cdir], compile_extra=['-DRPY_EXTERN=RPY_EXPORTED', '-DCPPYY_DUMMY_BACKEND', - '-fno-strict-aliasing', '-std=c++11'], + '-fno-strict-aliasing', '-std=c++14'], use_cpp_linker=True, ) @@ -65,7 +65,7 @@ outputfilename='libcppyy_dummy_backend', standalone=False) except CompilationError as e: - if '-std=c++11' in str(e): + if '-std=c++14' in str(e): global disabled disabled = str(e) return diff --git a/pypy/module/_cppyy/test/fragile.h b/pypy/module/_cppyy/test/fragile.h --- a/pypy/module/_cppyy/test/fragile.h +++ b/pypy/module/_cppyy/test/fragile.h @@ -30,6 +30,7 @@ void overload(int, no_such_class* p = 0) {} }; + static const int dummy_location = 0xdead; class E { @@ -105,6 +106,7 @@ class M { public: + virtual ~M() {} enum E1 { kOnce=42 }; enum E2 { kTwice=12 }; }; diff --git a/pypy/module/_cppyy/test/test_advancedcpp.py b/pypy/module/_cppyy/test/test_advancedcpp.py --- a/pypy/module/_cppyy/test/test_advancedcpp.py +++ b/pypy/module/_cppyy/test/test_advancedcpp.py @@ -656,3 +656,22 @@ # TODO: currently fails b/c double** not understood as &double* #assert cppyy.gbl.my_global_ptr[0] == 1234. + def test22_exceptions(self): + """Catching of C++ exceptions""" + + import _cppyy as cppyy + Thrower = cppyy.gbl.Thrower + + # TODO: clean up this interface: + Thrower.__cppdecl__.get_overload('throw_anything').__useffi__ = False + Thrower.__cppdecl__.get_overload('throw_exception').__useffi__ = False + + t = Thrower() + + assert raises(Exception, t.throw_anything) + assert raises(Exception, t.throw_exception) + + try: + t.throw_exception() + except Exception, e: + "C++ function failed" in str(e) diff --git a/pypy/module/_cppyy/test/test_overloads.py b/pypy/module/_cppyy/test/test_overloads.py --- a/pypy/module/_cppyy/test/test_overloads.py +++ b/pypy/module/_cppyy/test/test_overloads.py @@ -15,7 +15,6 @@ spaceconfig = dict(usemodules=['_cppyy', '_rawffi', 'itertools']) def setup_class(cls): - env = os.environ cls.w_test_dct = cls.space.newtext(test_dct) cls.w_overloads = cls.space.appexec([], """(): import ctypes diff --git a/pypy/module/_cppyy/test/test_zjit.py b/pypy/module/_cppyy/test/test_zjit.py --- a/pypy/module/_cppyy/test/test_zjit.py +++ b/pypy/module/_cppyy/test/test_zjit.py @@ -137,6 +137,7 @@ executor.get_executor(self, 'int').__class__.c_stubcall = staticmethod(c_call_i) self.w_AttributeError = FakeException(self, "AttributeError") + self.w_Exception = FakeException(self, "Exception") self.w_KeyError = FakeException(self, "KeyError") self.w_NotImplementedError = FakeException(self, "NotImplementedError") self.w_ReferenceError = FakeException(self, "ReferenceError") @@ -282,7 +283,7 @@ inst = interp_cppyy._bind_object(space, FakeInt(0), cls, True) cls.get_overload("__init__").call(inst, [FakeInt(0)]) cppmethod = cls.get_overload(method_name) - assert isinstance(inst, interp_cppyy.W_CPPClass) + assert isinstance(inst, interp_cppyy.W_CPPInstance) i = 10 while i > 0: drv.jit_merge_point(inst=inst, cppmethod=cppmethod, i=i) From pypy.commits at gmail.com Tue Apr 24 02:07:32 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 23 Apr 2018 23:07:32 -0700 (PDT) Subject: [pypy-commit] pypy default: update release notes to include cppyy Message-ID: <5adec9a4.1c69fb81.1d53.f59b@mx.google.com> Author: Matti Picus Branch: Changeset: r94436:53d84a6caf3c Date: 2018-04-24 08:58 +0300 http://bitbucket.org/pypy/pypy/changeset/53d84a6caf3c/ Log: update release notes to include cppyy diff --git a/pypy/doc/release-v6.0.0.rst b/pypy/doc/release-v6.0.0.rst --- a/pypy/doc/release-v6.0.0.rst +++ b/pypy/doc/release-v6.0.0.rst @@ -20,16 +20,18 @@ The GC now has `hooks`_ to gain more insights into its performance -The Windows PyPy3.5 release is still considered beta-quality. There are open -issues with unicode handling especially around system calls and c-extensions. +The Matplotlib TkAgg backend now works with PyPy, as do pygame and pygobject_. -The Matplotlib TkAgg backend now works with PyPy, as do pygame and pygobject_. +We updated the `cffi`_ module included in PyPy to version 1.11.5, and the +`cppyy`_ backend to 0.6.0. Please use these to wrap your C and C++ code, +respectively, for a JIT friendly experience. As always, this release is 100% compatible with the previous one and fixed several issues and bugs raised by the growing community of PyPy users. We strongly recommend updating. -We updated the cffi module included in PyPy to version 1.11.5 +The Windows PyPy3.5 release is still considered beta-quality. There are open +issues with unicode handling especially around system calls and c-extensions. The utf8 branch that changes internal representation of unicode to utf8 did not make it into the release, so there is still more goodness coming. We also @@ -56,6 +58,8 @@ .. _pygobject: https://lazka.github.io/posts/2018-04_pypy-pygobject/index.html .. _`syntax errors`: https://morepypy.blogspot.com/2018/04/improving-syntaxerror-in-pypy.html .. _`hooks`: gc_info.html#gc-hooks +.. _`cffi`: http://cffi.readthedocs.io +.. _`cppyy`: https://cppyy.readthedocs.io What is PyPy? ============= diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -3,7 +3,7 @@ ========================== .. this is a revision shortly after release-pypy-6.0.0 -.. startrev: ad79cc0ce9a8 +.. startrev: e50e11af23f1 diff --git a/pypy/doc/whatsnew-pypy2-6.0.0.rst b/pypy/doc/whatsnew-pypy2-6.0.0.rst --- a/pypy/doc/whatsnew-pypy2-6.0.0.rst +++ b/pypy/doc/whatsnew-pypy2-6.0.0.rst @@ -121,4 +121,8 @@ .. branch: gc-hook-better-timestamp -Improve GC hooksd +Improve GC hooks + +.. branch: cppyy-packaging + +Update backend to 0.6.0 and support exceptions through wrappers From pypy.commits at gmail.com Tue Apr 24 02:07:34 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 23 Apr 2018 23:07:34 -0700 (PDT) Subject: [pypy-commit] pypy default: add new contributors (thanks!!!) Message-ID: <5adec9a6.4d2f1c0a.c1476.91ad@mx.google.com> Author: Matti Picus Branch: Changeset: r94437:93fe826c4242 Date: 2018-04-24 09:03 +0300 http://bitbucket.org/pypy/pypy/changeset/93fe826c4242/ Log: add new contributors (thanks!!!) diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -247,6 +247,7 @@ Lukas Vacek Omer Katz Jacek Generowicz + Tomasz Dziopa Sylvain Thenault Jakub Stasiak Andrew Dalke @@ -307,6 +308,7 @@ Yury V. Zaytsev florinpapa Anders Sigfridsson + Matt Jackson Nikolay Zinov rafalgalczynski at gmail.com Joshua Gilbert diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -214,6 +214,7 @@ Lukas Vacek Omer Katz Jacek Generowicz + Tomasz Dziopa Sylvain Thenault Jakub Stasiak Andrew Dalke @@ -274,6 +275,7 @@ Yury V. Zaytsev florinpapa Anders Sigfridsson + Matt Jackson Nikolay Zinov rafalgalczynski at gmail.com Joshua Gilbert From pypy.commits at gmail.com Tue Apr 24 02:07:37 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 23 Apr 2018 23:07:37 -0700 (PDT) Subject: [pypy-commit] pypy release-pypy2.7-6.x: merge default into branch Message-ID: <5adec9a9.4b141c0a.bee12.5be5@mx.google.com> Author: Matti Picus Branch: release-pypy2.7-6.x Changeset: r94438:ab0b9caf307d Date: 2018-04-24 09:04 +0300 http://bitbucket.org/pypy/pypy/changeset/ab0b9caf307d/ Log: merge default into branch diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -247,6 +247,7 @@ Lukas Vacek Omer Katz Jacek Generowicz + Tomasz Dziopa Sylvain Thenault Jakub Stasiak Andrew Dalke @@ -307,6 +308,7 @@ Yury V. Zaytsev florinpapa Anders Sigfridsson + Matt Jackson Nikolay Zinov rafalgalczynski at gmail.com Joshua Gilbert diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -214,6 +214,7 @@ Lukas Vacek Omer Katz Jacek Generowicz + Tomasz Dziopa Sylvain Thenault Jakub Stasiak Andrew Dalke @@ -274,6 +275,7 @@ Yury V. Zaytsev florinpapa Anders Sigfridsson + Matt Jackson Nikolay Zinov rafalgalczynski at gmail.com Joshua Gilbert diff --git a/pypy/doc/release-v6.0.0.rst b/pypy/doc/release-v6.0.0.rst --- a/pypy/doc/release-v6.0.0.rst +++ b/pypy/doc/release-v6.0.0.rst @@ -20,16 +20,18 @@ The GC now has `hooks`_ to gain more insights into its performance -The Windows PyPy3.5 release is still considered beta-quality. There are open -issues with unicode handling especially around system calls and c-extensions. +The Matplotlib TkAgg backend now works with PyPy, as do pygame and pygobject_. -The Matplotlib TkAgg backend now works with PyPy, as do pygame and pygobject_. +We updated the `cffi`_ module included in PyPy to version 1.11.5, and the +`cppyy`_ backend to 0.6.0. Please use these to wrap your C and C++ code, +respectively, for a JIT friendly experience. As always, this release is 100% compatible with the previous one and fixed several issues and bugs raised by the growing community of PyPy users. We strongly recommend updating. -We updated the cffi module included in PyPy to version 1.11.5 +The Windows PyPy3.5 release is still considered beta-quality. There are open +issues with unicode handling especially around system calls and c-extensions. The utf8 branch that changes internal representation of unicode to utf8 did not make it into the release, so there is still more goodness coming. We also @@ -56,6 +58,8 @@ .. _pygobject: https://lazka.github.io/posts/2018-04_pypy-pygobject/index.html .. _`syntax errors`: https://morepypy.blogspot.com/2018/04/improving-syntaxerror-in-pypy.html .. _`hooks`: gc_info.html#gc-hooks +.. _`cffi`: http://cffi.readthedocs.io +.. _`cppyy`: https://cppyy.readthedocs.io What is PyPy? ============= diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -3,7 +3,7 @@ ========================== .. this is a revision shortly after release-pypy-6.0.0 -.. startrev: ad79cc0ce9a8 +.. startrev: e50e11af23f1 diff --git a/pypy/doc/whatsnew-pypy2-6.0.0.rst b/pypy/doc/whatsnew-pypy2-6.0.0.rst --- a/pypy/doc/whatsnew-pypy2-6.0.0.rst +++ b/pypy/doc/whatsnew-pypy2-6.0.0.rst @@ -121,4 +121,8 @@ .. branch: gc-hook-better-timestamp -Improve GC hooksd +Improve GC hooks + +.. branch: cppyy-packaging + +Update backend to 0.6.0 and support exceptions through wrappers diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -491,11 +491,17 @@ # 'action.fire()' happens to be called any time before # the corresponding perform(), the fire() has no # effect---which is the effect we want, because - # perform() will be called anyway. + # perform() will be called anyway. All such pending + # actions with _fired == True are still inside the old + # chained list. As soon as we reset _fired to False, + # we also reset _next to None and we are ready for + # another fire(). while action is not None: + next_action = action._next + action._next = None action._fired = False action.perform(ec, frame) - action._next, action = None, action._next + action = next_action self.action_dispatcher = action_dispatcher diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py --- a/pypy/interpreter/test/test_executioncontext.py +++ b/pypy/interpreter/test/test_executioncontext.py @@ -67,6 +67,47 @@ """) assert events == ['one'] + def test_fire_inside_perform(self): + # test what happens if we call AsyncAction.fire() while we are in the + # middle of an AsyncAction.perform(). In particular, this happens when + # PyObjectDeallocAction.fire() is called by rawrefcount: see issue + # 2805 + events = [] + + class Action1(executioncontext.AsyncAction): + _count = 0 + + def perform(self, ec, frame): + events.append('one') + if self._count == 0: + # a1 is no longer in the queue, so it will be enqueued + a1.fire() + # + # a2 is still in the queue, so the fire() is ignored and + # it's performed in its normal order, i.e. BEFORE a3 + a2.fire() + self._count += 1 + + class Action2(executioncontext.AsyncAction): + def perform(self, ec, frame): + events.append('two') + + class Action3(executioncontext.AsyncAction): + def perform(self, ec, frame): + events.append('three') + + space = self.space + a1 = Action1(space) + a2 = Action2(space) + a3 = Action3(space) + a1.fire() + a2.fire() + a3.fire() + space.appexec([], """(): + pass + """) + assert events == ['one', 'two', 'three', 'one'] + def test_periodic_action(self): from pypy.interpreter.executioncontext import ActionFlag diff --git a/pypy/module/_cppyy/__init__.py b/pypy/module/_cppyy/__init__.py --- a/pypy/module/_cppyy/__init__.py +++ b/pypy/module/_cppyy/__init__.py @@ -13,7 +13,7 @@ '_set_function_generator': 'interp_cppyy.set_function_generator', '_register_class' : 'interp_cppyy.register_class', '_get_nullptr' : 'interp_cppyy.get_nullptr', - 'CPPClassBase' : 'interp_cppyy.W_CPPClass', + 'CPPInstanceBase' : 'interp_cppyy.W_CPPInstance', 'addressof' : 'interp_cppyy.addressof', '_bind_object' : 'interp_cppyy._bind_object', 'bind_object' : 'interp_cppyy.bind_object', diff --git a/pypy/module/_cppyy/capi/loadable_capi.py b/pypy/module/_cppyy/capi/loadable_capi.py --- a/pypy/module/_cppyy/capi/loadable_capi.py +++ b/pypy/module/_cppyy/capi/loadable_capi.py @@ -121,11 +121,11 @@ # TODO: the following need to match up with the globally defined C_XYZ low-level # types (see capi/__init__.py), but by using strings here, that isn't guaranteed - c_opaque_ptr = state.c_ulong + c_opaque_ptr = state.c_ulong # not ptrdiff_t (which is signed) c_scope = c_opaque_ptr c_type = c_scope - c_object = c_opaque_ptr + c_object = c_opaque_ptr # not voidp (to stick with one handle type) c_method = c_opaque_ptr c_index = state.c_long c_index_array = state.c_voidp @@ -150,16 +150,17 @@ self.capi_call_ifaces = { # name to opaque C++ scope representation - 'num_scopes' : ([c_scope], c_int), - 'scope_name' : ([c_scope, c_int], c_ccharp), - 'resolve_name' : ([c_ccharp], c_ccharp), + 'resolve_enum' : ([c_ccharp], c_ccharp), 'get_scope' : ([c_ccharp], c_scope), 'actual_class' : ([c_type, c_object], c_type), + 'size_of_klass' : ([c_type], c_size_t), + 'size_of_type' : ([c_ccharp], c_size_t), # memory management 'allocate' : ([c_type], c_object), 'deallocate' : ([c_type, c_object], c_void), + 'construct' : ([c_type], c_object), 'destruct' : ([c_type, c_object], c_void), # method/function dispatching @@ -182,7 +183,8 @@ 'constructor' : ([c_method, c_object, c_int, c_voidp], c_object), 'call_o' : ([c_method, c_object, c_int, c_voidp, c_type], c_object), - 'get_function_address' : ([c_scope, c_index], c_voidp), # TODO: verify + 'function_address_from_index' : ([c_scope, c_index], c_voidp), # TODO: verify + 'function_address_from_method' : ([c_method], c_voidp), # id. # handling of function argument buffer 'allocate_function_args' : ([c_int], c_voidp), @@ -196,6 +198,8 @@ 'is_abstract' : ([c_type], c_int), 'is_enum' : ([c_ccharp], c_int), + 'get_all_cpp_names' : ([c_scope, c_voidp], c_voidp), # const char** + # type/class reflection information 'final_name' : ([c_type], c_ccharp), 'scoped_final_name' : ([c_type], c_ccharp), @@ -208,10 +212,10 @@ # method/function reflection information 'num_methods' : ([c_scope], c_int), - 'method_index_at' : ([c_scope, c_int], c_index), 'method_indices_from_name' : ([c_scope, c_ccharp], c_index_array), 'method_name' : ([c_scope, c_index], c_ccharp), + 'method_mangled_name' : ([c_scope, c_index], c_ccharp), 'method_result_type' : ([c_scope, c_index], c_ccharp), 'method_num_args' : ([c_scope, c_index], c_int), 'method_req_args' : ([c_scope, c_index], c_int), @@ -219,7 +223,9 @@ 'method_arg_default' : ([c_scope, c_index, c_int], c_ccharp), 'method_signature' : ([c_scope, c_index, c_int], c_ccharp), 'method_prototype' : ([c_scope, c_index, c_int], c_ccharp), + 'is_const_method' : ([c_method], c_int), + 'exists_method_template' : ([c_scope, c_ccharp], c_int), 'method_is_template' : ([c_scope, c_index], c_int), 'method_num_template_args' : ([c_scope, c_index], c_int), 'method_template_arg_name' : ([c_scope, c_index, c_index], c_ccharp), @@ -228,7 +234,9 @@ 'get_global_operator' : ([c_scope, c_scope, c_scope, c_ccharp], c_index), # method properties + 'is_public_method' : ([c_type, c_index], c_int), 'is_constructor' : ([c_type, c_index], c_int), + 'is_destructor' : ([c_type, c_index], c_int), 'is_staticmethod' : ([c_type, c_index], c_int), # data member reflection information @@ -236,12 +244,14 @@ 'datamember_name' : ([c_scope, c_int], c_ccharp), 'datamember_type' : ([c_scope, c_int], c_ccharp), 'datamember_offset' : ([c_scope, c_int], c_ptrdiff_t), - 'datamember_index' : ([c_scope, c_ccharp], c_int), # data member properties 'is_publicdata' : ([c_scope, c_int], c_int), 'is_staticdata' : ([c_scope, c_int], c_int), + 'is_const_data' : ([c_scope, c_int], c_int), + 'is_enum_data' : ([c_scope, c_int], c_int), + 'get_dimension_size' : ([c_scope, c_int, c_int], c_int), # misc helpers 'strtoll' : ([c_ccharp], c_llong), @@ -328,25 +338,27 @@ return rffi.cast(rffi.CCHARP, ptr) # name to opaque C++ scope representation ------------------------------------ -def c_num_scopes(space, cppscope): - return space.int_w(call_capi(space, 'num_scopes', [_ArgH(cppscope.handle)])) -def c_scope_name(space, cppscope, iscope): - args = [_ArgH(cppscope.handle), _ArgL(iscope)] - return charp2str_free(space, call_capi(space, 'scope_name', args)) - def c_resolve_name(space, name): return charp2str_free(space, call_capi(space, 'resolve_name', [_ArgS(name)])) +def c_resolve_enum(space, name): + return charp2str_free(space, call_capi(space, 'resolve_enum', [_ArgS(name)])) def c_get_scope_opaque(space, name): return rffi.cast(C_SCOPE, space.uint_w(call_capi(space, 'get_scope', [_ArgS(name)]))) def c_actual_class(space, cppclass, cppobj): args = [_ArgH(cppclass.handle), _ArgH(cppobj)] return rffi.cast(C_TYPE, space.uint_w(call_capi(space, 'actual_class', args))) +def c_size_of_klass(space, cppclass): + return _cdata_to_size_t(space, call_capi(space, 'size_of_klass', [_ArgH(cppclass.handle)])) +def c_size_of_type(space, name): + return _cdata_to_size_t(space, call_capi(space, 'size_of_type', [_ArgS(name)])) # memory management ---------------------------------------------------------- def c_allocate(space, cppclass): return _cdata_to_cobject(space, call_capi(space, 'allocate', [_ArgH(cppclass.handle)])) def c_deallocate(space, cppclass, cppobject): call_capi(space, 'deallocate', [_ArgH(cppclass.handle), _ArgH(cppobject)]) +def c_construct(space, cppclass): + return _cdata_to_cobject(space, call_capi(space, 'construct', [_ArgH(cppclass.handle)])) def c_destruct(space, cppclass, cppobject): call_capi(space, 'destruct', [_ArgH(cppclass.handle), _ArgH(cppobject)]) @@ -391,7 +403,7 @@ w_cstr = call_capi(space, 'call_s', [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs), _ArgP(rffi.cast(rffi.VOIDP, length))]) - cstr_len = intmask(length[0]) + cstr_len = int(intmask(length[0])) finally: lltype.free(length, flavor='raw') return _cdata_to_ccharp(space, w_cstr), cstr_len @@ -403,10 +415,13 @@ args = [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs), _ArgH(cppclass.handle)] return _cdata_to_cobject(space, call_capi(space, 'call_o', args)) -def c_get_function_address(space, cppscope, index): +def c_function_address_from_index(space, cppscope, index): args = [_ArgH(cppscope.handle), _ArgL(index)] return rffi.cast(C_FUNC_PTR, - _cdata_to_ptr(space, call_capi(space, 'get_function_address', args))) + _cdata_to_ptr(space, call_capi(space, 'function_address_from_index', args))) +def c_function_address_from_method(space, cppmethod): + return rffi.cast(C_FUNC_PTR, + _cdata_to_ptr(space, call_capi(space, 'function_address_from_method', _ArgH(cppmethod)))) # handling of function argument buffer --------------------------------------- def c_allocate_function_args(space, size): @@ -430,6 +445,23 @@ def c_is_enum(space, name): return space.bool_w(call_capi(space, 'is_enum', [_ArgS(name)])) +def c_get_all_cpp_names(space, scope): + sz = lltype.malloc(rffi.SIZE_TP.TO, 1, flavor='raw', zero=True) + try: + args = [_ArgH(scope.handle), _ArgP(rffi.cast(rffi.VOIDP, sz))] + rawnames = rffi.cast(rffi.CCHARPP, + _cdata_to_ptr(space, call_capi(space, 'get_all_cpp_names', args))) + count = int(intmask(sz[0])) + finally: + lltype.free(sz, flavor='raw') + allnames = [] + for i in range(count): + pystr = rffi.charp2str(rawnames[i]) + c_free(space, rffi.cast(rffi.VOIDP, rawnames[i])) # c_free defined below + allnames.append(pystr) + c_free(space, rffi.cast(rffi.VOIDP, rawnames)) # id. + return allnames + # type/class reflection information ------------------------------------------ def c_final_name(space, cpptype): return charp2str_free(space, call_capi(space, 'final_name', [_ArgH(cpptype)])) @@ -462,13 +494,10 @@ def c_num_methods(space, cppscope): args = [_ArgH(cppscope.handle)] return space.int_w(call_capi(space, 'num_methods', args)) -def c_method_index_at(space, cppscope, imethod): - args = [_ArgH(cppscope.handle), _ArgL(imethod)] - return space.int_w(call_capi(space, 'method_index_at', args)) def c_method_indices_from_name(space, cppscope, name): args = [_ArgH(cppscope.handle), _ArgS(name)] indices = rffi.cast(C_INDEX_ARRAY, - _cdata_to_ptr(space, call_capi(space, 'method_indices_from_name', args))) + _cdata_to_ptr(space, call_capi(space, 'method_indices_from_name', args))) if not indices: return [] py_indices = [] @@ -506,6 +535,9 @@ args = [_ArgH(cppscope.handle), _ArgL(index), _ArgL(show_formalargs)] return charp2str_free(space, call_capi(space, 'method_prototype', args)) +def c_exists_method_template(space, cppscope, name): + args = [_ArgH(cppscope.handle), _ArgS(name)] + return space.bool_w(call_capi(space, 'exists_method_template', args)) def c_method_is_template(space, cppscope, index): args = [_ArgH(cppscope.handle), _ArgL(index)] return space.bool_w(call_capi(space, 'method_is_template', args)) @@ -531,9 +563,15 @@ return rffi.cast(WLAVC_INDEX, -1) # method properties ---------------------------------------------------------- +def c_is_public_method(space, cppclass, index): + args = [_ArgH(cppclass.handle), _ArgL(index)] + return space.bool_w(call_capi(space, 'is_public_method', args)) def c_is_constructor(space, cppclass, index): args = [_ArgH(cppclass.handle), _ArgL(index)] return space.bool_w(call_capi(space, 'is_constructor', args)) +def c_is_destructor(space, cppclass, index): + args = [_ArgH(cppclass.handle), _ArgL(index)] + return space.bool_w(call_capi(space, 'is_destructor', args)) def c_is_staticmethod(space, cppclass, index): args = [_ArgH(cppclass.handle), _ArgL(index)] return space.bool_w(call_capi(space, 'is_staticmethod', args)) @@ -562,6 +600,15 @@ def c_is_staticdata(space, cppscope, datamember_index): args = [_ArgH(cppscope.handle), _ArgL(datamember_index)] return space.bool_w(call_capi(space, 'is_staticdata', args)) +def c_is_const_data(space, cppscope, datamember_index): + args = [_ArgH(cppscope.handle), _ArgL(datamember_index)] + return space.bool_w(call_capi(space, 'is_const_data', args)) +def c_is_enum_data(space, cppscope, datamember_index): + args = [_ArgH(cppscope.handle), _ArgL(datamember_index)] + return space.bool_w(call_capi(space, 'is_enum_data', args)) +def c_get_dimension_size(space, cppscope, datamember_index, dim_idx): + args = [_ArgH(cppscope.handle), _ArgL(datamember_index), _ArgL(dim_idx)] + return space.bool_w(call_capi(space, 'get_dimension_size', args)) # misc helpers --------------------------------------------------------------- def c_strtoll(space, svalue): @@ -585,7 +632,7 @@ try: w_cstr = call_capi(space, 'stdstring2charp', [_ArgH(cppstr), _ArgP(rffi.cast(rffi.VOIDP, sz))]) - cstr_len = intmask(sz[0]) + cstr_len = int(intmask(sz[0])) finally: lltype.free(sz, flavor='raw') return rffi.charpsize2str(_cdata_to_ccharp(space, w_cstr), cstr_len) @@ -607,7 +654,7 @@ """Return a python string taking into account \0""" from pypy.module._cppyy import interp_cppyy - cppstr = space.interp_w(interp_cppyy.W_CPPClass, w_self, can_be_None=False) + cppstr = space.interp_w(interp_cppyy.W_CPPInstance, w_self, can_be_None=False) return space.newtext(c_stdstring2charp(space, cppstr._rawobject)) # setup pythonizations for later use at run-time diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py --- a/pypy/module/_cppyy/converter.py +++ b/pypy/module/_cppyy/converter.py @@ -22,8 +22,8 @@ def get_rawobject(space, w_obj, can_be_None=True): - from pypy.module._cppyy.interp_cppyy import W_CPPClass - cppinstance = space.interp_w(W_CPPClass, w_obj, can_be_None=can_be_None) + from pypy.module._cppyy.interp_cppyy import W_CPPInstance + cppinstance = space.interp_w(W_CPPInstance, w_obj, can_be_None=can_be_None) if cppinstance: rawobject = cppinstance.get_rawobject() assert lltype.typeOf(rawobject) == capi.C_OBJECT @@ -31,15 +31,15 @@ return capi.C_NULL_OBJECT def set_rawobject(space, w_obj, address): - from pypy.module._cppyy.interp_cppyy import W_CPPClass - cppinstance = space.interp_w(W_CPPClass, w_obj, can_be_None=True) + from pypy.module._cppyy.interp_cppyy import W_CPPInstance + cppinstance = space.interp_w(W_CPPInstance, w_obj, can_be_None=True) if cppinstance: assert lltype.typeOf(cppinstance._rawobject) == capi.C_OBJECT cppinstance._rawobject = rffi.cast(capi.C_OBJECT, address) def get_rawobject_nonnull(space, w_obj): - from pypy.module._cppyy.interp_cppyy import W_CPPClass - cppinstance = space.interp_w(W_CPPClass, w_obj, can_be_None=True) + from pypy.module._cppyy.interp_cppyy import W_CPPInstance + cppinstance = space.interp_w(W_CPPInstance, w_obj, can_be_None=True) if cppinstance: cppinstance._nullcheck() rawobject = cppinstance.get_rawobject() @@ -502,8 +502,8 @@ self.clsdecl = clsdecl def _unwrap_object(self, space, w_obj): - from pypy.module._cppyy.interp_cppyy import W_CPPClass - if isinstance(w_obj, W_CPPClass): + from pypy.module._cppyy.interp_cppyy import W_CPPInstance + if isinstance(w_obj, W_CPPInstance): from pypy.module._cppyy.interp_cppyy import INSTANCE_FLAGS_IS_R_VALUE if w_obj.flags & INSTANCE_FLAGS_IS_R_VALUE: # reject moves as all are explicit @@ -534,8 +534,8 @@ class InstanceMoveConverter(InstanceRefConverter): def _unwrap_object(self, space, w_obj): # moving is same as by-ref, but have to check that move is allowed - from pypy.module._cppyy.interp_cppyy import W_CPPClass, INSTANCE_FLAGS_IS_R_VALUE - if isinstance(w_obj, W_CPPClass): + from pypy.module._cppyy.interp_cppyy import W_CPPInstance, INSTANCE_FLAGS_IS_R_VALUE + if isinstance(w_obj, W_CPPInstance): if w_obj.flags & INSTANCE_FLAGS_IS_R_VALUE: w_obj.flags &= ~INSTANCE_FLAGS_IS_R_VALUE return InstanceRefConverter._unwrap_object(self, space, w_obj) @@ -598,8 +598,8 @@ raise FastCallNotPossible def finalize_call(self, space, w_obj, call_local): - from pypy.module._cppyy.interp_cppyy import W_CPPClass - assert isinstance(w_obj, W_CPPClass) + from pypy.module._cppyy.interp_cppyy import W_CPPInstance + assert isinstance(w_obj, W_CPPInstance) r = rffi.cast(rffi.VOIDPP, call_local) w_obj._rawobject = rffi.cast(capi.C_OBJECT, r[0]) @@ -617,8 +617,8 @@ InstanceConverter.__init__(self, space, cppclass) def _unwrap_object(self, space, w_obj): - from pypy.module._cppyy.interp_cppyy import W_CPPClass - if isinstance(w_obj, W_CPPClass): + from pypy.module._cppyy.interp_cppyy import W_CPPInstance + if isinstance(w_obj, W_CPPInstance): arg = InstanceConverter._unwrap_object(self, space, w_obj) return capi.c_stdstring2stdstring(space, arg) else: @@ -749,8 +749,6 @@ return InstancePtrPtrConverter(space, clsdecl) elif compound == "": return InstanceConverter(space, clsdecl) - elif capi.c_is_enum(space, clean_name): - return _converters['unsigned'](space, default) # 5) void* or void converter (which fails on use) if 0 <= compound.find('*'): diff --git a/pypy/module/_cppyy/executor.py b/pypy/module/_cppyy/executor.py --- a/pypy/module/_cppyy/executor.py +++ b/pypy/module/_cppyy/executor.py @@ -289,8 +289,6 @@ return InstancePtrExecutor(space, cppclass) elif compound == '**' or compound == '*&': return InstancePtrPtrExecutor(space, cppclass) - elif capi.c_is_enum(space, clean_name): - return _executors['internal_enum_type_t'](space, None) # 4) additional special cases if compound == '*': diff --git a/pypy/module/_cppyy/include/capi.h b/pypy/module/_cppyy/include/capi.h --- a/pypy/module/_cppyy/include/capi.h +++ b/pypy/module/_cppyy/include/capi.h @@ -8,26 +8,29 @@ extern "C" { #endif // ifdef __cplusplus - typedef unsigned long cppyy_scope_t; + typedef ptrdiff_t cppyy_scope_t; typedef cppyy_scope_t cppyy_type_t; - typedef unsigned long cppyy_object_t; - typedef unsigned long cppyy_method_t; + typedef void* cppyy_object_t; + typedef ptrdiff_t cppyy_method_t; + typedef long cppyy_index_t; typedef void* cppyy_funcaddr_t; + typedef unsigned long cppyy_exctype_t; + /* name to opaque C++ scope representation -------------------------------- */ RPY_EXTERN - int cppyy_num_scopes(cppyy_scope_t parent); + char* cppyy_resolve_name(const char* cppitem_name); RPY_EXTERN - char* cppyy_scope_name(cppyy_scope_t parent, cppyy_index_t iscope); - RPY_EXTERN - char* cppyy_resolve_name(const char* cppitem_name); + char* cppyy_resolve_enum(const char* enum_type); RPY_EXTERN cppyy_scope_t cppyy_get_scope(const char* scope_name); RPY_EXTERN cppyy_type_t cppyy_actual_class(cppyy_type_t klass, cppyy_object_t obj); RPY_EXTERN - size_t cppyy_size_of(cppyy_type_t klass); + size_t cppyy_size_of_klass(cppyy_type_t klass); + RPY_EXTERN + size_t cppyy_size_of_type(const char* type_name); /* memory management ------------------------------------------------------ */ RPY_EXTERN @@ -35,48 +38,53 @@ RPY_EXTERN void cppyy_deallocate(cppyy_type_t type, cppyy_object_t self); RPY_EXTERN + cppyy_object_t cppyy_construct(cppyy_type_t type); + RPY_EXTERN void cppyy_destruct(cppyy_type_t type, cppyy_object_t self); /* method/function dispatching -------------------------------------------- */ RPY_EXTERN - void cppyy_call_v(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); + void cppyy_call_v(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN unsigned char cppyy_call_b(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN - char cppyy_call_c(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); + char cppyy_call_c(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN - short cppyy_call_h(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); + short cppyy_call_h(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN - int cppyy_call_i(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); + int cppyy_call_i(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN - long cppyy_call_l(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); + long cppyy_call_l(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN long long cppyy_call_ll(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN - float cppyy_call_f(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); + float cppyy_call_f(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN double cppyy_call_d(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN long double cppyy_call_ld(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN - void* cppyy_call_r(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); + void* cppyy_call_r(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN - char* cppyy_call_s(cppyy_method_t method, cppyy_object_t self, int nargs, void* args, size_t* length); - + char* cppyy_call_s(cppyy_method_t method, cppyy_object_t self, int nargs, void* args, size_t* length); RPY_EXTERN cppyy_object_t cppyy_constructor(cppyy_method_t method, cppyy_type_t klass, int nargs, void* args); RPY_EXTERN + void cppyy_destructor(cppyy_type_t type, cppyy_object_t self); + RPY_EXTERN cppyy_object_t cppyy_call_o(cppyy_method_t method, cppyy_object_t self, int nargs, void* args, cppyy_type_t result_type); RPY_EXTERN - cppyy_funcaddr_t cppyy_get_function_address(cppyy_scope_t scope, cppyy_index_t idx); + cppyy_funcaddr_t cppyy_function_address_from_index(cppyy_scope_t scope, cppyy_index_t idx); + RPY_EXTERN + cppyy_funcaddr_t cppyy_function_address_from_method(cppyy_method_t method); /* handling of function argument buffer ----------------------------------- */ RPY_EXTERN - void* cppyy_allocate_function_args(int nargs); + void* cppyy_allocate_function_args(int nargs); RPY_EXTERN - void cppyy_deallocate_function_args(void* args); + void cppyy_deallocate_function_args(void* args); RPY_EXTERN size_t cppyy_function_arg_sizeof(); RPY_EXTERN @@ -92,6 +100,9 @@ RPY_EXTERN int cppyy_is_enum(const char* type_name); + RPY_EXTERN + const char** cppyy_get_all_cpp_names(cppyy_scope_t scope, size_t* count); + /* class reflection information ------------------------------------------- */ RPY_EXTERN char* cppyy_final_name(cppyy_type_t type); @@ -105,6 +116,10 @@ char* cppyy_base_name(cppyy_type_t type, int base_index); RPY_EXTERN int cppyy_is_subtype(cppyy_type_t derived, cppyy_type_t base); + RPY_EXTERN + int cppyy_smartptr_info(const char* name, cppyy_type_t* raw, cppyy_method_t* deref); + RPY_EXTERN + void cppyy_add_smartptr_type(const char* type_name); /* calculate offsets between declared and actual type, up-cast: direction > 0; down-cast: direction < 0 */ RPY_EXTERN @@ -114,8 +129,6 @@ RPY_EXTERN int cppyy_num_methods(cppyy_scope_t scope); RPY_EXTERN - cppyy_index_t cppyy_method_index_at(cppyy_scope_t scope, int imeth); - RPY_EXTERN cppyy_index_t* cppyy_method_indices_from_name(cppyy_scope_t scope, const char* name); RPY_EXTERN @@ -136,8 +149,12 @@ char* cppyy_method_signature(cppyy_scope_t scope, cppyy_index_t idx, int show_formalargs); RPY_EXTERN char* cppyy_method_prototype(cppyy_scope_t scope, cppyy_index_t idx, int show_formalargs); + RPY_EXTERN + int cppyy_is_const_method(cppyy_method_t); RPY_EXTERN + int cppyy_exists_method_template(cppyy_scope_t scope, const char* name); + RPY_EXTERN int cppyy_method_is_template(cppyy_scope_t scope, cppyy_index_t idx); RPY_EXTERN int cppyy_method_num_template_args(cppyy_scope_t scope, cppyy_index_t idx); @@ -169,15 +186,20 @@ char* cppyy_datamember_type(cppyy_scope_t scope, int datamember_index); RPY_EXTERN ptrdiff_t cppyy_datamember_offset(cppyy_scope_t scope, int datamember_index); - RPY_EXTERN int cppyy_datamember_index(cppyy_scope_t scope, const char* name); /* data member properties ------------------------------------------------- */ RPY_EXTERN - int cppyy_is_publicdata(cppyy_type_t type, int datamember_index); + int cppyy_is_publicdata(cppyy_type_t type, cppyy_index_t datamember_index); RPY_EXTERN - int cppyy_is_staticdata(cppyy_type_t type, int datamember_index); + int cppyy_is_staticdata(cppyy_type_t type, cppyy_index_t datamember_index); + RPY_EXTERN + int cppyy_is_const_data(cppyy_scope_t scope, cppyy_index_t idata); + RPY_EXTERN + int cppyy_is_enum_data(cppyy_scope_t scope, cppyy_index_t idata); + RPY_EXTERN + int cppyy_get_dimension_size(cppyy_scope_t scope, cppyy_index_t idata, int dimension); /* misc helpers ----------------------------------------------------------- */ RPY_EXTERN @@ -197,7 +219,7 @@ RPY_EXTERN const char* cppyy_stdvector_valuetype(const char* clname); RPY_EXTERN - size_t cppyy_stdvector_valuesize(const char* clname); + size_t cppyy_stdvector_valuesize(const char* clname); #ifdef __cplusplus } diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py --- a/pypy/module/_cppyy/interp_cppyy.py +++ b/pypy/module/_cppyy/interp_cppyy.py @@ -19,6 +19,9 @@ INSTANCE_FLAGS_IS_REF = 0x0002 INSTANCE_FLAGS_IS_R_VALUE = 0x0004 +OVERLOAD_FLAGS_USE_FFI = 0x0001 + + class FastCallNotPossible(Exception): pass @@ -174,7 +177,7 @@ @staticmethod def unpack_cppthis(space, w_cppinstance, declaring_scope): - cppinstance = space.interp_w(W_CPPClass, w_cppinstance, can_be_None=False) + cppinstance = space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=False) cppinstance._nullcheck() return cppinstance.get_cppthis(declaring_scope) @@ -186,7 +189,7 @@ return rffi.cast(rffi.VOIDP, loc_idx) @jit.unroll_safe - def call(self, cppthis, args_w): + def call(self, cppthis, args_w, useffi): jit.promote(self) assert lltype.typeOf(cppthis) == capi.C_OBJECT @@ -218,16 +221,23 @@ try: # attempt to call directly through ffi chain - if self._funcaddr: + if useffi and self._funcaddr: try: return self.do_fast_call(cppthis, args_w, call_local) except FastCallNotPossible: pass # can happen if converters or executor does not implement ffi # ffi chain must have failed; using stub functions instead - args = self.prepare_arguments(args_w, call_local) + args, stat = self.prepare_arguments(args_w, call_local) try: - return self.executor.execute(self.space, self.cppmethod, cppthis, len(args_w), args) + result = self.executor.execute( + self.space, self.cppmethod, cppthis, len(args_w), args) + if stat[0] != rffi.cast(rffi.ULONG, 0): + what = rffi.cast(rffi.CCHARP, stat[1]) + pywhat = rffi.charp2str(what) + capi.c_free(self.space, rffi.cast(rffi.VOIDP, what)) + raise OperationError(self.space.w_Exception, self.space.newtext(pywhat)) + return result finally: self.finalize_call(args, args_w, call_local) finally: @@ -322,7 +332,7 @@ # Each CPPMethod corresponds one-to-one to a C++ equivalent and cppthis # has been offset to the matching class. Hence, the libffi pointer is # uniquely defined and needs to be setup only once. - funcaddr = capi.c_get_function_address(self.space, self.scope, self.index) + funcaddr = capi.c_function_address_from_index(self.space, self.scope, self.index) if funcaddr and cppthis: # methods only for now state = self.space.fromcache(ffitypes.State) @@ -373,7 +383,10 @@ conv.free_argument(self.space, rffi.cast(capi.C_OBJECT, arg_j), loc_j) capi.c_deallocate_function_args(self.space, args) raise - return args + stat = rffi.cast(rffi.ULONGP, + lltype.direct_ptradd(rffi.cast(rffi.CCHARP, args), int(len(args_w))*stride)) + stat[0] = rffi.cast(rffi.ULONG, 0) + return args, stat @jit.unroll_safe def finalize_call(self, args, args_w, call_local): @@ -435,7 +448,7 @@ # TODO: might have to specialize for CPPTemplatedCall on CPPMethod/CPPFunction here CPPMethod.__init__(self, space, declaring_scope, method_index, arg_defs, args_required) - def call(self, cppthis, args_w): + def call(self, cppthis, args_w, useffi): assert lltype.typeOf(cppthis) == capi.C_OBJECT for i in range(len(args_w)): try: @@ -447,10 +460,10 @@ raise oefmt(self.space.w_TypeError, "non-matching template (got %s where %s expected)", s, self.templ_args[i]) - return W_CPPBoundMethod(cppthis, self) + return W_CPPBoundMethod(cppthis, self, useffi) - def bound_call(self, cppthis, args_w): - return CPPMethod.call(self, cppthis, args_w) + def bound_call(self, cppthis, args_w, useffi): + return CPPMethod.call(self, cppthis, args_w, useffi) def __repr__(self): return "CPPTemplatedCall: %s" % self.prototype() @@ -468,11 +481,11 @@ def unpack_cppthis(space, w_cppinstance, declaring_scope): return rffi.cast(capi.C_OBJECT, declaring_scope.handle) - def call(self, cppthis, args_w): + def call(self, cppthis, args_w, useffi): # Note: this does not return a wrapped instance, just a pointer to the # new instance; the overload must still wrap it before returning. Also, # cppthis is declaring_scope.handle (as per unpack_cppthis(), above). - return CPPMethod.call(self, cppthis, args_w) + return CPPMethod.call(self, cppthis, args_w, useffi) def __repr__(self): return "CPPConstructor: %s" % self.prototype() @@ -485,7 +498,7 @@ _immutable_ = True - def call(self, cppthis, args_w): + def call(self, cppthis, args_w, useffi): end = len(args_w)-1 if 0 <= end: w_item = args_w[end] @@ -493,7 +506,7 @@ if self.converters is None: self._setup(cppthis) self.executor.set_item(self.space, w_item) # TODO: what about threads? - CPPMethod.call(self, cppthis, args_w) + CPPMethod.call(self, cppthis, args_w, useffi) class W_CPPOverload(W_Root): @@ -501,7 +514,7 @@ collection of (possibly) overloaded methods or functions. It calls these in order and deals with error handling and reporting.""" - _attrs_ = ['space', 'scope', 'functions'] + _attrs_ = ['space', 'scope', 'functions', 'flags'] _immutable_fields_ = ['scope', 'functions[*]'] def __init__(self, space, declaring_scope, functions): @@ -510,6 +523,19 @@ assert len(functions) from rpython.rlib import debug self.functions = debug.make_sure_not_resized(functions) + self.flags = 0 + self.flags |= OVERLOAD_FLAGS_USE_FFI + + # allow user to determine ffi use rules per overload + def fget_useffi(self, space): + return space.newbool(bool(self.flags & OVERLOAD_FLAGS_USE_FFI)) + + @unwrap_spec(value=bool) + def fset_useffi(self, space, value): + if space.is_true(value): + self.flags |= OVERLOAD_FLAGS_USE_FFI + else: + self.flags &= ~OVERLOAD_FLAGS_USE_FFI @jit.elidable_promote() def is_static(self): @@ -540,7 +566,7 @@ for i in range(len(self.functions)): cppyyfunc = self.functions[i] try: - return cppyyfunc.call(cppthis, args_w) + return cppyyfunc.call(cppthis, args_w, self.flags & OVERLOAD_FLAGS_USE_FFI) except Exception: pass @@ -553,7 +579,7 @@ for i in range(len(self.functions)): cppyyfunc = self.functions[i] try: - return cppyyfunc.call(cppthis, args_w) + return cppyyfunc.call(cppthis, args_w, self.flags & OVERLOAD_FLAGS_USE_FFI) except OperationError as e: # special case if there's just one function, to prevent clogging the error message if len(self.functions) == 1: @@ -588,6 +614,7 @@ 'CPPOverload', is_static = interp2app(W_CPPOverload.is_static), call = interp2app(W_CPPOverload.call), + __useffi__ = GetSetProperty(W_CPPOverload.fget_useffi, W_CPPOverload.fset_useffi), prototype = interp2app(W_CPPOverload.prototype), ) @@ -611,7 +638,7 @@ self.scope.name) w_result = W_CPPOverload.call(self, w_cppinstance, args_w) newthis = rffi.cast(capi.C_OBJECT, self.space.uint_w(w_result)) - cppinstance = self.space.interp_w(W_CPPClass, w_cppinstance, can_be_None=True) + cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=True) if cppinstance is not None: cppinstance._rawobject = newthis memory_regulator.register(cppinstance) @@ -642,17 +669,18 @@ class W_CPPBoundMethod(W_Root): - _attrs_ = ['cppthis', 'method'] + _attrs_ = ['cppthis', 'method', 'useffi'] - def __init__(self, cppthis, method): + def __init__(self, cppthis, method, useffi): self.cppthis = cppthis self.method = method + self.useffi = useffi def __call__(self, args_w): - return self.method.bound_call(self.cppthis, args_w) + return self.method.bound_call(self.cppthis, args_w, self.useffi) def __repr__(self): - return "W_CPPBoundMethod(%s)" % [f.prototype() for f in self.functions] + return "W_CPPBoundMethod(%s)" % self.method.prototype() W_CPPBoundMethod.typedef = TypeDef( 'CPPBoundMethod', @@ -682,7 +710,7 @@ return offset def get(self, w_cppinstance, w_pycppclass): - cppinstance = self.space.interp_w(W_CPPClass, w_cppinstance, can_be_None=True) + cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=True) if not cppinstance: raise oefmt(self.space.w_AttributeError, "attribute access requires an instance") @@ -690,7 +718,7 @@ return self.converter.from_memory(self.space, w_cppinstance, w_pycppclass, offset) def set(self, w_cppinstance, w_value): - cppinstance = self.space.interp_w(W_CPPClass, w_cppinstance, can_be_None=True) + cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=True) if not cppinstance: raise oefmt(self.space.w_AttributeError, "attribute access requires an instance") @@ -845,24 +873,11 @@ return self.space.w_True def ns__dir__(self): - # Collect a list of everything (currently) available in the namespace. - # The backend can filter by returning empty strings. Special care is - # taken for functions, which need not be unique (overloading). - alldir = [] - for i in range(capi.c_num_scopes(self.space, self)): - sname = capi.c_scope_name(self.space, self, i) - if sname: alldir.append(self.space.newtext(sname)) - allmeth = {} - for i in range(capi.c_num_methods(self.space, self)): - idx = capi.c_method_index_at(self.space, self, i) - mname = capi.c_method_name(self.space, self, idx) - if mname: allmeth.setdefault(mname, 0) - for m in allmeth.keys(): - alldir.append(self.space.newtext(m)) - for i in range(capi.c_num_datamembers(self.space, self)): - dname = capi.c_datamember_name(self.space, self, i) - if dname: alldir.append(self.space.newtext(dname)) - return self.space.newlist(alldir) + alldir = capi.c_get_all_cpp_names(self.space, self) + w_alldir = self.space.newlist([]) + for name in alldir: + w_alldir.append(self.space.newtext(name)) + return w_alldir def missing_attribute_error(self, name): return oefmt(self.space.w_AttributeError, @@ -890,8 +905,7 @@ def _build_methods(self): assert len(self.methods) == 0 methods_temp = {} - for i in range(capi.c_num_methods(self.space, self)): - idx = capi.c_method_index_at(self.space, self, i) + for idx in range(capi.c_num_methods(self.space, self)): if capi.c_is_constructor(self.space, self, idx): pyname = '__init__' else: @@ -1029,7 +1043,7 @@ W_CPPComplexClassDecl.typedef.acceptable_as_base_class = False -class W_CPPClass(W_Root): +class W_CPPInstance(W_Root): _attrs_ = ['space', 'clsdecl', '_rawobject', 'flags', 'finalizer_registered'] _immutable_fields_ = ['clsdecl'] @@ -1109,8 +1123,8 @@ # find a global overload in gbl, in __gnu_cxx (for iterators), or in the # scopes of the argument classes (TODO: implement that last option) try: - # TODO: expecting w_other to be an W_CPPClass is too limiting - other = self.space.interp_w(W_CPPClass, w_other, can_be_None=False) + # TODO: expecting w_other to be an W_CPPInstance is too limiting + other = self.space.interp_w(W_CPPInstance, w_other, can_be_None=False) for name in ["", "__gnu_cxx", "__1"]: nss = scope_byname(self.space, name) meth_idx = capi.c_get_global_operator( @@ -1132,7 +1146,7 @@ # fallback 2: direct pointer comparison (the class comparison is needed since # the first data member in a struct and the struct have the same address) - other = self.space.interp_w(W_CPPClass, w_other, can_be_None=False) # TODO: factor out + other = self.space.interp_w(W_CPPInstance, w_other, can_be_None=False) # TODO: factor out iseq = (self._rawobject == other._rawobject) and (self.clsdecl == other.clsdecl) return self.space.newbool(iseq) @@ -1176,19 +1190,19 @@ if self.flags & INSTANCE_FLAGS_PYTHON_OWNS: self.destruct() -W_CPPClass.typedef = TypeDef( - 'CPPClass', - __python_owns__ = GetSetProperty(W_CPPClass.fget_python_owns, W_CPPClass.fset_python_owns), - __init__ = interp2app(W_CPPClass.instance__init__), - __eq__ = interp2app(W_CPPClass.instance__eq__), - __ne__ = interp2app(W_CPPClass.instance__ne__), - __nonzero__ = interp2app(W_CPPClass.instance__nonzero__), - __len__ = interp2app(W_CPPClass.instance__len__), - __cmp__ = interp2app(W_CPPClass.instance__cmp__), - __repr__ = interp2app(W_CPPClass.instance__repr__), - __destruct__ = interp2app(W_CPPClass.destruct), +W_CPPInstance.typedef = TypeDef( + 'CPPInstance', + __python_owns__ = GetSetProperty(W_CPPInstance.fget_python_owns, W_CPPInstance.fset_python_owns), + __init__ = interp2app(W_CPPInstance.instance__init__), + __eq__ = interp2app(W_CPPInstance.instance__eq__), + __ne__ = interp2app(W_CPPInstance.instance__ne__), + __nonzero__ = interp2app(W_CPPInstance.instance__nonzero__), + __len__ = interp2app(W_CPPInstance.instance__len__), + __cmp__ = interp2app(W_CPPInstance.instance__cmp__), + __repr__ = interp2app(W_CPPInstance.instance__repr__), + __destruct__ = interp2app(W_CPPInstance.destruct), ) -W_CPPClass.typedef.acceptable_as_base_class = True +W_CPPInstance.typedef.acceptable_as_base_class = True class MemoryRegulator: @@ -1200,7 +1214,7 @@ # Note that for now, the associated test carries an m_padding to make # a difference in the addresses. def __init__(self): - self.objects = rweakref.RWeakValueDictionary(int, W_CPPClass) + self.objects = rweakref.RWeakValueDictionary(int, W_CPPInstance) def register(self, obj): if not obj._rawobject: @@ -1266,8 +1280,8 @@ return obj # fresh creation - w_cppinstance = space.allocate_instance(W_CPPClass, w_pycppclass) - cppinstance = space.interp_w(W_CPPClass, w_cppinstance, can_be_None=False) + w_cppinstance = space.allocate_instance(W_CPPInstance, w_pycppclass) + cppinstance = space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=False) cppinstance.__init__(space, clsdecl, rawobject, is_ref, python_owns) memory_regulator.register(cppinstance) return w_cppinstance @@ -1311,7 +1325,7 @@ def move(space, w_obj): """Casts the given instance into an C++-style rvalue.""" - obj = space.interp_w(W_CPPClass, w_obj, can_be_None=True) + obj = space.interp_w(W_CPPInstance, w_obj, can_be_None=True) if obj: obj.flags |= INSTANCE_FLAGS_IS_R_VALUE return w_obj diff --git a/pypy/module/_cppyy/pythonify.py b/pypy/module/_cppyy/pythonify.py --- a/pypy/module/_cppyy/pythonify.py +++ b/pypy/module/_cppyy/pythonify.py @@ -7,7 +7,7 @@ # Metaclasses are needed to store C++ static data members as properties. Since # the interp-level does not support metaclasses, they are created at app-level. # These are the metaclass base classes: -class CPPMetaScope(type): +class CPPScope(type): def __getattr__(self, name): try: return get_scoped_pycppitem(self, name) # will cache on self @@ -15,11 +15,11 @@ raise AttributeError("%s object has no attribute '%s' (details: %s)" % (self, name, str(e))) -class CPPMetaNamespace(CPPMetaScope): +class CPPMetaNamespace(CPPScope): def __dir__(self): return self.__cppdecl__.__dir__() -class CPPMetaClass(CPPMetaScope): +class CPPClass(CPPScope): pass # namespace base class (class base class defined in _init_pythonify) @@ -173,7 +173,7 @@ # get a list of base classes for class creation bases = [get_pycppclass(base) for base in decl.get_base_names()] if not bases: - bases = [CPPClass,] + bases = [CPPInstance,] else: # it's possible that the required class now has been built if one of # the base classes uses it in e.g. a function interface @@ -214,7 +214,7 @@ # create a metaclass to allow properties (for static data write access) metabases = [type(base) for base in bases] - metacpp = type(CPPMetaScope)(cl_name+'_meta', _drop_cycles(metabases), d_meta) + metacpp = type(CPPScope)(cl_name+'_meta', _drop_cycles(metabases), d_meta) # create the python-side C++ class pycls = metacpp(cl_name, _drop_cycles(bases), d_class) @@ -412,11 +412,11 @@ # at pypy-c startup, rather than on the "import _cppyy" statement import _cppyy - # root of all proxy classes: CPPClass in pythonify exists to combine the - # CPPMetaScope metaclass with the interp-level CPPClassBase - global CPPClass - class CPPClass(_cppyy.CPPClassBase): - __metaclass__ = CPPMetaScope + # root of all proxy classes: CPPInstance in pythonify exists to combine + # the CPPScope metaclass with the interp-level CPPInstanceBase + global CPPInstance + class CPPInstance(_cppyy.CPPInstanceBase): + __metaclass__ = CPPScope pass # class generator callback @@ -438,9 +438,8 @@ gbl.std.move = _cppyy.move # install a type for enums to refer to - # TODO: this is correct for C++98, not for C++11 and in general there will - # be the same issue for all typedef'd builtin types setattr(gbl, 'internal_enum_type_t', int) + setattr(gbl, 'unsigned int', int) # if resolved # install for user access _cppyy.gbl = gbl diff --git a/pypy/module/_cppyy/src/dummy_backend.cxx b/pypy/module/_cppyy/src/dummy_backend.cxx --- a/pypy/module/_cppyy/src/dummy_backend.cxx +++ b/pypy/module/_cppyy/src/dummy_backend.cxx @@ -407,11 +407,9 @@ /* name to opaque C++ scope representation -------------------------------- */ -int cppyy_num_scopes(cppyy_scope_t handle) { - return 0; -} - char* cppyy_resolve_name(const char* cppitem_name) { + if (cppyy_is_enum(cppitem_name)) + return cppstring_to_cstring("internal_enum_type_t"); return cppstring_to_cstring(cppitem_name); } @@ -851,10 +849,13 @@ return (cppyy_object_t)result; } -cppyy_funcaddr_t cppyy_get_function_address(cppyy_scope_t /* scope */, cppyy_index_t /* idx */) { +cppyy_funcaddr_t cppyy_function_address_from_index(cppyy_scope_t /* scope */, cppyy_index_t /* idx */) { return (cppyy_funcaddr_t)0; } +cppyy_funcaddr_t cppyy_function_address_from_method(cppyy_method_t /* method */) { + return (cppyy_funcaddr_t)0; +} /* handling of function argument buffer ----------------------------------- */ void* cppyy_allocate_function_args(int nargs) { @@ -926,10 +927,6 @@ return s_scopes[handle].m_methods.size(); } -cppyy_index_t cppyy_method_index_at(cppyy_scope_t /* scope */, int imeth) { - return (cppyy_index_t)imeth; -} - char* cppyy_method_name(cppyy_scope_t handle, cppyy_index_t method_index) { return cppstring_to_cstring(s_scopes[handle].m_methods[(int)method_index].m_name); } @@ -978,8 +975,17 @@ return (cppyy_method_t)0; } +cppyy_index_t cppyy_get_global_operator(cppyy_scope_t /* scope */, + cppyy_scope_t /* lc */, cppyy_scope_t /* rc */, const char* /* op */) { + return (cppyy_index_t)-1; +} + /* method properties ----------------------------------------------------- */ +int cppyy_is_publicmethod(cppyy_type_t /* handle */, cppyy_index_t /* method_index */) { + return 1; +} + int cppyy_is_constructor(cppyy_type_t handle, cppyy_index_t method_index) { if (s_scopes.find(handle) != s_scopes.end()) return s_scopes[handle].m_methods[method_index].m_type == kConstructor; @@ -987,6 +993,10 @@ return 0; } +int cppyy_is_destructor(cppyy_type_t /* handle */, cppyy_index_t /* method_index */) { + return 0; +} + int cppyy_is_staticmethod(cppyy_type_t handle, cppyy_index_t method_index) { if (s_scopes.find(handle) != s_scopes.end()) return s_scopes[handle].m_methods[method_index].m_type == kStatic; @@ -1014,14 +1024,22 @@ /* data member properties ------------------------------------------------ */ -int cppyy_is_publicdata(cppyy_scope_t handle, int idatambr) { +int cppyy_is_publicdata(cppyy_scope_t /* handle */, cppyy_index_t /* idatambr */) { return 1; } -int cppyy_is_staticdata(cppyy_scope_t handle, int idatambr) { +int cppyy_is_staticdata(cppyy_scope_t handle, cppyy_index_t idatambr) { return s_scopes[handle].m_datambrs[idatambr].m_isstatic; } +int cppyy_is_const_data(cppyy_scope_t /* handle */, cppyy_index_t /* idatambr */) { + return 0; +} + +int cppyy_is_enum_data(cppyy_scope_t /* handle */, cppyy_index_t /* idatambr */) { + return 0; +} + /* misc helpers ----------------------------------------------------------- */ #if defined(_MSC_VER) diff --git a/pypy/module/_cppyy/test/Makefile b/pypy/module/_cppyy/test/Makefile --- a/pypy/module/_cppyy/test/Makefile +++ b/pypy/module/_cppyy/test/Makefile @@ -14,7 +14,7 @@ HASGENREFLEX:=$(shell command -v genreflex 2> /dev/null) -cppflags=-std=c++11 -O3 -m64 -fPIC -rdynamic +cppflags=-std=c++14 -O3 -m64 -fPIC -rdynamic ifdef HASGENREFLEX genreflex_flags:=$(shell genreflex --cppflags) cppflags+=$(genreflex_flags) diff --git a/pypy/module/_cppyy/test/advancedcpp.cxx b/pypy/module/_cppyy/test/advancedcpp.cxx --- a/pypy/module/_cppyy/test/advancedcpp.cxx +++ b/pypy/module/_cppyy/test/advancedcpp.cxx @@ -1,5 +1,7 @@ #include "advancedcpp.h" +#include + // for testing of default arguments #define IMPLEMENT_DEFAULTER_CLASS(type, tname) \ @@ -112,3 +114,13 @@ std::string overload_the_other_way::gime() { return "aap"; } int overload_the_other_way::gime() const { return 1; } + + +// exception handling testing +void Thrower::throw_anything() { + throw 1; +} + +void Thrower::throw_exception() { + throw std::runtime_error("C++ function failed"); +} diff --git a/pypy/module/_cppyy/test/advancedcpp.h b/pypy/module/_cppyy/test/advancedcpp.h --- a/pypy/module/_cppyy/test/advancedcpp.h +++ b/pypy/module/_cppyy/test/advancedcpp.h @@ -59,7 +59,7 @@ class a_class { // for esoteric inheritance testing public: a_class() { m_a = 1; m_da = 1.1; } - ~a_class() {} + virtual ~a_class() {} virtual int get_value() = 0; public: @@ -221,6 +221,7 @@ //=========================================================================== class some_abstract_class { // to test abstract class handling public: + virtual ~some_abstract_class() {} virtual void a_virtual_method() = 0; }; @@ -399,3 +400,11 @@ std::string gime(); int gime() const; }; + + +//=========================================================================== +class Thrower { // exception handling testing +public: + void throw_anything(); + void throw_exception(); +}; diff --git a/pypy/module/_cppyy/test/advancedcpp.xml b/pypy/module/_cppyy/test/advancedcpp.xml --- a/pypy/module/_cppyy/test/advancedcpp.xml +++ b/pypy/module/_cppyy/test/advancedcpp.xml @@ -57,4 +57,6 @@ + + diff --git a/pypy/module/_cppyy/test/conftest.py b/pypy/module/_cppyy/test/conftest.py --- a/pypy/module/_cppyy/test/conftest.py +++ b/pypy/module/_cppyy/test/conftest.py @@ -55,7 +55,7 @@ separate_module_files=[srcpath.join('dummy_backend.cxx')], include_dirs=[incpath, tstpath, cdir], compile_extra=['-DRPY_EXTERN=RPY_EXPORTED', '-DCPPYY_DUMMY_BACKEND', - '-fno-strict-aliasing', '-std=c++11'], + '-fno-strict-aliasing', '-std=c++14'], use_cpp_linker=True, ) @@ -65,7 +65,7 @@ outputfilename='libcppyy_dummy_backend', standalone=False) except CompilationError as e: - if '-std=c++11' in str(e): + if '-std=c++14' in str(e): global disabled disabled = str(e) return diff --git a/pypy/module/_cppyy/test/fragile.h b/pypy/module/_cppyy/test/fragile.h --- a/pypy/module/_cppyy/test/fragile.h +++ b/pypy/module/_cppyy/test/fragile.h @@ -30,6 +30,7 @@ void overload(int, no_such_class* p = 0) {} }; + static const int dummy_location = 0xdead; class E { @@ -105,6 +106,7 @@ class M { public: + virtual ~M() {} enum E1 { kOnce=42 }; enum E2 { kTwice=12 }; }; diff --git a/pypy/module/_cppyy/test/test_advancedcpp.py b/pypy/module/_cppyy/test/test_advancedcpp.py --- a/pypy/module/_cppyy/test/test_advancedcpp.py +++ b/pypy/module/_cppyy/test/test_advancedcpp.py @@ -656,3 +656,22 @@ # TODO: currently fails b/c double** not understood as &double* #assert cppyy.gbl.my_global_ptr[0] == 1234. + def test22_exceptions(self): + """Catching of C++ exceptions""" + + import _cppyy as cppyy + Thrower = cppyy.gbl.Thrower + + # TODO: clean up this interface: + Thrower.__cppdecl__.get_overload('throw_anything').__useffi__ = False + Thrower.__cppdecl__.get_overload('throw_exception').__useffi__ = False + + t = Thrower() + + assert raises(Exception, t.throw_anything) + assert raises(Exception, t.throw_exception) + + try: + t.throw_exception() + except Exception, e: + "C++ function failed" in str(e) diff --git a/pypy/module/_cppyy/test/test_overloads.py b/pypy/module/_cppyy/test/test_overloads.py --- a/pypy/module/_cppyy/test/test_overloads.py +++ b/pypy/module/_cppyy/test/test_overloads.py @@ -15,7 +15,6 @@ spaceconfig = dict(usemodules=['_cppyy', '_rawffi', 'itertools']) def setup_class(cls): - env = os.environ cls.w_test_dct = cls.space.newtext(test_dct) cls.w_overloads = cls.space.appexec([], """(): import ctypes diff --git a/pypy/module/_cppyy/test/test_zjit.py b/pypy/module/_cppyy/test/test_zjit.py --- a/pypy/module/_cppyy/test/test_zjit.py +++ b/pypy/module/_cppyy/test/test_zjit.py @@ -137,6 +137,7 @@ executor.get_executor(self, 'int').__class__.c_stubcall = staticmethod(c_call_i) self.w_AttributeError = FakeException(self, "AttributeError") + self.w_Exception = FakeException(self, "Exception") self.w_KeyError = FakeException(self, "KeyError") self.w_NotImplementedError = FakeException(self, "NotImplementedError") self.w_ReferenceError = FakeException(self, "ReferenceError") @@ -282,7 +283,7 @@ inst = interp_cppyy._bind_object(space, FakeInt(0), cls, True) cls.get_overload("__init__").call(inst, [FakeInt(0)]) cppmethod = cls.get_overload(method_name) - assert isinstance(inst, interp_cppyy.W_CPPClass) + assert isinstance(inst, interp_cppyy.W_CPPInstance) i = 10 while i > 0: drv.jit_merge_point(inst=inst, cppmethod=cppmethod, i=i) diff --git a/pypy/module/cpyext/test/test_datetime.py b/pypy/module/cpyext/test/test_datetime.py --- a/pypy/module/cpyext/test/test_datetime.py +++ b/pypy/module/cpyext/test/test_datetime.py @@ -1,3 +1,5 @@ +import pytest + from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.cdatetime import * @@ -82,6 +84,14 @@ date = datetime.datetime.fromtimestamp(0) assert space.unwrap(space.str(w_date)) == str(date) + @pytest.mark.parametrize('name', ['Time', 'DateTime', 'Date', 'Delta']) + def test_basicsize(self, space, name): + datetime = _PyDateTime_Import(space) + py_size = getattr(datetime, "c_%sType" % name).c_tp_basicsize + c_size = rffi.sizeof(cts.gettype("PyDateTime_%s" % name)) + assert py_size == c_size + + class AppTestDatetime(AppTestCpythonExtensionBase): def test_CAPI(self): module = self.import_extension('foo', [ @@ -271,9 +281,9 @@ 6, 6, 6, 6, args, PyDateTimeAPI->TimeType); """), ("datetime_with_tzinfo", "METH_O", - """ + """ PyObject * obj; - int tzrefcnt = args->ob_refcnt; + int tzrefcnt = args->ob_refcnt; PyDateTime_IMPORT; obj = PyDateTimeAPI->DateTime_FromDateAndTime( 2000, 6, 6, 6, 6, 6, 6, args, @@ -291,7 +301,7 @@ return NULL; } return obj; - + """), ], prologue='#include "datetime.h"\n') from datetime import tzinfo, datetime, timedelta, time @@ -339,4 +349,4 @@ assert module.checks(o) == (True,) * 3 + (False,) * 7 # isinstance(datetime, date) o = tzinfo() assert module.checks(o) == (False,) * 8 + (True,) * 2 - + diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py --- a/pypy/objspace/std/listobject.py +++ b/pypy/objspace/std/listobject.py @@ -991,6 +991,14 @@ def setslice(self, w_list, start, step, slicelength, w_other): strategy = w_other.strategy + if step != 1: + len2 = strategy.length(w_other) + if len2 == 0: + return + else: + raise oefmt(self.space.w_ValueError, + "attempt to assign sequence of size %d to extended " + "slice of size %d", len2, 0) storage = strategy.getstorage_copy(w_other) w_list.strategy = strategy w_list.lstorage = storage diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py --- a/pypy/objspace/std/test/test_listobject.py +++ b/pypy/objspace/std/test/test_listobject.py @@ -1080,6 +1080,15 @@ l[::3] = ('a', 'b') assert l == ['a', 1.1, 2.2, 'b', 4.4, 5.5] + l_int = [5]; l_int.pop() # IntListStrategy + l_empty = [] # EmptyListStrategy + raises(ValueError, "l_int[::-1] = [42]") + raises(ValueError, "l_int[::7] = [42]") + raises(ValueError, "l_empty[::-1] = [42]") + raises(ValueError, "l_empty[::7] = [42]") + l_int[::1] = [42]; assert l_int == [42] + l_empty[::1] = [42]; assert l_empty == [42] + def test_setslice_with_self(self): l = [1,2,3,4] l[:] = l From pypy.commits at gmail.com Tue Apr 24 02:07:39 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 23 Apr 2018 23:07:39 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: merge default into branch Message-ID: <5adec9ab.1c69fb81.83f65.e219@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r94439:6416cd28171a Date: 2018-04-24 09:04 +0300 http://bitbucket.org/pypy/pypy/changeset/6416cd28171a/ Log: merge default into branch diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -247,6 +247,7 @@ Lukas Vacek Omer Katz Jacek Generowicz + Tomasz Dziopa Sylvain Thenault Jakub Stasiak Andrew Dalke @@ -307,6 +308,7 @@ Yury V. Zaytsev florinpapa Anders Sigfridsson + Matt Jackson Nikolay Zinov rafalgalczynski at gmail.com Joshua Gilbert diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -214,6 +214,7 @@ Lukas Vacek Omer Katz Jacek Generowicz + Tomasz Dziopa Sylvain Thenault Jakub Stasiak Andrew Dalke @@ -274,6 +275,7 @@ Yury V. Zaytsev florinpapa Anders Sigfridsson + Matt Jackson Nikolay Zinov rafalgalczynski at gmail.com Joshua Gilbert diff --git a/pypy/doc/release-v6.0.0.rst b/pypy/doc/release-v6.0.0.rst --- a/pypy/doc/release-v6.0.0.rst +++ b/pypy/doc/release-v6.0.0.rst @@ -20,16 +20,18 @@ The GC now has `hooks`_ to gain more insights into its performance -The Windows PyPy3.5 release is still considered beta-quality. There are open -issues with unicode handling especially around system calls and c-extensions. +The Matplotlib TkAgg backend now works with PyPy, as do pygame and pygobject_. -The Matplotlib TkAgg backend now works with PyPy, as do pygame and pygobject_. +We updated the `cffi`_ module included in PyPy to version 1.11.5, and the +`cppyy`_ backend to 0.6.0. Please use these to wrap your C and C++ code, +respectively, for a JIT friendly experience. As always, this release is 100% compatible with the previous one and fixed several issues and bugs raised by the growing community of PyPy users. We strongly recommend updating. -We updated the cffi module included in PyPy to version 1.11.5 +The Windows PyPy3.5 release is still considered beta-quality. There are open +issues with unicode handling especially around system calls and c-extensions. The utf8 branch that changes internal representation of unicode to utf8 did not make it into the release, so there is still more goodness coming. We also @@ -56,6 +58,8 @@ .. _pygobject: https://lazka.github.io/posts/2018-04_pypy-pygobject/index.html .. _`syntax errors`: https://morepypy.blogspot.com/2018/04/improving-syntaxerror-in-pypy.html .. _`hooks`: gc_info.html#gc-hooks +.. _`cffi`: http://cffi.readthedocs.io +.. _`cppyy`: https://cppyy.readthedocs.io What is PyPy? ============= diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -3,7 +3,7 @@ ========================== .. this is a revision shortly after release-pypy-6.0.0 -.. startrev: ad79cc0ce9a8 +.. startrev: e50e11af23f1 diff --git a/pypy/doc/whatsnew-pypy2-6.0.0.rst b/pypy/doc/whatsnew-pypy2-6.0.0.rst --- a/pypy/doc/whatsnew-pypy2-6.0.0.rst +++ b/pypy/doc/whatsnew-pypy2-6.0.0.rst @@ -125,4 +125,8 @@ .. branch: gc-hook-better-timestamp -Improve GC hooksd +Improve GC hooks + +.. branch: cppyy-packaging + +Update backend to 0.6.0 and support exceptions through wrappers diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -479,11 +479,17 @@ # 'action.fire()' happens to be called any time before # the corresponding perform(), the fire() has no # effect---which is the effect we want, because - # perform() will be called anyway. + # perform() will be called anyway. All such pending + # actions with _fired == True are still inside the old + # chained list. As soon as we reset _fired to False, + # we also reset _next to None and we are ready for + # another fire(). while action is not None: + next_action = action._next + action._next = None action._fired = False action.perform(ec, frame) - action._next, action = None, action._next + action = next_action self.action_dispatcher = action_dispatcher diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py --- a/pypy/interpreter/test/test_executioncontext.py +++ b/pypy/interpreter/test/test_executioncontext.py @@ -66,6 +66,47 @@ """) assert events == ['one'] + def test_fire_inside_perform(self): + # test what happens if we call AsyncAction.fire() while we are in the + # middle of an AsyncAction.perform(). In particular, this happens when + # PyObjectDeallocAction.fire() is called by rawrefcount: see issue + # 2805 + events = [] + + class Action1(executioncontext.AsyncAction): + _count = 0 + + def perform(self, ec, frame): + events.append('one') + if self._count == 0: + # a1 is no longer in the queue, so it will be enqueued + a1.fire() + # + # a2 is still in the queue, so the fire() is ignored and + # it's performed in its normal order, i.e. BEFORE a3 + a2.fire() + self._count += 1 + + class Action2(executioncontext.AsyncAction): + def perform(self, ec, frame): + events.append('two') + + class Action3(executioncontext.AsyncAction): + def perform(self, ec, frame): + events.append('three') + + space = self.space + a1 = Action1(space) + a2 = Action2(space) + a3 = Action3(space) + a1.fire() + a2.fire() + a3.fire() + space.appexec([], """(): + pass + """) + assert events == ['one', 'two', 'three', 'one'] + def test_periodic_action(self): from pypy.interpreter.executioncontext import ActionFlag diff --git a/pypy/module/_cppyy/__init__.py b/pypy/module/_cppyy/__init__.py --- a/pypy/module/_cppyy/__init__.py +++ b/pypy/module/_cppyy/__init__.py @@ -13,7 +13,7 @@ '_set_function_generator': 'interp_cppyy.set_function_generator', '_register_class' : 'interp_cppyy.register_class', '_get_nullptr' : 'interp_cppyy.get_nullptr', - 'CPPClassBase' : 'interp_cppyy.W_CPPClass', + 'CPPInstanceBase' : 'interp_cppyy.W_CPPInstance', 'addressof' : 'interp_cppyy.addressof', '_bind_object' : 'interp_cppyy._bind_object', 'bind_object' : 'interp_cppyy.bind_object', diff --git a/pypy/module/_cppyy/capi/loadable_capi.py b/pypy/module/_cppyy/capi/loadable_capi.py --- a/pypy/module/_cppyy/capi/loadable_capi.py +++ b/pypy/module/_cppyy/capi/loadable_capi.py @@ -121,11 +121,11 @@ # TODO: the following need to match up with the globally defined C_XYZ low-level # types (see capi/__init__.py), but by using strings here, that isn't guaranteed - c_opaque_ptr = state.c_ulong + c_opaque_ptr = state.c_ulong # not ptrdiff_t (which is signed) c_scope = c_opaque_ptr c_type = c_scope - c_object = c_opaque_ptr + c_object = c_opaque_ptr # not voidp (to stick with one handle type) c_method = c_opaque_ptr c_index = state.c_long c_index_array = state.c_voidp @@ -150,16 +150,17 @@ self.capi_call_ifaces = { # name to opaque C++ scope representation - 'num_scopes' : ([c_scope], c_int), - 'scope_name' : ([c_scope, c_int], c_ccharp), - 'resolve_name' : ([c_ccharp], c_ccharp), + 'resolve_enum' : ([c_ccharp], c_ccharp), 'get_scope' : ([c_ccharp], c_scope), 'actual_class' : ([c_type, c_object], c_type), + 'size_of_klass' : ([c_type], c_size_t), + 'size_of_type' : ([c_ccharp], c_size_t), # memory management 'allocate' : ([c_type], c_object), 'deallocate' : ([c_type, c_object], c_void), + 'construct' : ([c_type], c_object), 'destruct' : ([c_type, c_object], c_void), # method/function dispatching @@ -182,7 +183,8 @@ 'constructor' : ([c_method, c_object, c_int, c_voidp], c_object), 'call_o' : ([c_method, c_object, c_int, c_voidp, c_type], c_object), - 'get_function_address' : ([c_scope, c_index], c_voidp), # TODO: verify + 'function_address_from_index' : ([c_scope, c_index], c_voidp), # TODO: verify + 'function_address_from_method' : ([c_method], c_voidp), # id. # handling of function argument buffer 'allocate_function_args' : ([c_int], c_voidp), @@ -196,6 +198,8 @@ 'is_abstract' : ([c_type], c_int), 'is_enum' : ([c_ccharp], c_int), + 'get_all_cpp_names' : ([c_scope, c_voidp], c_voidp), # const char** + # type/class reflection information 'final_name' : ([c_type], c_ccharp), 'scoped_final_name' : ([c_type], c_ccharp), @@ -208,10 +212,10 @@ # method/function reflection information 'num_methods' : ([c_scope], c_int), - 'method_index_at' : ([c_scope, c_int], c_index), 'method_indices_from_name' : ([c_scope, c_ccharp], c_index_array), 'method_name' : ([c_scope, c_index], c_ccharp), + 'method_mangled_name' : ([c_scope, c_index], c_ccharp), 'method_result_type' : ([c_scope, c_index], c_ccharp), 'method_num_args' : ([c_scope, c_index], c_int), 'method_req_args' : ([c_scope, c_index], c_int), @@ -219,7 +223,9 @@ 'method_arg_default' : ([c_scope, c_index, c_int], c_ccharp), 'method_signature' : ([c_scope, c_index, c_int], c_ccharp), 'method_prototype' : ([c_scope, c_index, c_int], c_ccharp), + 'is_const_method' : ([c_method], c_int), + 'exists_method_template' : ([c_scope, c_ccharp], c_int), 'method_is_template' : ([c_scope, c_index], c_int), 'method_num_template_args' : ([c_scope, c_index], c_int), 'method_template_arg_name' : ([c_scope, c_index, c_index], c_ccharp), @@ -228,7 +234,9 @@ 'get_global_operator' : ([c_scope, c_scope, c_scope, c_ccharp], c_index), # method properties + 'is_public_method' : ([c_type, c_index], c_int), 'is_constructor' : ([c_type, c_index], c_int), + 'is_destructor' : ([c_type, c_index], c_int), 'is_staticmethod' : ([c_type, c_index], c_int), # data member reflection information @@ -236,12 +244,14 @@ 'datamember_name' : ([c_scope, c_int], c_ccharp), 'datamember_type' : ([c_scope, c_int], c_ccharp), 'datamember_offset' : ([c_scope, c_int], c_ptrdiff_t), - 'datamember_index' : ([c_scope, c_ccharp], c_int), # data member properties 'is_publicdata' : ([c_scope, c_int], c_int), 'is_staticdata' : ([c_scope, c_int], c_int), + 'is_const_data' : ([c_scope, c_int], c_int), + 'is_enum_data' : ([c_scope, c_int], c_int), + 'get_dimension_size' : ([c_scope, c_int, c_int], c_int), # misc helpers 'strtoll' : ([c_ccharp], c_llong), @@ -328,25 +338,27 @@ return rffi.cast(rffi.CCHARP, ptr) # name to opaque C++ scope representation ------------------------------------ -def c_num_scopes(space, cppscope): - return space.int_w(call_capi(space, 'num_scopes', [_ArgH(cppscope.handle)])) -def c_scope_name(space, cppscope, iscope): - args = [_ArgH(cppscope.handle), _ArgL(iscope)] - return charp2str_free(space, call_capi(space, 'scope_name', args)) - def c_resolve_name(space, name): return charp2str_free(space, call_capi(space, 'resolve_name', [_ArgS(name)])) +def c_resolve_enum(space, name): + return charp2str_free(space, call_capi(space, 'resolve_enum', [_ArgS(name)])) def c_get_scope_opaque(space, name): return rffi.cast(C_SCOPE, space.uint_w(call_capi(space, 'get_scope', [_ArgS(name)]))) def c_actual_class(space, cppclass, cppobj): args = [_ArgH(cppclass.handle), _ArgH(cppobj)] return rffi.cast(C_TYPE, space.uint_w(call_capi(space, 'actual_class', args))) +def c_size_of_klass(space, cppclass): + return _cdata_to_size_t(space, call_capi(space, 'size_of_klass', [_ArgH(cppclass.handle)])) +def c_size_of_type(space, name): + return _cdata_to_size_t(space, call_capi(space, 'size_of_type', [_ArgS(name)])) # memory management ---------------------------------------------------------- def c_allocate(space, cppclass): return _cdata_to_cobject(space, call_capi(space, 'allocate', [_ArgH(cppclass.handle)])) def c_deallocate(space, cppclass, cppobject): call_capi(space, 'deallocate', [_ArgH(cppclass.handle), _ArgH(cppobject)]) +def c_construct(space, cppclass): + return _cdata_to_cobject(space, call_capi(space, 'construct', [_ArgH(cppclass.handle)])) def c_destruct(space, cppclass, cppobject): call_capi(space, 'destruct', [_ArgH(cppclass.handle), _ArgH(cppobject)]) @@ -391,7 +403,7 @@ w_cstr = call_capi(space, 'call_s', [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs), _ArgP(rffi.cast(rffi.VOIDP, length))]) - cstr_len = intmask(length[0]) + cstr_len = int(intmask(length[0])) finally: lltype.free(length, flavor='raw') return _cdata_to_ccharp(space, w_cstr), cstr_len @@ -403,10 +415,13 @@ args = [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs), _ArgH(cppclass.handle)] return _cdata_to_cobject(space, call_capi(space, 'call_o', args)) -def c_get_function_address(space, cppscope, index): +def c_function_address_from_index(space, cppscope, index): args = [_ArgH(cppscope.handle), _ArgL(index)] return rffi.cast(C_FUNC_PTR, - _cdata_to_ptr(space, call_capi(space, 'get_function_address', args))) + _cdata_to_ptr(space, call_capi(space, 'function_address_from_index', args))) +def c_function_address_from_method(space, cppmethod): + return rffi.cast(C_FUNC_PTR, + _cdata_to_ptr(space, call_capi(space, 'function_address_from_method', _ArgH(cppmethod)))) # handling of function argument buffer --------------------------------------- def c_allocate_function_args(space, size): @@ -430,6 +445,23 @@ def c_is_enum(space, name): return space.bool_w(call_capi(space, 'is_enum', [_ArgS(name)])) +def c_get_all_cpp_names(space, scope): + sz = lltype.malloc(rffi.SIZE_TP.TO, 1, flavor='raw', zero=True) + try: + args = [_ArgH(scope.handle), _ArgP(rffi.cast(rffi.VOIDP, sz))] + rawnames = rffi.cast(rffi.CCHARPP, + _cdata_to_ptr(space, call_capi(space, 'get_all_cpp_names', args))) + count = int(intmask(sz[0])) + finally: + lltype.free(sz, flavor='raw') + allnames = [] + for i in range(count): + pystr = rffi.charp2str(rawnames[i]) + c_free(space, rffi.cast(rffi.VOIDP, rawnames[i])) # c_free defined below + allnames.append(pystr) + c_free(space, rffi.cast(rffi.VOIDP, rawnames)) # id. + return allnames + # type/class reflection information ------------------------------------------ def c_final_name(space, cpptype): return charp2str_free(space, call_capi(space, 'final_name', [_ArgH(cpptype)])) @@ -462,13 +494,10 @@ def c_num_methods(space, cppscope): args = [_ArgH(cppscope.handle)] return space.int_w(call_capi(space, 'num_methods', args)) -def c_method_index_at(space, cppscope, imethod): - args = [_ArgH(cppscope.handle), _ArgL(imethod)] - return space.int_w(call_capi(space, 'method_index_at', args)) def c_method_indices_from_name(space, cppscope, name): args = [_ArgH(cppscope.handle), _ArgS(name)] indices = rffi.cast(C_INDEX_ARRAY, - _cdata_to_ptr(space, call_capi(space, 'method_indices_from_name', args))) + _cdata_to_ptr(space, call_capi(space, 'method_indices_from_name', args))) if not indices: return [] py_indices = [] @@ -506,6 +535,9 @@ args = [_ArgH(cppscope.handle), _ArgL(index), _ArgL(show_formalargs)] return charp2str_free(space, call_capi(space, 'method_prototype', args)) +def c_exists_method_template(space, cppscope, name): + args = [_ArgH(cppscope.handle), _ArgS(name)] + return space.bool_w(call_capi(space, 'exists_method_template', args)) def c_method_is_template(space, cppscope, index): args = [_ArgH(cppscope.handle), _ArgL(index)] return space.bool_w(call_capi(space, 'method_is_template', args)) @@ -531,9 +563,15 @@ return rffi.cast(WLAVC_INDEX, -1) # method properties ---------------------------------------------------------- +def c_is_public_method(space, cppclass, index): + args = [_ArgH(cppclass.handle), _ArgL(index)] + return space.bool_w(call_capi(space, 'is_public_method', args)) def c_is_constructor(space, cppclass, index): args = [_ArgH(cppclass.handle), _ArgL(index)] return space.bool_w(call_capi(space, 'is_constructor', args)) +def c_is_destructor(space, cppclass, index): + args = [_ArgH(cppclass.handle), _ArgL(index)] + return space.bool_w(call_capi(space, 'is_destructor', args)) def c_is_staticmethod(space, cppclass, index): args = [_ArgH(cppclass.handle), _ArgL(index)] return space.bool_w(call_capi(space, 'is_staticmethod', args)) @@ -562,6 +600,15 @@ def c_is_staticdata(space, cppscope, datamember_index): args = [_ArgH(cppscope.handle), _ArgL(datamember_index)] return space.bool_w(call_capi(space, 'is_staticdata', args)) +def c_is_const_data(space, cppscope, datamember_index): + args = [_ArgH(cppscope.handle), _ArgL(datamember_index)] + return space.bool_w(call_capi(space, 'is_const_data', args)) +def c_is_enum_data(space, cppscope, datamember_index): + args = [_ArgH(cppscope.handle), _ArgL(datamember_index)] + return space.bool_w(call_capi(space, 'is_enum_data', args)) +def c_get_dimension_size(space, cppscope, datamember_index, dim_idx): + args = [_ArgH(cppscope.handle), _ArgL(datamember_index), _ArgL(dim_idx)] + return space.bool_w(call_capi(space, 'get_dimension_size', args)) # misc helpers --------------------------------------------------------------- def c_strtoll(space, svalue): @@ -585,7 +632,7 @@ try: w_cstr = call_capi(space, 'stdstring2charp', [_ArgH(cppstr), _ArgP(rffi.cast(rffi.VOIDP, sz))]) - cstr_len = intmask(sz[0]) + cstr_len = int(intmask(sz[0])) finally: lltype.free(sz, flavor='raw') return rffi.charpsize2str(_cdata_to_ccharp(space, w_cstr), cstr_len) @@ -607,7 +654,7 @@ """Return a python string taking into account \0""" from pypy.module._cppyy import interp_cppyy - cppstr = space.interp_w(interp_cppyy.W_CPPClass, w_self, can_be_None=False) + cppstr = space.interp_w(interp_cppyy.W_CPPInstance, w_self, can_be_None=False) return space.newtext(c_stdstring2charp(space, cppstr._rawobject)) # setup pythonizations for later use at run-time diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py --- a/pypy/module/_cppyy/converter.py +++ b/pypy/module/_cppyy/converter.py @@ -22,8 +22,8 @@ def get_rawobject(space, w_obj, can_be_None=True): - from pypy.module._cppyy.interp_cppyy import W_CPPClass - cppinstance = space.interp_w(W_CPPClass, w_obj, can_be_None=can_be_None) + from pypy.module._cppyy.interp_cppyy import W_CPPInstance + cppinstance = space.interp_w(W_CPPInstance, w_obj, can_be_None=can_be_None) if cppinstance: rawobject = cppinstance.get_rawobject() assert lltype.typeOf(rawobject) == capi.C_OBJECT @@ -31,15 +31,15 @@ return capi.C_NULL_OBJECT def set_rawobject(space, w_obj, address): - from pypy.module._cppyy.interp_cppyy import W_CPPClass - cppinstance = space.interp_w(W_CPPClass, w_obj, can_be_None=True) + from pypy.module._cppyy.interp_cppyy import W_CPPInstance + cppinstance = space.interp_w(W_CPPInstance, w_obj, can_be_None=True) if cppinstance: assert lltype.typeOf(cppinstance._rawobject) == capi.C_OBJECT cppinstance._rawobject = rffi.cast(capi.C_OBJECT, address) def get_rawobject_nonnull(space, w_obj): - from pypy.module._cppyy.interp_cppyy import W_CPPClass - cppinstance = space.interp_w(W_CPPClass, w_obj, can_be_None=True) + from pypy.module._cppyy.interp_cppyy import W_CPPInstance + cppinstance = space.interp_w(W_CPPInstance, w_obj, can_be_None=True) if cppinstance: cppinstance._nullcheck() rawobject = cppinstance.get_rawobject() @@ -502,8 +502,8 @@ self.clsdecl = clsdecl def _unwrap_object(self, space, w_obj): - from pypy.module._cppyy.interp_cppyy import W_CPPClass - if isinstance(w_obj, W_CPPClass): + from pypy.module._cppyy.interp_cppyy import W_CPPInstance + if isinstance(w_obj, W_CPPInstance): from pypy.module._cppyy.interp_cppyy import INSTANCE_FLAGS_IS_R_VALUE if w_obj.flags & INSTANCE_FLAGS_IS_R_VALUE: # reject moves as all are explicit @@ -534,8 +534,8 @@ class InstanceMoveConverter(InstanceRefConverter): def _unwrap_object(self, space, w_obj): # moving is same as by-ref, but have to check that move is allowed - from pypy.module._cppyy.interp_cppyy import W_CPPClass, INSTANCE_FLAGS_IS_R_VALUE - if isinstance(w_obj, W_CPPClass): + from pypy.module._cppyy.interp_cppyy import W_CPPInstance, INSTANCE_FLAGS_IS_R_VALUE + if isinstance(w_obj, W_CPPInstance): if w_obj.flags & INSTANCE_FLAGS_IS_R_VALUE: w_obj.flags &= ~INSTANCE_FLAGS_IS_R_VALUE return InstanceRefConverter._unwrap_object(self, space, w_obj) @@ -598,8 +598,8 @@ raise FastCallNotPossible def finalize_call(self, space, w_obj, call_local): - from pypy.module._cppyy.interp_cppyy import W_CPPClass - assert isinstance(w_obj, W_CPPClass) + from pypy.module._cppyy.interp_cppyy import W_CPPInstance + assert isinstance(w_obj, W_CPPInstance) r = rffi.cast(rffi.VOIDPP, call_local) w_obj._rawobject = rffi.cast(capi.C_OBJECT, r[0]) @@ -617,8 +617,8 @@ InstanceConverter.__init__(self, space, cppclass) def _unwrap_object(self, space, w_obj): - from pypy.module._cppyy.interp_cppyy import W_CPPClass - if isinstance(w_obj, W_CPPClass): + from pypy.module._cppyy.interp_cppyy import W_CPPInstance + if isinstance(w_obj, W_CPPInstance): arg = InstanceConverter._unwrap_object(self, space, w_obj) return capi.c_stdstring2stdstring(space, arg) else: @@ -749,8 +749,6 @@ return InstancePtrPtrConverter(space, clsdecl) elif compound == "": return InstanceConverter(space, clsdecl) - elif capi.c_is_enum(space, clean_name): - return _converters['unsigned'](space, default) # 5) void* or void converter (which fails on use) if 0 <= compound.find('*'): diff --git a/pypy/module/_cppyy/executor.py b/pypy/module/_cppyy/executor.py --- a/pypy/module/_cppyy/executor.py +++ b/pypy/module/_cppyy/executor.py @@ -293,8 +293,6 @@ return InstancePtrExecutor(space, cppclass) elif compound == '**' or compound == '*&': return InstancePtrPtrExecutor(space, cppclass) - elif capi.c_is_enum(space, clean_name): - return _executors['internal_enum_type_t'](space, None) # 4) additional special cases if compound == '*': diff --git a/pypy/module/_cppyy/include/capi.h b/pypy/module/_cppyy/include/capi.h --- a/pypy/module/_cppyy/include/capi.h +++ b/pypy/module/_cppyy/include/capi.h @@ -8,26 +8,29 @@ extern "C" { #endif // ifdef __cplusplus - typedef unsigned long cppyy_scope_t; + typedef ptrdiff_t cppyy_scope_t; typedef cppyy_scope_t cppyy_type_t; - typedef unsigned long cppyy_object_t; - typedef unsigned long cppyy_method_t; + typedef void* cppyy_object_t; + typedef ptrdiff_t cppyy_method_t; + typedef long cppyy_index_t; typedef void* cppyy_funcaddr_t; + typedef unsigned long cppyy_exctype_t; + /* name to opaque C++ scope representation -------------------------------- */ RPY_EXTERN - int cppyy_num_scopes(cppyy_scope_t parent); + char* cppyy_resolve_name(const char* cppitem_name); RPY_EXTERN - char* cppyy_scope_name(cppyy_scope_t parent, cppyy_index_t iscope); - RPY_EXTERN - char* cppyy_resolve_name(const char* cppitem_name); + char* cppyy_resolve_enum(const char* enum_type); RPY_EXTERN cppyy_scope_t cppyy_get_scope(const char* scope_name); RPY_EXTERN cppyy_type_t cppyy_actual_class(cppyy_type_t klass, cppyy_object_t obj); RPY_EXTERN - size_t cppyy_size_of(cppyy_type_t klass); + size_t cppyy_size_of_klass(cppyy_type_t klass); + RPY_EXTERN + size_t cppyy_size_of_type(const char* type_name); /* memory management ------------------------------------------------------ */ RPY_EXTERN @@ -35,48 +38,53 @@ RPY_EXTERN void cppyy_deallocate(cppyy_type_t type, cppyy_object_t self); RPY_EXTERN + cppyy_object_t cppyy_construct(cppyy_type_t type); + RPY_EXTERN void cppyy_destruct(cppyy_type_t type, cppyy_object_t self); /* method/function dispatching -------------------------------------------- */ RPY_EXTERN - void cppyy_call_v(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); + void cppyy_call_v(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN unsigned char cppyy_call_b(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN - char cppyy_call_c(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); + char cppyy_call_c(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN - short cppyy_call_h(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); + short cppyy_call_h(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN - int cppyy_call_i(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); + int cppyy_call_i(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN - long cppyy_call_l(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); + long cppyy_call_l(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN long long cppyy_call_ll(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN - float cppyy_call_f(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); + float cppyy_call_f(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN double cppyy_call_d(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN long double cppyy_call_ld(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN - void* cppyy_call_r(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); + void* cppyy_call_r(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN - char* cppyy_call_s(cppyy_method_t method, cppyy_object_t self, int nargs, void* args, size_t* length); - + char* cppyy_call_s(cppyy_method_t method, cppyy_object_t self, int nargs, void* args, size_t* length); RPY_EXTERN cppyy_object_t cppyy_constructor(cppyy_method_t method, cppyy_type_t klass, int nargs, void* args); RPY_EXTERN + void cppyy_destructor(cppyy_type_t type, cppyy_object_t self); + RPY_EXTERN cppyy_object_t cppyy_call_o(cppyy_method_t method, cppyy_object_t self, int nargs, void* args, cppyy_type_t result_type); RPY_EXTERN - cppyy_funcaddr_t cppyy_get_function_address(cppyy_scope_t scope, cppyy_index_t idx); + cppyy_funcaddr_t cppyy_function_address_from_index(cppyy_scope_t scope, cppyy_index_t idx); + RPY_EXTERN + cppyy_funcaddr_t cppyy_function_address_from_method(cppyy_method_t method); /* handling of function argument buffer ----------------------------------- */ RPY_EXTERN - void* cppyy_allocate_function_args(int nargs); + void* cppyy_allocate_function_args(int nargs); RPY_EXTERN - void cppyy_deallocate_function_args(void* args); + void cppyy_deallocate_function_args(void* args); RPY_EXTERN size_t cppyy_function_arg_sizeof(); RPY_EXTERN @@ -92,6 +100,9 @@ RPY_EXTERN int cppyy_is_enum(const char* type_name); + RPY_EXTERN + const char** cppyy_get_all_cpp_names(cppyy_scope_t scope, size_t* count); + /* class reflection information ------------------------------------------- */ RPY_EXTERN char* cppyy_final_name(cppyy_type_t type); @@ -105,6 +116,10 @@ char* cppyy_base_name(cppyy_type_t type, int base_index); RPY_EXTERN int cppyy_is_subtype(cppyy_type_t derived, cppyy_type_t base); + RPY_EXTERN + int cppyy_smartptr_info(const char* name, cppyy_type_t* raw, cppyy_method_t* deref); + RPY_EXTERN + void cppyy_add_smartptr_type(const char* type_name); /* calculate offsets between declared and actual type, up-cast: direction > 0; down-cast: direction < 0 */ RPY_EXTERN @@ -114,8 +129,6 @@ RPY_EXTERN int cppyy_num_methods(cppyy_scope_t scope); RPY_EXTERN - cppyy_index_t cppyy_method_index_at(cppyy_scope_t scope, int imeth); - RPY_EXTERN cppyy_index_t* cppyy_method_indices_from_name(cppyy_scope_t scope, const char* name); RPY_EXTERN @@ -136,8 +149,12 @@ char* cppyy_method_signature(cppyy_scope_t scope, cppyy_index_t idx, int show_formalargs); RPY_EXTERN char* cppyy_method_prototype(cppyy_scope_t scope, cppyy_index_t idx, int show_formalargs); + RPY_EXTERN + int cppyy_is_const_method(cppyy_method_t); RPY_EXTERN + int cppyy_exists_method_template(cppyy_scope_t scope, const char* name); + RPY_EXTERN int cppyy_method_is_template(cppyy_scope_t scope, cppyy_index_t idx); RPY_EXTERN int cppyy_method_num_template_args(cppyy_scope_t scope, cppyy_index_t idx); @@ -169,15 +186,20 @@ char* cppyy_datamember_type(cppyy_scope_t scope, int datamember_index); RPY_EXTERN ptrdiff_t cppyy_datamember_offset(cppyy_scope_t scope, int datamember_index); - RPY_EXTERN int cppyy_datamember_index(cppyy_scope_t scope, const char* name); /* data member properties ------------------------------------------------- */ RPY_EXTERN - int cppyy_is_publicdata(cppyy_type_t type, int datamember_index); + int cppyy_is_publicdata(cppyy_type_t type, cppyy_index_t datamember_index); RPY_EXTERN - int cppyy_is_staticdata(cppyy_type_t type, int datamember_index); + int cppyy_is_staticdata(cppyy_type_t type, cppyy_index_t datamember_index); + RPY_EXTERN + int cppyy_is_const_data(cppyy_scope_t scope, cppyy_index_t idata); + RPY_EXTERN + int cppyy_is_enum_data(cppyy_scope_t scope, cppyy_index_t idata); + RPY_EXTERN + int cppyy_get_dimension_size(cppyy_scope_t scope, cppyy_index_t idata, int dimension); /* misc helpers ----------------------------------------------------------- */ RPY_EXTERN @@ -197,7 +219,7 @@ RPY_EXTERN const char* cppyy_stdvector_valuetype(const char* clname); RPY_EXTERN - size_t cppyy_stdvector_valuesize(const char* clname); + size_t cppyy_stdvector_valuesize(const char* clname); #ifdef __cplusplus } diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py --- a/pypy/module/_cppyy/interp_cppyy.py +++ b/pypy/module/_cppyy/interp_cppyy.py @@ -19,6 +19,9 @@ INSTANCE_FLAGS_IS_REF = 0x0002 INSTANCE_FLAGS_IS_R_VALUE = 0x0004 +OVERLOAD_FLAGS_USE_FFI = 0x0001 + + class FastCallNotPossible(Exception): pass @@ -174,7 +177,7 @@ @staticmethod def unpack_cppthis(space, w_cppinstance, declaring_scope): - cppinstance = space.interp_w(W_CPPClass, w_cppinstance, can_be_None=False) + cppinstance = space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=False) cppinstance._nullcheck() return cppinstance.get_cppthis(declaring_scope) @@ -186,7 +189,7 @@ return rffi.cast(rffi.VOIDP, loc_idx) @jit.unroll_safe - def call(self, cppthis, args_w): + def call(self, cppthis, args_w, useffi): jit.promote(self) assert lltype.typeOf(cppthis) == capi.C_OBJECT @@ -218,16 +221,23 @@ try: # attempt to call directly through ffi chain - if self._funcaddr: + if useffi and self._funcaddr: try: return self.do_fast_call(cppthis, args_w, call_local) except FastCallNotPossible: pass # can happen if converters or executor does not implement ffi # ffi chain must have failed; using stub functions instead - args = self.prepare_arguments(args_w, call_local) + args, stat = self.prepare_arguments(args_w, call_local) try: - return self.executor.execute(self.space, self.cppmethod, cppthis, len(args_w), args) + result = self.executor.execute( + self.space, self.cppmethod, cppthis, len(args_w), args) + if stat[0] != rffi.cast(rffi.ULONG, 0): + what = rffi.cast(rffi.CCHARP, stat[1]) + pywhat = rffi.charp2str(what) + capi.c_free(self.space, rffi.cast(rffi.VOIDP, what)) + raise OperationError(self.space.w_Exception, self.space.newtext(pywhat)) + return result finally: self.finalize_call(args, args_w, call_local) finally: @@ -322,7 +332,7 @@ # Each CPPMethod corresponds one-to-one to a C++ equivalent and cppthis # has been offset to the matching class. Hence, the libffi pointer is # uniquely defined and needs to be setup only once. - funcaddr = capi.c_get_function_address(self.space, self.scope, self.index) + funcaddr = capi.c_function_address_from_index(self.space, self.scope, self.index) if funcaddr and cppthis: # methods only for now state = self.space.fromcache(ffitypes.State) @@ -373,7 +383,10 @@ conv.free_argument(self.space, rffi.cast(capi.C_OBJECT, arg_j), loc_j) capi.c_deallocate_function_args(self.space, args) raise - return args + stat = rffi.cast(rffi.ULONGP, + lltype.direct_ptradd(rffi.cast(rffi.CCHARP, args), int(len(args_w))*stride)) + stat[0] = rffi.cast(rffi.ULONG, 0) + return args, stat @jit.unroll_safe def finalize_call(self, args, args_w, call_local): @@ -435,7 +448,7 @@ # TODO: might have to specialize for CPPTemplatedCall on CPPMethod/CPPFunction here CPPMethod.__init__(self, space, declaring_scope, method_index, arg_defs, args_required) - def call(self, cppthis, args_w): + def call(self, cppthis, args_w, useffi): assert lltype.typeOf(cppthis) == capi.C_OBJECT for i in range(len(args_w)): try: @@ -447,10 +460,10 @@ raise oefmt(self.space.w_TypeError, "non-matching template (got %s where %s expected)", s, self.templ_args[i]) - return W_CPPBoundMethod(cppthis, self) + return W_CPPBoundMethod(cppthis, self, useffi) - def bound_call(self, cppthis, args_w): - return CPPMethod.call(self, cppthis, args_w) + def bound_call(self, cppthis, args_w, useffi): + return CPPMethod.call(self, cppthis, args_w, useffi) def __repr__(self): return "CPPTemplatedCall: %s" % self.prototype() @@ -468,11 +481,11 @@ def unpack_cppthis(space, w_cppinstance, declaring_scope): return rffi.cast(capi.C_OBJECT, declaring_scope.handle) - def call(self, cppthis, args_w): + def call(self, cppthis, args_w, useffi): # Note: this does not return a wrapped instance, just a pointer to the # new instance; the overload must still wrap it before returning. Also, # cppthis is declaring_scope.handle (as per unpack_cppthis(), above). - return CPPMethod.call(self, cppthis, args_w) + return CPPMethod.call(self, cppthis, args_w, useffi) def __repr__(self): return "CPPConstructor: %s" % self.prototype() @@ -485,7 +498,7 @@ _immutable_ = True - def call(self, cppthis, args_w): + def call(self, cppthis, args_w, useffi): end = len(args_w)-1 if 0 <= end: w_item = args_w[end] @@ -493,7 +506,7 @@ if self.converters is None: self._setup(cppthis) self.executor.set_item(self.space, w_item) # TODO: what about threads? - CPPMethod.call(self, cppthis, args_w) + CPPMethod.call(self, cppthis, args_w, useffi) class W_CPPOverload(W_Root): @@ -501,7 +514,7 @@ collection of (possibly) overloaded methods or functions. It calls these in order and deals with error handling and reporting.""" - _attrs_ = ['space', 'scope', 'functions'] + _attrs_ = ['space', 'scope', 'functions', 'flags'] _immutable_fields_ = ['scope', 'functions[*]'] def __init__(self, space, declaring_scope, functions): @@ -510,6 +523,19 @@ assert len(functions) from rpython.rlib import debug self.functions = debug.make_sure_not_resized(functions) + self.flags = 0 + self.flags |= OVERLOAD_FLAGS_USE_FFI + + # allow user to determine ffi use rules per overload + def fget_useffi(self, space): + return space.newbool(bool(self.flags & OVERLOAD_FLAGS_USE_FFI)) + + @unwrap_spec(value=bool) + def fset_useffi(self, space, value): + if space.is_true(value): + self.flags |= OVERLOAD_FLAGS_USE_FFI + else: + self.flags &= ~OVERLOAD_FLAGS_USE_FFI @jit.elidable_promote() def is_static(self): @@ -540,7 +566,7 @@ for i in range(len(self.functions)): cppyyfunc = self.functions[i] try: - return cppyyfunc.call(cppthis, args_w) + return cppyyfunc.call(cppthis, args_w, self.flags & OVERLOAD_FLAGS_USE_FFI) except Exception: pass @@ -553,7 +579,7 @@ for i in range(len(self.functions)): cppyyfunc = self.functions[i] try: - return cppyyfunc.call(cppthis, args_w) + return cppyyfunc.call(cppthis, args_w, self.flags & OVERLOAD_FLAGS_USE_FFI) except OperationError as e: # special case if there's just one function, to prevent clogging the error message if len(self.functions) == 1: @@ -588,6 +614,7 @@ 'CPPOverload', is_static = interp2app(W_CPPOverload.is_static), call = interp2app(W_CPPOverload.call), + __useffi__ = GetSetProperty(W_CPPOverload.fget_useffi, W_CPPOverload.fset_useffi), prototype = interp2app(W_CPPOverload.prototype), ) @@ -611,7 +638,7 @@ self.scope.name) w_result = W_CPPOverload.call(self, w_cppinstance, args_w) newthis = rffi.cast(capi.C_OBJECT, self.space.uint_w(w_result)) - cppinstance = self.space.interp_w(W_CPPClass, w_cppinstance, can_be_None=True) + cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=True) if cppinstance is not None: cppinstance._rawobject = newthis memory_regulator.register(cppinstance) @@ -642,17 +669,18 @@ class W_CPPBoundMethod(W_Root): - _attrs_ = ['cppthis', 'method'] + _attrs_ = ['cppthis', 'method', 'useffi'] - def __init__(self, cppthis, method): + def __init__(self, cppthis, method, useffi): self.cppthis = cppthis self.method = method + self.useffi = useffi def __call__(self, args_w): - return self.method.bound_call(self.cppthis, args_w) + return self.method.bound_call(self.cppthis, args_w, self.useffi) def __repr__(self): - return "W_CPPBoundMethod(%s)" % [f.prototype() for f in self.functions] + return "W_CPPBoundMethod(%s)" % self.method.prototype() W_CPPBoundMethod.typedef = TypeDef( 'CPPBoundMethod', @@ -682,7 +710,7 @@ return offset def get(self, w_cppinstance, w_pycppclass): - cppinstance = self.space.interp_w(W_CPPClass, w_cppinstance, can_be_None=True) + cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=True) if not cppinstance: raise oefmt(self.space.w_AttributeError, "attribute access requires an instance") @@ -690,7 +718,7 @@ return self.converter.from_memory(self.space, w_cppinstance, w_pycppclass, offset) def set(self, w_cppinstance, w_value): - cppinstance = self.space.interp_w(W_CPPClass, w_cppinstance, can_be_None=True) + cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=True) if not cppinstance: raise oefmt(self.space.w_AttributeError, "attribute access requires an instance") @@ -845,24 +873,11 @@ return self.space.w_True def ns__dir__(self): - # Collect a list of everything (currently) available in the namespace. - # The backend can filter by returning empty strings. Special care is - # taken for functions, which need not be unique (overloading). - alldir = [] - for i in range(capi.c_num_scopes(self.space, self)): - sname = capi.c_scope_name(self.space, self, i) - if sname: alldir.append(self.space.newtext(sname)) - allmeth = {} - for i in range(capi.c_num_methods(self.space, self)): - idx = capi.c_method_index_at(self.space, self, i) - mname = capi.c_method_name(self.space, self, idx) - if mname: allmeth.setdefault(mname, 0) - for m in allmeth.keys(): - alldir.append(self.space.newtext(m)) - for i in range(capi.c_num_datamembers(self.space, self)): - dname = capi.c_datamember_name(self.space, self, i) - if dname: alldir.append(self.space.newtext(dname)) - return self.space.newlist(alldir) + alldir = capi.c_get_all_cpp_names(self.space, self) + w_alldir = self.space.newlist([]) + for name in alldir: + w_alldir.append(self.space.newtext(name)) + return w_alldir def missing_attribute_error(self, name): return oefmt(self.space.w_AttributeError, @@ -890,8 +905,7 @@ def _build_methods(self): assert len(self.methods) == 0 methods_temp = {} - for i in range(capi.c_num_methods(self.space, self)): - idx = capi.c_method_index_at(self.space, self, i) + for idx in range(capi.c_num_methods(self.space, self)): if capi.c_is_constructor(self.space, self, idx): pyname = '__init__' else: @@ -1029,7 +1043,7 @@ W_CPPComplexClassDecl.typedef.acceptable_as_base_class = False -class W_CPPClass(W_Root): +class W_CPPInstance(W_Root): _attrs_ = ['space', 'clsdecl', '_rawobject', 'flags', 'finalizer_registered'] _immutable_fields_ = ['clsdecl'] @@ -1109,8 +1123,8 @@ # find a global overload in gbl, in __gnu_cxx (for iterators), or in the # scopes of the argument classes (TODO: implement that last option) try: - # TODO: expecting w_other to be an W_CPPClass is too limiting - other = self.space.interp_w(W_CPPClass, w_other, can_be_None=False) + # TODO: expecting w_other to be an W_CPPInstance is too limiting + other = self.space.interp_w(W_CPPInstance, w_other, can_be_None=False) for name in ["", "__gnu_cxx", "__1"]: nss = scope_byname(self.space, name) meth_idx = capi.c_get_global_operator( @@ -1132,7 +1146,7 @@ # fallback 2: direct pointer comparison (the class comparison is needed since # the first data member in a struct and the struct have the same address) - other = self.space.interp_w(W_CPPClass, w_other, can_be_None=False) # TODO: factor out + other = self.space.interp_w(W_CPPInstance, w_other, can_be_None=False) # TODO: factor out iseq = (self._rawobject == other._rawobject) and (self.clsdecl == other.clsdecl) return self.space.newbool(iseq) @@ -1176,19 +1190,19 @@ if self.flags & INSTANCE_FLAGS_PYTHON_OWNS: self.destruct() -W_CPPClass.typedef = TypeDef( - 'CPPClass', - __python_owns__ = GetSetProperty(W_CPPClass.fget_python_owns, W_CPPClass.fset_python_owns), - __init__ = interp2app(W_CPPClass.instance__init__), - __eq__ = interp2app(W_CPPClass.instance__eq__), - __ne__ = interp2app(W_CPPClass.instance__ne__), - __nonzero__ = interp2app(W_CPPClass.instance__nonzero__), - __len__ = interp2app(W_CPPClass.instance__len__), - __cmp__ = interp2app(W_CPPClass.instance__cmp__), - __repr__ = interp2app(W_CPPClass.instance__repr__), - __destruct__ = interp2app(W_CPPClass.destruct), +W_CPPInstance.typedef = TypeDef( + 'CPPInstance', + __python_owns__ = GetSetProperty(W_CPPInstance.fget_python_owns, W_CPPInstance.fset_python_owns), + __init__ = interp2app(W_CPPInstance.instance__init__), + __eq__ = interp2app(W_CPPInstance.instance__eq__), + __ne__ = interp2app(W_CPPInstance.instance__ne__), + __nonzero__ = interp2app(W_CPPInstance.instance__nonzero__), + __len__ = interp2app(W_CPPInstance.instance__len__), + __cmp__ = interp2app(W_CPPInstance.instance__cmp__), + __repr__ = interp2app(W_CPPInstance.instance__repr__), + __destruct__ = interp2app(W_CPPInstance.destruct), ) -W_CPPClass.typedef.acceptable_as_base_class = True +W_CPPInstance.typedef.acceptable_as_base_class = True class MemoryRegulator: @@ -1200,7 +1214,7 @@ # Note that for now, the associated test carries an m_padding to make # a difference in the addresses. def __init__(self): - self.objects = rweakref.RWeakValueDictionary(int, W_CPPClass) + self.objects = rweakref.RWeakValueDictionary(int, W_CPPInstance) def register(self, obj): if not obj._rawobject: @@ -1266,8 +1280,8 @@ return obj # fresh creation - w_cppinstance = space.allocate_instance(W_CPPClass, w_pycppclass) - cppinstance = space.interp_w(W_CPPClass, w_cppinstance, can_be_None=False) + w_cppinstance = space.allocate_instance(W_CPPInstance, w_pycppclass) + cppinstance = space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=False) cppinstance.__init__(space, clsdecl, rawobject, is_ref, python_owns) memory_regulator.register(cppinstance) return w_cppinstance @@ -1311,7 +1325,7 @@ def move(space, w_obj): """Casts the given instance into an C++-style rvalue.""" - obj = space.interp_w(W_CPPClass, w_obj, can_be_None=True) + obj = space.interp_w(W_CPPInstance, w_obj, can_be_None=True) if obj: obj.flags |= INSTANCE_FLAGS_IS_R_VALUE return w_obj diff --git a/pypy/module/_cppyy/pythonify.py b/pypy/module/_cppyy/pythonify.py --- a/pypy/module/_cppyy/pythonify.py +++ b/pypy/module/_cppyy/pythonify.py @@ -7,7 +7,7 @@ # Metaclasses are needed to store C++ static data members as properties. Since # the interp-level does not support metaclasses, they are created at app-level. # These are the metaclass base classes: -class CPPMetaScope(type): +class CPPScope(type): def __getattr__(self, name): try: return get_scoped_pycppitem(self, name) # will cache on self @@ -15,11 +15,11 @@ raise AttributeError("%s object has no attribute '%s' (details: %s)" % (self, name, str(e))) -class CPPMetaNamespace(CPPMetaScope): +class CPPMetaNamespace(CPPScope): def __dir__(self): return self.__cppdecl__.__dir__() -class CPPMetaClass(CPPMetaScope): +class CPPClass(CPPScope): pass # namespace base class (class base class defined in _init_pythonify) @@ -173,7 +173,7 @@ # get a list of base classes for class creation bases = [get_pycppclass(base) for base in decl.get_base_names()] if not bases: - bases = [CPPClass,] + bases = [CPPInstance,] else: # it's possible that the required class now has been built if one of # the base classes uses it in e.g. a function interface @@ -214,7 +214,7 @@ # create a metaclass to allow properties (for static data write access) metabases = [type(base) for base in bases] - metacpp = type(CPPMetaScope)(cl_name+'_meta', _drop_cycles(metabases), d_meta) + metacpp = type(CPPScope)(cl_name+'_meta', _drop_cycles(metabases), d_meta) # create the python-side C++ class pycls = metacpp(cl_name, _drop_cycles(bases), d_class) @@ -412,11 +412,11 @@ # at pypy-c startup, rather than on the "import _cppyy" statement import _cppyy - # root of all proxy classes: CPPClass in pythonify exists to combine the - # CPPMetaScope metaclass with the interp-level CPPClassBase - global CPPClass - class CPPClass(_cppyy.CPPClassBase): - __metaclass__ = CPPMetaScope + # root of all proxy classes: CPPInstance in pythonify exists to combine + # the CPPScope metaclass with the interp-level CPPInstanceBase + global CPPInstance + class CPPInstance(_cppyy.CPPInstanceBase): + __metaclass__ = CPPScope pass # class generator callback @@ -438,9 +438,8 @@ gbl.std.move = _cppyy.move # install a type for enums to refer to - # TODO: this is correct for C++98, not for C++11 and in general there will - # be the same issue for all typedef'd builtin types setattr(gbl, 'internal_enum_type_t', int) + setattr(gbl, 'unsigned int', int) # if resolved # install for user access _cppyy.gbl = gbl diff --git a/pypy/module/_cppyy/src/dummy_backend.cxx b/pypy/module/_cppyy/src/dummy_backend.cxx --- a/pypy/module/_cppyy/src/dummy_backend.cxx +++ b/pypy/module/_cppyy/src/dummy_backend.cxx @@ -407,11 +407,9 @@ /* name to opaque C++ scope representation -------------------------------- */ -int cppyy_num_scopes(cppyy_scope_t handle) { - return 0; -} - char* cppyy_resolve_name(const char* cppitem_name) { + if (cppyy_is_enum(cppitem_name)) + return cppstring_to_cstring("internal_enum_type_t"); return cppstring_to_cstring(cppitem_name); } @@ -851,10 +849,13 @@ return (cppyy_object_t)result; } -cppyy_funcaddr_t cppyy_get_function_address(cppyy_scope_t /* scope */, cppyy_index_t /* idx */) { +cppyy_funcaddr_t cppyy_function_address_from_index(cppyy_scope_t /* scope */, cppyy_index_t /* idx */) { return (cppyy_funcaddr_t)0; } +cppyy_funcaddr_t cppyy_function_address_from_method(cppyy_method_t /* method */) { + return (cppyy_funcaddr_t)0; +} /* handling of function argument buffer ----------------------------------- */ void* cppyy_allocate_function_args(int nargs) { @@ -926,10 +927,6 @@ return s_scopes[handle].m_methods.size(); } -cppyy_index_t cppyy_method_index_at(cppyy_scope_t /* scope */, int imeth) { - return (cppyy_index_t)imeth; -} - char* cppyy_method_name(cppyy_scope_t handle, cppyy_index_t method_index) { return cppstring_to_cstring(s_scopes[handle].m_methods[(int)method_index].m_name); } @@ -978,8 +975,17 @@ return (cppyy_method_t)0; } +cppyy_index_t cppyy_get_global_operator(cppyy_scope_t /* scope */, + cppyy_scope_t /* lc */, cppyy_scope_t /* rc */, const char* /* op */) { + return (cppyy_index_t)-1; +} + /* method properties ----------------------------------------------------- */ +int cppyy_is_publicmethod(cppyy_type_t /* handle */, cppyy_index_t /* method_index */) { + return 1; +} + int cppyy_is_constructor(cppyy_type_t handle, cppyy_index_t method_index) { if (s_scopes.find(handle) != s_scopes.end()) return s_scopes[handle].m_methods[method_index].m_type == kConstructor; @@ -987,6 +993,10 @@ return 0; } +int cppyy_is_destructor(cppyy_type_t /* handle */, cppyy_index_t /* method_index */) { + return 0; +} + int cppyy_is_staticmethod(cppyy_type_t handle, cppyy_index_t method_index) { if (s_scopes.find(handle) != s_scopes.end()) return s_scopes[handle].m_methods[method_index].m_type == kStatic; @@ -1014,14 +1024,22 @@ /* data member properties ------------------------------------------------ */ -int cppyy_is_publicdata(cppyy_scope_t handle, int idatambr) { +int cppyy_is_publicdata(cppyy_scope_t /* handle */, cppyy_index_t /* idatambr */) { return 1; } -int cppyy_is_staticdata(cppyy_scope_t handle, int idatambr) { +int cppyy_is_staticdata(cppyy_scope_t handle, cppyy_index_t idatambr) { return s_scopes[handle].m_datambrs[idatambr].m_isstatic; } +int cppyy_is_const_data(cppyy_scope_t /* handle */, cppyy_index_t /* idatambr */) { + return 0; +} + +int cppyy_is_enum_data(cppyy_scope_t /* handle */, cppyy_index_t /* idatambr */) { + return 0; +} + /* misc helpers ----------------------------------------------------------- */ #if defined(_MSC_VER) diff --git a/pypy/module/_cppyy/test/Makefile b/pypy/module/_cppyy/test/Makefile --- a/pypy/module/_cppyy/test/Makefile +++ b/pypy/module/_cppyy/test/Makefile @@ -14,7 +14,7 @@ HASGENREFLEX:=$(shell command -v genreflex 2> /dev/null) -cppflags=-std=c++11 -O3 -m64 -fPIC -rdynamic +cppflags=-std=c++14 -O3 -m64 -fPIC -rdynamic ifdef HASGENREFLEX genreflex_flags:=$(shell genreflex --cppflags) cppflags+=$(genreflex_flags) diff --git a/pypy/module/_cppyy/test/advancedcpp.cxx b/pypy/module/_cppyy/test/advancedcpp.cxx --- a/pypy/module/_cppyy/test/advancedcpp.cxx +++ b/pypy/module/_cppyy/test/advancedcpp.cxx @@ -1,5 +1,7 @@ #include "advancedcpp.h" +#include + // for testing of default arguments #define IMPLEMENT_DEFAULTER_CLASS(type, tname) \ @@ -112,3 +114,13 @@ std::string overload_the_other_way::gime() { return "aap"; } int overload_the_other_way::gime() const { return 1; } + + +// exception handling testing +void Thrower::throw_anything() { + throw 1; +} + +void Thrower::throw_exception() { + throw std::runtime_error("C++ function failed"); +} diff --git a/pypy/module/_cppyy/test/advancedcpp.h b/pypy/module/_cppyy/test/advancedcpp.h --- a/pypy/module/_cppyy/test/advancedcpp.h +++ b/pypy/module/_cppyy/test/advancedcpp.h @@ -59,7 +59,7 @@ class a_class { // for esoteric inheritance testing public: a_class() { m_a = 1; m_da = 1.1; } - ~a_class() {} + virtual ~a_class() {} virtual int get_value() = 0; public: @@ -221,6 +221,7 @@ //=========================================================================== class some_abstract_class { // to test abstract class handling public: + virtual ~some_abstract_class() {} virtual void a_virtual_method() = 0; }; @@ -399,3 +400,11 @@ std::string gime(); int gime() const; }; + + +//=========================================================================== +class Thrower { // exception handling testing +public: + void throw_anything(); + void throw_exception(); +}; diff --git a/pypy/module/_cppyy/test/advancedcpp.xml b/pypy/module/_cppyy/test/advancedcpp.xml --- a/pypy/module/_cppyy/test/advancedcpp.xml +++ b/pypy/module/_cppyy/test/advancedcpp.xml @@ -57,4 +57,6 @@ + + diff --git a/pypy/module/_cppyy/test/conftest.py b/pypy/module/_cppyy/test/conftest.py --- a/pypy/module/_cppyy/test/conftest.py +++ b/pypy/module/_cppyy/test/conftest.py @@ -55,7 +55,7 @@ separate_module_files=[srcpath.join('dummy_backend.cxx')], include_dirs=[incpath, tstpath, cdir], compile_extra=['-DRPY_EXTERN=RPY_EXPORTED', '-DCPPYY_DUMMY_BACKEND', - '-fno-strict-aliasing', '-std=c++11'], + '-fno-strict-aliasing', '-std=c++14'], use_cpp_linker=True, ) @@ -65,7 +65,7 @@ outputfilename='libcppyy_dummy_backend', standalone=False) except CompilationError as e: - if '-std=c++11' in str(e): + if '-std=c++14' in str(e): global disabled disabled = str(e) return diff --git a/pypy/module/_cppyy/test/fragile.h b/pypy/module/_cppyy/test/fragile.h --- a/pypy/module/_cppyy/test/fragile.h +++ b/pypy/module/_cppyy/test/fragile.h @@ -30,6 +30,7 @@ void overload(int, no_such_class* p = 0) {} }; + static const int dummy_location = 0xdead; class E { @@ -105,6 +106,7 @@ class M { public: + virtual ~M() {} enum E1 { kOnce=42 }; enum E2 { kTwice=12 }; }; diff --git a/pypy/module/_cppyy/test/test_advancedcpp.py b/pypy/module/_cppyy/test/test_advancedcpp.py --- a/pypy/module/_cppyy/test/test_advancedcpp.py +++ b/pypy/module/_cppyy/test/test_advancedcpp.py @@ -656,3 +656,22 @@ # TODO: currently fails b/c double** not understood as &double* #assert cppyy.gbl.my_global_ptr[0] == 1234. + def test22_exceptions(self): + """Catching of C++ exceptions""" + + import _cppyy as cppyy + Thrower = cppyy.gbl.Thrower + + # TODO: clean up this interface: + Thrower.__cppdecl__.get_overload('throw_anything').__useffi__ = False + Thrower.__cppdecl__.get_overload('throw_exception').__useffi__ = False + + t = Thrower() + + assert raises(Exception, t.throw_anything) + assert raises(Exception, t.throw_exception) + + try: + t.throw_exception() + except Exception, e: + "C++ function failed" in str(e) diff --git a/pypy/module/_cppyy/test/test_overloads.py b/pypy/module/_cppyy/test/test_overloads.py --- a/pypy/module/_cppyy/test/test_overloads.py +++ b/pypy/module/_cppyy/test/test_overloads.py @@ -15,7 +15,6 @@ spaceconfig = dict(usemodules=['_cppyy', '_rawffi', 'itertools']) def setup_class(cls): - env = os.environ cls.w_test_dct = cls.space.newtext(test_dct) cls.w_overloads = cls.space.appexec([], """(): import ctypes diff --git a/pypy/module/_cppyy/test/test_zjit.py b/pypy/module/_cppyy/test/test_zjit.py --- a/pypy/module/_cppyy/test/test_zjit.py +++ b/pypy/module/_cppyy/test/test_zjit.py @@ -137,6 +137,7 @@ executor.get_executor(self, 'int').__class__.c_stubcall = staticmethod(c_call_i) self.w_AttributeError = FakeException(self, "AttributeError") + self.w_Exception = FakeException(self, "Exception") self.w_KeyError = FakeException(self, "KeyError") self.w_NotImplementedError = FakeException(self, "NotImplementedError") self.w_ReferenceError = FakeException(self, "ReferenceError") @@ -282,7 +283,7 @@ inst = interp_cppyy._bind_object(space, FakeInt(0), cls, True) cls.get_overload("__init__").call(inst, [FakeInt(0)]) cppmethod = cls.get_overload(method_name) - assert isinstance(inst, interp_cppyy.W_CPPClass) + assert isinstance(inst, interp_cppyy.W_CPPInstance) i = 10 while i > 0: drv.jit_merge_point(inst=inst, cppmethod=cppmethod, i=i) diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py --- a/pypy/objspace/std/listobject.py +++ b/pypy/objspace/std/listobject.py @@ -968,6 +968,14 @@ def setslice(self, w_list, start, step, slicelength, w_other): strategy = w_other.strategy + if step != 1: + len2 = strategy.length(w_other) + if len2 == 0: + return + else: + raise oefmt(self.space.w_ValueError, + "attempt to assign sequence of size %d to extended " + "slice of size %d", len2, 0) storage = strategy.getstorage_copy(w_other) w_list.strategy = strategy w_list.lstorage = storage diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py --- a/pypy/objspace/std/test/test_listobject.py +++ b/pypy/objspace/std/test/test_listobject.py @@ -1078,6 +1078,15 @@ l[::3] = ('a', 'b') assert l == ['a', 1.1, 2.2, 'b', 4.4, 5.5] + l_int = [5]; l_int.pop() # IntListStrategy + l_empty = [] # EmptyListStrategy + raises(ValueError, "l_int[::-1] = [42]") + raises(ValueError, "l_int[::7] = [42]") + raises(ValueError, "l_empty[::-1] = [42]") + raises(ValueError, "l_empty[::7] = [42]") + l_int[::1] = [42]; assert l_int == [42] + l_empty[::1] = [42]; assert l_empty == [42] + def test_setslice_with_self(self): l = [1,2,3,4] l[:] = l From pypy.commits at gmail.com Tue Apr 24 02:07:41 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 23 Apr 2018 23:07:41 -0700 (PDT) Subject: [pypy-commit] pypy release-pypy3.5-6.x: merge py3.5 into release Message-ID: <5adec9ad.1c69fb81.29b59.c7f7@mx.google.com> Author: Matti Picus Branch: release-pypy3.5-6.x Changeset: r94440:fdd60ed87e94 Date: 2018-04-24 09:05 +0300 http://bitbucket.org/pypy/pypy/changeset/fdd60ed87e94/ Log: merge py3.5 into release diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -247,6 +247,7 @@ Lukas Vacek Omer Katz Jacek Generowicz + Tomasz Dziopa Sylvain Thenault Jakub Stasiak Andrew Dalke @@ -307,6 +308,7 @@ Yury V. Zaytsev florinpapa Anders Sigfridsson + Matt Jackson Nikolay Zinov rafalgalczynski at gmail.com Joshua Gilbert diff --git a/lib-python/3/datetime.py b/lib-python/3/datetime.py --- a/lib-python/3/datetime.py +++ b/lib-python/3/datetime.py @@ -7,6 +7,9 @@ import time as _time import math as _math +# for cpyext, use these as base classes +from __pypy__._pypydatetime import dateinterop, deltainterop, timeinterop + def _cmp(x, y): return 0 if x == y else 1 if x > y else -1 @@ -316,7 +319,7 @@ return q -class timedelta: +class timedelta(deltainterop): """Represent the difference between two datetime objects. Supported operators: @@ -429,7 +432,7 @@ if abs(d) > 999999999: raise OverflowError("timedelta # of days is too large: %d" % d) - self = object.__new__(cls) + self = deltainterop.__new__(cls) self._days = d self._seconds = s self._microseconds = us @@ -638,7 +641,7 @@ microseconds=999999) timedelta.resolution = timedelta(microseconds=1) -class date: +class date(dateinterop): """Concrete date type. Constructors: @@ -678,12 +681,12 @@ if month is None and isinstance(year, bytes) and len(year) == 4 and \ 1 <= year[2] <= 12: # Pickle support - self = object.__new__(cls) + self = dateinterop.__new__(cls) self.__setstate(year) self._hashcode = -1 return self year, month, day = _check_date_fields(year, month, day) - self = object.__new__(cls) + self = dateinterop.__new__(cls) self._year = year self._month = month self._day = day @@ -1008,7 +1011,7 @@ _tzinfo_class = tzinfo -class time: +class time(timeinterop): """Time with time zone. Constructors: @@ -1044,14 +1047,14 @@ """ if isinstance(hour, bytes) and len(hour) == 6 and hour[0] < 24: # Pickle support - self = object.__new__(cls) + self = timeinterop.__new__(cls) self.__setstate(hour, minute or None) self._hashcode = -1 return self hour, minute, second, microsecond = _check_time_fields( hour, minute, second, microsecond) _check_tzinfo_arg(tzinfo) - self = object.__new__(cls) + self = timeinterop.__new__(cls) self._hour = hour self._minute = minute self._second = second @@ -1329,7 +1332,7 @@ microsecond=0, tzinfo=None): if isinstance(year, bytes) and len(year) == 10 and 1 <= year[2] <= 12: # Pickle support - self = object.__new__(cls) + self = dateinterop.__new__(cls) self.__setstate(year, month) self._hashcode = -1 return self @@ -1337,7 +1340,7 @@ hour, minute, second, microsecond = _check_time_fields( hour, minute, second, microsecond) _check_tzinfo_arg(tzinfo) - self = object.__new__(cls) + self = dateinterop.__new__(cls) self._year = year self._month = month self._day = day diff --git a/lib_pypy/_cffi_ssl/README.md b/lib_pypy/_cffi_ssl/README.md --- a/lib_pypy/_cffi_ssl/README.md +++ b/lib_pypy/_cffi_ssl/README.md @@ -14,6 +14,8 @@ * ``_cffi_src/openssl/x509_vfy.py`` for issue #2605 (ca4d0c90f5a1) +* ``_cffi_src/openssl/pypy_win32_extra.py`` for Win32-only functionality like ssl.enum_certificates() + # Tests? diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/pypy_win32_extra.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/pypy_win32_extra.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/pypy_win32_extra.py @@ -0,0 +1,84 @@ +# +# An extra bit of logic for the Win32-only functionality that is missing from the +# version from cryptography. +# + +import sys + +INCLUDES = """ +#include +""" + +TYPES = """ +typedef ... *HCERTSTORE; +typedef ... *HCRYPTPROV_LEGACY; + +typedef struct { + DWORD dwCertEncodingType; + BYTE *pbCertEncoded; + DWORD cbCertEncoded; + ...; +} CERT_CONTEXT, *PCCERT_CONTEXT; + +typedef struct { + DWORD dwCertEncodingType; + BYTE *pbCrlEncoded; + DWORD cbCrlEncoded; + ...; +} CRL_CONTEXT, *PCCRL_CONTEXT; + +typedef struct { + DWORD cUsageIdentifier; + LPSTR *rgpszUsageIdentifier; + ...; +} CERT_ENHKEY_USAGE, *PCERT_ENHKEY_USAGE; +""" + +FUNCTIONS = """ +HCERTSTORE WINAPI CertOpenStore( + LPCSTR lpszStoreProvider, + DWORD dwMsgAndCertEncodingType, + HCRYPTPROV_LEGACY hCryptProv, + DWORD dwFlags, + const char *pvPara +); +PCCERT_CONTEXT WINAPI CertEnumCertificatesInStore( + HCERTSTORE hCertStore, + PCCERT_CONTEXT pPrevCertContext +); +BOOL WINAPI CertFreeCertificateContext( + PCCERT_CONTEXT pCertContext +); +BOOL WINAPI CertFreeCRLContext( + PCCRL_CONTEXT pCrlContext +); +BOOL WINAPI CertCloseStore( + HCERTSTORE hCertStore, + DWORD dwFlags +); +BOOL WINAPI CertGetEnhancedKeyUsage( + PCCERT_CONTEXT pCertContext, + DWORD dwFlags, + PCERT_ENHKEY_USAGE pUsage, + DWORD *pcbUsage +); +PCCRL_CONTEXT WINAPI CertEnumCRLsInStore( + HCERTSTORE hCertStore, + PCCRL_CONTEXT pPrevCrlContext +); +""" + +MACROS = """ +#define CERT_STORE_READONLY_FLAG ... +#define CERT_SYSTEM_STORE_LOCAL_MACHINE ... +#define CRYPT_E_NOT_FOUND ... +#define CERT_FIND_PROP_ONLY_ENHKEY_USAGE_FLAG ... +#define CERT_FIND_EXT_ONLY_ENHKEY_USAGE_FLAG ... +#define X509_ASN_ENCODING ... +#define PKCS_7_ASN_ENCODING ... + +static const LPCSTR CERT_STORE_PROV_SYSTEM_A; +""" + +CUSTOMIZATIONS = """ +""" diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py --- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py +++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py @@ -24,6 +24,7 @@ from enum import IntEnum as _IntEnum if sys.platform == 'win32': + from _cffi_ssl._stdssl.win32_extra import enum_certificates, enum_crls HAVE_POLL = False else: from select import poll, POLLIN, POLLOUT diff --git a/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py b/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py @@ -0,0 +1,101 @@ +from _pypy_openssl import lib, ffi + + +def enum_certificates(store_name): + """Retrieve certificates from Windows' cert store. + +store_name may be one of 'CA', 'ROOT' or 'MY'. The system may provide +more cert storages, too. The function returns a list of (bytes, +encoding_type, trust) tuples. The encoding_type flag can be interpreted +with X509_ASN_ENCODING or PKCS_7_ASN_ENCODING. The trust setting is either +a set of OIDs or the boolean True. + """ + hStore = lib.CertOpenStore(lib.CERT_STORE_PROV_SYSTEM_A, 0, ffi.NULL, + lib.CERT_STORE_READONLY_FLAG | lib.CERT_SYSTEM_STORE_LOCAL_MACHINE, + bytes(store_name, "ascii")) + if hStore == ffi.NULL: + raise WindowsError(*ffi.getwinerror()) + + result = [] + pCertCtx = ffi.NULL + try: + while True: + pCertCtx = lib.CertEnumCertificatesInStore(hStore, pCertCtx) + if pCertCtx == ffi.NULL: + break + cert = ffi.buffer(pCertCtx.pbCertEncoded, pCertCtx.cbCertEncoded)[:] + enc = certEncodingType(pCertCtx.dwCertEncodingType) + keyusage = parseKeyUsage(pCertCtx, lib.CERT_FIND_PROP_ONLY_ENHKEY_USAGE_FLAG) + if keyusage is True: + keyusage = parseKeyUsage(pCertCtx, lib.CERT_FIND_EXT_ONLY_ENHKEY_USAGE_FLAG) + result.append((cert, enc, keyusage)) + finally: + if pCertCtx != ffi.NULL: + lib.CertFreeCertificateContext(pCertCtx) + if not lib.CertCloseStore(hStore, 0): + # This error case might shadow another exception. + raise WindowsError(*ffi.getwinerror()) + return result + + +def enum_crls(store_name): + """Retrieve CRLs from Windows' cert store. + +store_name may be one of 'CA', 'ROOT' or 'MY'. The system may provide +more cert storages, too. The function returns a list of (bytes, +encoding_type) tuples. The encoding_type flag can be interpreted with +X509_ASN_ENCODING or PKCS_7_ASN_ENCODING.""" + hStore = lib.CertOpenStore(lib.CERT_STORE_PROV_SYSTEM_A, 0, ffi.NULL, + lib.CERT_STORE_READONLY_FLAG | lib.CERT_SYSTEM_STORE_LOCAL_MACHINE, + bytes(store_name, "ascii")) + if hStore == ffi.NULL: + raise WindowsError(*ffi.getwinerror()) + + result = [] + pCrlCtx = ffi.NULL + try: + while True: + pCrlCtx = lib.CertEnumCRLsInStore(hStore, pCrlCtx) + if pCrlCtx == ffi.NULL: + break + crl = ffi.buffer(pCrlCtx.pbCrlEncoded, pCrlCtx.cbCrlEncoded)[:] + enc = certEncodingType(pCrlCtx.dwCertEncodingType) + result.append((crl, enc)) + finally: + if pCrlCtx != ffi.NULL: + lib.CertFreeCRLContext(pCrlCtx) + if not lib.CertCloseStore(hStore, 0): + # This error case might shadow another exception. + raise WindowsError(*ffi.getwinerror()) + return result + + +def certEncodingType(encodingType): + if encodingType == lib.X509_ASN_ENCODING: + return "x509_asn" + if encodingType == lib.PKCS_7_ASN_ENCODING: + return "pkcs_7_asn" + return encodingType + +def parseKeyUsage(pCertCtx, flags): + pSize = ffi.new("DWORD *") + if not lib.CertGetEnhancedKeyUsage(pCertCtx, flags, ffi.NULL, pSize): + error_with_message = ffi.getwinerror() + if error_with_message[0] == lib.CRYPT_E_NOT_FOUND: + return True + raise WindowsError(*error_with_message) + + pUsageMem = ffi.new("char[]", pSize[0]) + pUsage = ffi.cast("PCERT_ENHKEY_USAGE", pUsageMem) + if not lib.CertGetEnhancedKeyUsage(pCertCtx, flags, pUsage, pSize): + error_with_message = ffi.getwinerror() + if error_with_message[0] == lib.CRYPT_E_NOT_FOUND: + return True + raise WindowsError(*error_with_message) + + retval = set() + for i in range(pUsage.cUsageIdentifier): + if pUsage.rgpszUsageIdentifier[i]: + oid = ffi.string(pUsage.rgpszUsageIdentifier[i]).decode('ascii') + retval.add(oid) + return retval diff --git a/lib_pypy/_ssl/__init__.py b/lib_pypy/_ssl/__init__.py --- a/lib_pypy/_ssl/__init__.py +++ b/lib_pypy/_ssl/__init__.py @@ -16,12 +16,14 @@ RAND_egd = builtinify(RAND_egd) import sys -if sys.platform == "win32" and 'enum_certificates' not in globals(): - def enum_certificates(*args, **kwds): - import warnings - warnings.warn("ssl.enum_certificates() is not implemented") - return [] - def enum_crls(*args, **kwds): - import warnings - warnings.warn("ssl.enum_crls() is not implemented") - return [] +if sys.platform == "win32": + if 'enum_certificates' not in globals(): + def enum_certificates(*args, **kwds): + import warnings + warnings.warn("ssl.enum_certificates() is not implemented") + return [] + if 'enum_crls' not in globals(): + def enum_crls(*args, **kwds): + import warnings + warnings.warn("ssl.enum_crls() is not implemented") + return [] diff --git a/lib_pypy/_ssl_build.py b/lib_pypy/_ssl_build.py --- a/lib_pypy/_ssl_build.py +++ b/lib_pypy/_ssl_build.py @@ -5,6 +5,11 @@ from _cffi_ssl._cffi_src.build_openssl import (build_ffi_for_binding, _get_openssl_libraries, extra_link_args, compiler_type) +if sys.platform == "win32": + pypy_win32_extra = ["pypy_win32_extra"] +else: + pypy_win32_extra = [] + ffi = build_ffi_for_binding( module_name="_pypy_openssl", module_prefix="_cffi_src.openssl.", @@ -44,10 +49,10 @@ "x509_vfy", "pkcs7", "callbacks", - ], + ] + pypy_win32_extra, libraries=_get_openssl_libraries(sys.platform), extra_link_args=extra_link_args(compiler_type()), ) if __name__ == '__main__': - ffi.compile() + ffi.compile(verbose=True) diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -214,6 +214,7 @@ Lukas Vacek Omer Katz Jacek Generowicz + Tomasz Dziopa Sylvain Thenault Jakub Stasiak Andrew Dalke @@ -274,6 +275,7 @@ Yury V. Zaytsev florinpapa Anders Sigfridsson + Matt Jackson Nikolay Zinov rafalgalczynski at gmail.com Joshua Gilbert diff --git a/pypy/doc/release-v6.0.0.rst b/pypy/doc/release-v6.0.0.rst --- a/pypy/doc/release-v6.0.0.rst +++ b/pypy/doc/release-v6.0.0.rst @@ -20,16 +20,18 @@ The GC now has `hooks`_ to gain more insights into its performance -The Windows PyPy3.5 release is still considered beta-quality. There are open -issues with unicode handling especially around system calls and c-extensions. +The Matplotlib TkAgg backend now works with PyPy, as do pygame and pygobject_. -The Matplotlib TkAgg backend now works with PyPy, as do pygame and pygobject_. +We updated the `cffi`_ module included in PyPy to version 1.11.5, and the +`cppyy`_ backend to 0.6.0. Please use these to wrap your C and C++ code, +respectively, for a JIT friendly experience. As always, this release is 100% compatible with the previous one and fixed several issues and bugs raised by the growing community of PyPy users. We strongly recommend updating. -We updated the cffi module included in PyPy to version 1.11.5 +The Windows PyPy3.5 release is still considered beta-quality. There are open +issues with unicode handling especially around system calls and c-extensions. The utf8 branch that changes internal representation of unicode to utf8 did not make it into the release, so there is still more goodness coming. We also @@ -56,6 +58,8 @@ .. _pygobject: https://lazka.github.io/posts/2018-04_pypy-pygobject/index.html .. _`syntax errors`: https://morepypy.blogspot.com/2018/04/improving-syntaxerror-in-pypy.html .. _`hooks`: gc_info.html#gc-hooks +.. _`cffi`: http://cffi.readthedocs.io +.. _`cppyy`: https://cppyy.readthedocs.io What is PyPy? ============= diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -3,7 +3,7 @@ ========================== .. this is a revision shortly after release-pypy-6.0.0 -.. startrev: ad79cc0ce9a8 +.. startrev: e50e11af23f1 diff --git a/pypy/doc/whatsnew-pypy2-6.0.0.rst b/pypy/doc/whatsnew-pypy2-6.0.0.rst --- a/pypy/doc/whatsnew-pypy2-6.0.0.rst +++ b/pypy/doc/whatsnew-pypy2-6.0.0.rst @@ -125,4 +125,8 @@ .. branch: gc-hook-better-timestamp -Improve GC hooksd +Improve GC hooks + +.. branch: cppyy-packaging + +Update backend to 0.6.0 and support exceptions through wrappers diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -479,11 +479,17 @@ # 'action.fire()' happens to be called any time before # the corresponding perform(), the fire() has no # effect---which is the effect we want, because - # perform() will be called anyway. + # perform() will be called anyway. All such pending + # actions with _fired == True are still inside the old + # chained list. As soon as we reset _fired to False, + # we also reset _next to None and we are ready for + # another fire(). while action is not None: + next_action = action._next + action._next = None action._fired = False action.perform(ec, frame) - action._next, action = None, action._next + action = next_action self.action_dispatcher = action_dispatcher diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py --- a/pypy/interpreter/test/test_executioncontext.py +++ b/pypy/interpreter/test/test_executioncontext.py @@ -66,6 +66,47 @@ """) assert events == ['one'] + def test_fire_inside_perform(self): + # test what happens if we call AsyncAction.fire() while we are in the + # middle of an AsyncAction.perform(). In particular, this happens when + # PyObjectDeallocAction.fire() is called by rawrefcount: see issue + # 2805 + events = [] + + class Action1(executioncontext.AsyncAction): + _count = 0 + + def perform(self, ec, frame): + events.append('one') + if self._count == 0: + # a1 is no longer in the queue, so it will be enqueued + a1.fire() + # + # a2 is still in the queue, so the fire() is ignored and + # it's performed in its normal order, i.e. BEFORE a3 + a2.fire() + self._count += 1 + + class Action2(executioncontext.AsyncAction): + def perform(self, ec, frame): + events.append('two') + + class Action3(executioncontext.AsyncAction): + def perform(self, ec, frame): + events.append('three') + + space = self.space + a1 = Action1(space) + a2 = Action2(space) + a3 = Action3(space) + a1.fire() + a2.fire() + a3.fire() + space.appexec([], """(): + pass + """) + assert events == ['one', 'two', 'three', 'one'] + def test_periodic_action(self): from pypy.interpreter.executioncontext import ActionFlag diff --git a/pypy/module/__pypy__/interp_pypydatetime.py b/pypy/module/__pypy__/interp_pypydatetime.py --- a/pypy/module/__pypy__/interp_pypydatetime.py +++ b/pypy/module/__pypy__/interp_pypydatetime.py @@ -5,7 +5,7 @@ def create_class(name): class W_Class(W_Root): - 'builtin base clasee for datetime.%s to allow interop with cpyext' % name + 'builtin base class for datetime.%s to allow interop with cpyext' % name def descr_new__(space, w_type): return space.allocate_instance(W_Class, w_type) diff --git a/pypy/module/_cppyy/__init__.py b/pypy/module/_cppyy/__init__.py --- a/pypy/module/_cppyy/__init__.py +++ b/pypy/module/_cppyy/__init__.py @@ -13,7 +13,7 @@ '_set_function_generator': 'interp_cppyy.set_function_generator', '_register_class' : 'interp_cppyy.register_class', '_get_nullptr' : 'interp_cppyy.get_nullptr', - 'CPPClassBase' : 'interp_cppyy.W_CPPClass', + 'CPPInstanceBase' : 'interp_cppyy.W_CPPInstance', 'addressof' : 'interp_cppyy.addressof', '_bind_object' : 'interp_cppyy._bind_object', 'bind_object' : 'interp_cppyy.bind_object', diff --git a/pypy/module/_cppyy/capi/loadable_capi.py b/pypy/module/_cppyy/capi/loadable_capi.py --- a/pypy/module/_cppyy/capi/loadable_capi.py +++ b/pypy/module/_cppyy/capi/loadable_capi.py @@ -121,11 +121,11 @@ # TODO: the following need to match up with the globally defined C_XYZ low-level # types (see capi/__init__.py), but by using strings here, that isn't guaranteed - c_opaque_ptr = state.c_ulong + c_opaque_ptr = state.c_ulong # not ptrdiff_t (which is signed) c_scope = c_opaque_ptr c_type = c_scope - c_object = c_opaque_ptr + c_object = c_opaque_ptr # not voidp (to stick with one handle type) c_method = c_opaque_ptr c_index = state.c_long c_index_array = state.c_voidp @@ -150,16 +150,17 @@ self.capi_call_ifaces = { # name to opaque C++ scope representation - 'num_scopes' : ([c_scope], c_int), - 'scope_name' : ([c_scope, c_int], c_ccharp), - 'resolve_name' : ([c_ccharp], c_ccharp), + 'resolve_enum' : ([c_ccharp], c_ccharp), 'get_scope' : ([c_ccharp], c_scope), 'actual_class' : ([c_type, c_object], c_type), + 'size_of_klass' : ([c_type], c_size_t), + 'size_of_type' : ([c_ccharp], c_size_t), # memory management 'allocate' : ([c_type], c_object), 'deallocate' : ([c_type, c_object], c_void), + 'construct' : ([c_type], c_object), 'destruct' : ([c_type, c_object], c_void), # method/function dispatching @@ -182,7 +183,8 @@ 'constructor' : ([c_method, c_object, c_int, c_voidp], c_object), 'call_o' : ([c_method, c_object, c_int, c_voidp, c_type], c_object), - 'get_function_address' : ([c_scope, c_index], c_voidp), # TODO: verify + 'function_address_from_index' : ([c_scope, c_index], c_voidp), # TODO: verify + 'function_address_from_method' : ([c_method], c_voidp), # id. # handling of function argument buffer 'allocate_function_args' : ([c_int], c_voidp), @@ -196,6 +198,8 @@ 'is_abstract' : ([c_type], c_int), 'is_enum' : ([c_ccharp], c_int), + 'get_all_cpp_names' : ([c_scope, c_voidp], c_voidp), # const char** + # type/class reflection information 'final_name' : ([c_type], c_ccharp), 'scoped_final_name' : ([c_type], c_ccharp), @@ -208,10 +212,10 @@ # method/function reflection information 'num_methods' : ([c_scope], c_int), - 'method_index_at' : ([c_scope, c_int], c_index), 'method_indices_from_name' : ([c_scope, c_ccharp], c_index_array), 'method_name' : ([c_scope, c_index], c_ccharp), + 'method_mangled_name' : ([c_scope, c_index], c_ccharp), 'method_result_type' : ([c_scope, c_index], c_ccharp), 'method_num_args' : ([c_scope, c_index], c_int), 'method_req_args' : ([c_scope, c_index], c_int), @@ -219,7 +223,9 @@ 'method_arg_default' : ([c_scope, c_index, c_int], c_ccharp), 'method_signature' : ([c_scope, c_index, c_int], c_ccharp), 'method_prototype' : ([c_scope, c_index, c_int], c_ccharp), + 'is_const_method' : ([c_method], c_int), + 'exists_method_template' : ([c_scope, c_ccharp], c_int), 'method_is_template' : ([c_scope, c_index], c_int), 'method_num_template_args' : ([c_scope, c_index], c_int), 'method_template_arg_name' : ([c_scope, c_index, c_index], c_ccharp), @@ -228,7 +234,9 @@ 'get_global_operator' : ([c_scope, c_scope, c_scope, c_ccharp], c_index), # method properties + 'is_public_method' : ([c_type, c_index], c_int), 'is_constructor' : ([c_type, c_index], c_int), + 'is_destructor' : ([c_type, c_index], c_int), 'is_staticmethod' : ([c_type, c_index], c_int), # data member reflection information @@ -236,12 +244,14 @@ 'datamember_name' : ([c_scope, c_int], c_ccharp), 'datamember_type' : ([c_scope, c_int], c_ccharp), 'datamember_offset' : ([c_scope, c_int], c_ptrdiff_t), - 'datamember_index' : ([c_scope, c_ccharp], c_int), # data member properties 'is_publicdata' : ([c_scope, c_int], c_int), 'is_staticdata' : ([c_scope, c_int], c_int), + 'is_const_data' : ([c_scope, c_int], c_int), + 'is_enum_data' : ([c_scope, c_int], c_int), + 'get_dimension_size' : ([c_scope, c_int, c_int], c_int), # misc helpers 'strtoll' : ([c_ccharp], c_llong), @@ -328,25 +338,27 @@ return rffi.cast(rffi.CCHARP, ptr) # name to opaque C++ scope representation ------------------------------------ -def c_num_scopes(space, cppscope): - return space.int_w(call_capi(space, 'num_scopes', [_ArgH(cppscope.handle)])) -def c_scope_name(space, cppscope, iscope): - args = [_ArgH(cppscope.handle), _ArgL(iscope)] - return charp2str_free(space, call_capi(space, 'scope_name', args)) - def c_resolve_name(space, name): return charp2str_free(space, call_capi(space, 'resolve_name', [_ArgS(name)])) +def c_resolve_enum(space, name): + return charp2str_free(space, call_capi(space, 'resolve_enum', [_ArgS(name)])) def c_get_scope_opaque(space, name): return rffi.cast(C_SCOPE, space.uint_w(call_capi(space, 'get_scope', [_ArgS(name)]))) def c_actual_class(space, cppclass, cppobj): args = [_ArgH(cppclass.handle), _ArgH(cppobj)] return rffi.cast(C_TYPE, space.uint_w(call_capi(space, 'actual_class', args))) +def c_size_of_klass(space, cppclass): + return _cdata_to_size_t(space, call_capi(space, 'size_of_klass', [_ArgH(cppclass.handle)])) +def c_size_of_type(space, name): + return _cdata_to_size_t(space, call_capi(space, 'size_of_type', [_ArgS(name)])) # memory management ---------------------------------------------------------- def c_allocate(space, cppclass): return _cdata_to_cobject(space, call_capi(space, 'allocate', [_ArgH(cppclass.handle)])) def c_deallocate(space, cppclass, cppobject): call_capi(space, 'deallocate', [_ArgH(cppclass.handle), _ArgH(cppobject)]) +def c_construct(space, cppclass): + return _cdata_to_cobject(space, call_capi(space, 'construct', [_ArgH(cppclass.handle)])) def c_destruct(space, cppclass, cppobject): call_capi(space, 'destruct', [_ArgH(cppclass.handle), _ArgH(cppobject)]) @@ -391,7 +403,7 @@ w_cstr = call_capi(space, 'call_s', [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs), _ArgP(rffi.cast(rffi.VOIDP, length))]) - cstr_len = intmask(length[0]) + cstr_len = int(intmask(length[0])) finally: lltype.free(length, flavor='raw') return _cdata_to_ccharp(space, w_cstr), cstr_len @@ -403,10 +415,13 @@ args = [_ArgH(cppmethod), _ArgH(cppobject), _ArgL(nargs), _ArgP(cargs), _ArgH(cppclass.handle)] return _cdata_to_cobject(space, call_capi(space, 'call_o', args)) -def c_get_function_address(space, cppscope, index): +def c_function_address_from_index(space, cppscope, index): args = [_ArgH(cppscope.handle), _ArgL(index)] return rffi.cast(C_FUNC_PTR, - _cdata_to_ptr(space, call_capi(space, 'get_function_address', args))) + _cdata_to_ptr(space, call_capi(space, 'function_address_from_index', args))) +def c_function_address_from_method(space, cppmethod): + return rffi.cast(C_FUNC_PTR, + _cdata_to_ptr(space, call_capi(space, 'function_address_from_method', _ArgH(cppmethod)))) # handling of function argument buffer --------------------------------------- def c_allocate_function_args(space, size): @@ -430,6 +445,23 @@ def c_is_enum(space, name): return space.bool_w(call_capi(space, 'is_enum', [_ArgS(name)])) +def c_get_all_cpp_names(space, scope): + sz = lltype.malloc(rffi.SIZE_TP.TO, 1, flavor='raw', zero=True) + try: + args = [_ArgH(scope.handle), _ArgP(rffi.cast(rffi.VOIDP, sz))] + rawnames = rffi.cast(rffi.CCHARPP, + _cdata_to_ptr(space, call_capi(space, 'get_all_cpp_names', args))) + count = int(intmask(sz[0])) + finally: + lltype.free(sz, flavor='raw') + allnames = [] + for i in range(count): + pystr = rffi.charp2str(rawnames[i]) + c_free(space, rffi.cast(rffi.VOIDP, rawnames[i])) # c_free defined below + allnames.append(pystr) + c_free(space, rffi.cast(rffi.VOIDP, rawnames)) # id. + return allnames + # type/class reflection information ------------------------------------------ def c_final_name(space, cpptype): return charp2str_free(space, call_capi(space, 'final_name', [_ArgH(cpptype)])) @@ -462,13 +494,10 @@ def c_num_methods(space, cppscope): args = [_ArgH(cppscope.handle)] return space.int_w(call_capi(space, 'num_methods', args)) -def c_method_index_at(space, cppscope, imethod): - args = [_ArgH(cppscope.handle), _ArgL(imethod)] - return space.int_w(call_capi(space, 'method_index_at', args)) def c_method_indices_from_name(space, cppscope, name): args = [_ArgH(cppscope.handle), _ArgS(name)] indices = rffi.cast(C_INDEX_ARRAY, - _cdata_to_ptr(space, call_capi(space, 'method_indices_from_name', args))) + _cdata_to_ptr(space, call_capi(space, 'method_indices_from_name', args))) if not indices: return [] py_indices = [] @@ -506,6 +535,9 @@ args = [_ArgH(cppscope.handle), _ArgL(index), _ArgL(show_formalargs)] return charp2str_free(space, call_capi(space, 'method_prototype', args)) +def c_exists_method_template(space, cppscope, name): + args = [_ArgH(cppscope.handle), _ArgS(name)] + return space.bool_w(call_capi(space, 'exists_method_template', args)) def c_method_is_template(space, cppscope, index): args = [_ArgH(cppscope.handle), _ArgL(index)] return space.bool_w(call_capi(space, 'method_is_template', args)) @@ -531,9 +563,15 @@ return rffi.cast(WLAVC_INDEX, -1) # method properties ---------------------------------------------------------- +def c_is_public_method(space, cppclass, index): + args = [_ArgH(cppclass.handle), _ArgL(index)] + return space.bool_w(call_capi(space, 'is_public_method', args)) def c_is_constructor(space, cppclass, index): args = [_ArgH(cppclass.handle), _ArgL(index)] return space.bool_w(call_capi(space, 'is_constructor', args)) +def c_is_destructor(space, cppclass, index): + args = [_ArgH(cppclass.handle), _ArgL(index)] + return space.bool_w(call_capi(space, 'is_destructor', args)) def c_is_staticmethod(space, cppclass, index): args = [_ArgH(cppclass.handle), _ArgL(index)] return space.bool_w(call_capi(space, 'is_staticmethod', args)) @@ -562,6 +600,15 @@ def c_is_staticdata(space, cppscope, datamember_index): args = [_ArgH(cppscope.handle), _ArgL(datamember_index)] return space.bool_w(call_capi(space, 'is_staticdata', args)) +def c_is_const_data(space, cppscope, datamember_index): + args = [_ArgH(cppscope.handle), _ArgL(datamember_index)] + return space.bool_w(call_capi(space, 'is_const_data', args)) +def c_is_enum_data(space, cppscope, datamember_index): + args = [_ArgH(cppscope.handle), _ArgL(datamember_index)] + return space.bool_w(call_capi(space, 'is_enum_data', args)) +def c_get_dimension_size(space, cppscope, datamember_index, dim_idx): + args = [_ArgH(cppscope.handle), _ArgL(datamember_index), _ArgL(dim_idx)] + return space.bool_w(call_capi(space, 'get_dimension_size', args)) # misc helpers --------------------------------------------------------------- def c_strtoll(space, svalue): @@ -585,7 +632,7 @@ try: w_cstr = call_capi(space, 'stdstring2charp', [_ArgH(cppstr), _ArgP(rffi.cast(rffi.VOIDP, sz))]) - cstr_len = intmask(sz[0]) + cstr_len = int(intmask(sz[0])) finally: lltype.free(sz, flavor='raw') return rffi.charpsize2str(_cdata_to_ccharp(space, w_cstr), cstr_len) @@ -607,7 +654,7 @@ """Return a python string taking into account \0""" from pypy.module._cppyy import interp_cppyy - cppstr = space.interp_w(interp_cppyy.W_CPPClass, w_self, can_be_None=False) + cppstr = space.interp_w(interp_cppyy.W_CPPInstance, w_self, can_be_None=False) return space.newtext(c_stdstring2charp(space, cppstr._rawobject)) # setup pythonizations for later use at run-time diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py --- a/pypy/module/_cppyy/converter.py +++ b/pypy/module/_cppyy/converter.py @@ -22,8 +22,8 @@ def get_rawobject(space, w_obj, can_be_None=True): - from pypy.module._cppyy.interp_cppyy import W_CPPClass - cppinstance = space.interp_w(W_CPPClass, w_obj, can_be_None=can_be_None) + from pypy.module._cppyy.interp_cppyy import W_CPPInstance + cppinstance = space.interp_w(W_CPPInstance, w_obj, can_be_None=can_be_None) if cppinstance: rawobject = cppinstance.get_rawobject() assert lltype.typeOf(rawobject) == capi.C_OBJECT @@ -31,15 +31,15 @@ return capi.C_NULL_OBJECT def set_rawobject(space, w_obj, address): - from pypy.module._cppyy.interp_cppyy import W_CPPClass - cppinstance = space.interp_w(W_CPPClass, w_obj, can_be_None=True) + from pypy.module._cppyy.interp_cppyy import W_CPPInstance + cppinstance = space.interp_w(W_CPPInstance, w_obj, can_be_None=True) if cppinstance: assert lltype.typeOf(cppinstance._rawobject) == capi.C_OBJECT cppinstance._rawobject = rffi.cast(capi.C_OBJECT, address) def get_rawobject_nonnull(space, w_obj): - from pypy.module._cppyy.interp_cppyy import W_CPPClass - cppinstance = space.interp_w(W_CPPClass, w_obj, can_be_None=True) + from pypy.module._cppyy.interp_cppyy import W_CPPInstance + cppinstance = space.interp_w(W_CPPInstance, w_obj, can_be_None=True) if cppinstance: cppinstance._nullcheck() rawobject = cppinstance.get_rawobject() @@ -502,8 +502,8 @@ self.clsdecl = clsdecl def _unwrap_object(self, space, w_obj): - from pypy.module._cppyy.interp_cppyy import W_CPPClass - if isinstance(w_obj, W_CPPClass): + from pypy.module._cppyy.interp_cppyy import W_CPPInstance + if isinstance(w_obj, W_CPPInstance): from pypy.module._cppyy.interp_cppyy import INSTANCE_FLAGS_IS_R_VALUE if w_obj.flags & INSTANCE_FLAGS_IS_R_VALUE: # reject moves as all are explicit @@ -534,8 +534,8 @@ class InstanceMoveConverter(InstanceRefConverter): def _unwrap_object(self, space, w_obj): # moving is same as by-ref, but have to check that move is allowed - from pypy.module._cppyy.interp_cppyy import W_CPPClass, INSTANCE_FLAGS_IS_R_VALUE - if isinstance(w_obj, W_CPPClass): + from pypy.module._cppyy.interp_cppyy import W_CPPInstance, INSTANCE_FLAGS_IS_R_VALUE + if isinstance(w_obj, W_CPPInstance): if w_obj.flags & INSTANCE_FLAGS_IS_R_VALUE: w_obj.flags &= ~INSTANCE_FLAGS_IS_R_VALUE return InstanceRefConverter._unwrap_object(self, space, w_obj) @@ -598,8 +598,8 @@ raise FastCallNotPossible def finalize_call(self, space, w_obj, call_local): - from pypy.module._cppyy.interp_cppyy import W_CPPClass - assert isinstance(w_obj, W_CPPClass) + from pypy.module._cppyy.interp_cppyy import W_CPPInstance + assert isinstance(w_obj, W_CPPInstance) r = rffi.cast(rffi.VOIDPP, call_local) w_obj._rawobject = rffi.cast(capi.C_OBJECT, r[0]) @@ -617,8 +617,8 @@ InstanceConverter.__init__(self, space, cppclass) def _unwrap_object(self, space, w_obj): - from pypy.module._cppyy.interp_cppyy import W_CPPClass - if isinstance(w_obj, W_CPPClass): + from pypy.module._cppyy.interp_cppyy import W_CPPInstance + if isinstance(w_obj, W_CPPInstance): arg = InstanceConverter._unwrap_object(self, space, w_obj) return capi.c_stdstring2stdstring(space, arg) else: @@ -749,8 +749,6 @@ return InstancePtrPtrConverter(space, clsdecl) elif compound == "": return InstanceConverter(space, clsdecl) - elif capi.c_is_enum(space, clean_name): - return _converters['unsigned'](space, default) # 5) void* or void converter (which fails on use) if 0 <= compound.find('*'): diff --git a/pypy/module/_cppyy/executor.py b/pypy/module/_cppyy/executor.py --- a/pypy/module/_cppyy/executor.py +++ b/pypy/module/_cppyy/executor.py @@ -293,8 +293,6 @@ return InstancePtrExecutor(space, cppclass) elif compound == '**' or compound == '*&': return InstancePtrPtrExecutor(space, cppclass) - elif capi.c_is_enum(space, clean_name): - return _executors['internal_enum_type_t'](space, None) # 4) additional special cases if compound == '*': diff --git a/pypy/module/_cppyy/include/capi.h b/pypy/module/_cppyy/include/capi.h --- a/pypy/module/_cppyy/include/capi.h +++ b/pypy/module/_cppyy/include/capi.h @@ -8,26 +8,29 @@ extern "C" { #endif // ifdef __cplusplus - typedef unsigned long cppyy_scope_t; + typedef ptrdiff_t cppyy_scope_t; typedef cppyy_scope_t cppyy_type_t; - typedef unsigned long cppyy_object_t; - typedef unsigned long cppyy_method_t; + typedef void* cppyy_object_t; + typedef ptrdiff_t cppyy_method_t; + typedef long cppyy_index_t; typedef void* cppyy_funcaddr_t; + typedef unsigned long cppyy_exctype_t; + /* name to opaque C++ scope representation -------------------------------- */ RPY_EXTERN - int cppyy_num_scopes(cppyy_scope_t parent); + char* cppyy_resolve_name(const char* cppitem_name); RPY_EXTERN - char* cppyy_scope_name(cppyy_scope_t parent, cppyy_index_t iscope); - RPY_EXTERN - char* cppyy_resolve_name(const char* cppitem_name); + char* cppyy_resolve_enum(const char* enum_type); RPY_EXTERN cppyy_scope_t cppyy_get_scope(const char* scope_name); RPY_EXTERN cppyy_type_t cppyy_actual_class(cppyy_type_t klass, cppyy_object_t obj); RPY_EXTERN - size_t cppyy_size_of(cppyy_type_t klass); + size_t cppyy_size_of_klass(cppyy_type_t klass); + RPY_EXTERN + size_t cppyy_size_of_type(const char* type_name); /* memory management ------------------------------------------------------ */ RPY_EXTERN @@ -35,48 +38,53 @@ RPY_EXTERN void cppyy_deallocate(cppyy_type_t type, cppyy_object_t self); RPY_EXTERN + cppyy_object_t cppyy_construct(cppyy_type_t type); + RPY_EXTERN void cppyy_destruct(cppyy_type_t type, cppyy_object_t self); /* method/function dispatching -------------------------------------------- */ RPY_EXTERN - void cppyy_call_v(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); + void cppyy_call_v(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN unsigned char cppyy_call_b(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN - char cppyy_call_c(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); + char cppyy_call_c(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN - short cppyy_call_h(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); + short cppyy_call_h(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN - int cppyy_call_i(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); + int cppyy_call_i(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN - long cppyy_call_l(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); + long cppyy_call_l(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN long long cppyy_call_ll(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN - float cppyy_call_f(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); + float cppyy_call_f(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN double cppyy_call_d(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN long double cppyy_call_ld(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN - void* cppyy_call_r(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); + void* cppyy_call_r(cppyy_method_t method, cppyy_object_t self, int nargs, void* args); RPY_EXTERN - char* cppyy_call_s(cppyy_method_t method, cppyy_object_t self, int nargs, void* args, size_t* length); - + char* cppyy_call_s(cppyy_method_t method, cppyy_object_t self, int nargs, void* args, size_t* length); RPY_EXTERN cppyy_object_t cppyy_constructor(cppyy_method_t method, cppyy_type_t klass, int nargs, void* args); RPY_EXTERN + void cppyy_destructor(cppyy_type_t type, cppyy_object_t self); + RPY_EXTERN cppyy_object_t cppyy_call_o(cppyy_method_t method, cppyy_object_t self, int nargs, void* args, cppyy_type_t result_type); RPY_EXTERN - cppyy_funcaddr_t cppyy_get_function_address(cppyy_scope_t scope, cppyy_index_t idx); + cppyy_funcaddr_t cppyy_function_address_from_index(cppyy_scope_t scope, cppyy_index_t idx); + RPY_EXTERN + cppyy_funcaddr_t cppyy_function_address_from_method(cppyy_method_t method); /* handling of function argument buffer ----------------------------------- */ RPY_EXTERN - void* cppyy_allocate_function_args(int nargs); + void* cppyy_allocate_function_args(int nargs); RPY_EXTERN - void cppyy_deallocate_function_args(void* args); + void cppyy_deallocate_function_args(void* args); RPY_EXTERN size_t cppyy_function_arg_sizeof(); RPY_EXTERN @@ -92,6 +100,9 @@ RPY_EXTERN int cppyy_is_enum(const char* type_name); + RPY_EXTERN + const char** cppyy_get_all_cpp_names(cppyy_scope_t scope, size_t* count); + /* class reflection information ------------------------------------------- */ RPY_EXTERN char* cppyy_final_name(cppyy_type_t type); @@ -105,6 +116,10 @@ char* cppyy_base_name(cppyy_type_t type, int base_index); RPY_EXTERN int cppyy_is_subtype(cppyy_type_t derived, cppyy_type_t base); + RPY_EXTERN + int cppyy_smartptr_info(const char* name, cppyy_type_t* raw, cppyy_method_t* deref); + RPY_EXTERN + void cppyy_add_smartptr_type(const char* type_name); /* calculate offsets between declared and actual type, up-cast: direction > 0; down-cast: direction < 0 */ RPY_EXTERN @@ -114,8 +129,6 @@ RPY_EXTERN int cppyy_num_methods(cppyy_scope_t scope); RPY_EXTERN - cppyy_index_t cppyy_method_index_at(cppyy_scope_t scope, int imeth); - RPY_EXTERN cppyy_index_t* cppyy_method_indices_from_name(cppyy_scope_t scope, const char* name); RPY_EXTERN @@ -136,8 +149,12 @@ char* cppyy_method_signature(cppyy_scope_t scope, cppyy_index_t idx, int show_formalargs); RPY_EXTERN char* cppyy_method_prototype(cppyy_scope_t scope, cppyy_index_t idx, int show_formalargs); + RPY_EXTERN + int cppyy_is_const_method(cppyy_method_t); RPY_EXTERN + int cppyy_exists_method_template(cppyy_scope_t scope, const char* name); + RPY_EXTERN int cppyy_method_is_template(cppyy_scope_t scope, cppyy_index_t idx); RPY_EXTERN int cppyy_method_num_template_args(cppyy_scope_t scope, cppyy_index_t idx); @@ -169,15 +186,20 @@ char* cppyy_datamember_type(cppyy_scope_t scope, int datamember_index); RPY_EXTERN ptrdiff_t cppyy_datamember_offset(cppyy_scope_t scope, int datamember_index); - RPY_EXTERN int cppyy_datamember_index(cppyy_scope_t scope, const char* name); /* data member properties ------------------------------------------------- */ RPY_EXTERN - int cppyy_is_publicdata(cppyy_type_t type, int datamember_index); + int cppyy_is_publicdata(cppyy_type_t type, cppyy_index_t datamember_index); RPY_EXTERN - int cppyy_is_staticdata(cppyy_type_t type, int datamember_index); + int cppyy_is_staticdata(cppyy_type_t type, cppyy_index_t datamember_index); + RPY_EXTERN + int cppyy_is_const_data(cppyy_scope_t scope, cppyy_index_t idata); + RPY_EXTERN + int cppyy_is_enum_data(cppyy_scope_t scope, cppyy_index_t idata); + RPY_EXTERN + int cppyy_get_dimension_size(cppyy_scope_t scope, cppyy_index_t idata, int dimension); /* misc helpers ----------------------------------------------------------- */ RPY_EXTERN @@ -197,7 +219,7 @@ RPY_EXTERN const char* cppyy_stdvector_valuetype(const char* clname); RPY_EXTERN - size_t cppyy_stdvector_valuesize(const char* clname); + size_t cppyy_stdvector_valuesize(const char* clname); #ifdef __cplusplus } diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py --- a/pypy/module/_cppyy/interp_cppyy.py +++ b/pypy/module/_cppyy/interp_cppyy.py @@ -19,6 +19,9 @@ INSTANCE_FLAGS_IS_REF = 0x0002 INSTANCE_FLAGS_IS_R_VALUE = 0x0004 +OVERLOAD_FLAGS_USE_FFI = 0x0001 + + class FastCallNotPossible(Exception): pass @@ -174,7 +177,7 @@ @staticmethod def unpack_cppthis(space, w_cppinstance, declaring_scope): - cppinstance = space.interp_w(W_CPPClass, w_cppinstance, can_be_None=False) + cppinstance = space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=False) cppinstance._nullcheck() return cppinstance.get_cppthis(declaring_scope) @@ -186,7 +189,7 @@ return rffi.cast(rffi.VOIDP, loc_idx) @jit.unroll_safe - def call(self, cppthis, args_w): + def call(self, cppthis, args_w, useffi): jit.promote(self) assert lltype.typeOf(cppthis) == capi.C_OBJECT @@ -218,16 +221,23 @@ try: # attempt to call directly through ffi chain - if self._funcaddr: + if useffi and self._funcaddr: try: return self.do_fast_call(cppthis, args_w, call_local) except FastCallNotPossible: pass # can happen if converters or executor does not implement ffi # ffi chain must have failed; using stub functions instead - args = self.prepare_arguments(args_w, call_local) + args, stat = self.prepare_arguments(args_w, call_local) try: - return self.executor.execute(self.space, self.cppmethod, cppthis, len(args_w), args) + result = self.executor.execute( + self.space, self.cppmethod, cppthis, len(args_w), args) + if stat[0] != rffi.cast(rffi.ULONG, 0): + what = rffi.cast(rffi.CCHARP, stat[1]) + pywhat = rffi.charp2str(what) + capi.c_free(self.space, rffi.cast(rffi.VOIDP, what)) + raise OperationError(self.space.w_Exception, self.space.newtext(pywhat)) + return result finally: self.finalize_call(args, args_w, call_local) finally: @@ -322,7 +332,7 @@ # Each CPPMethod corresponds one-to-one to a C++ equivalent and cppthis # has been offset to the matching class. Hence, the libffi pointer is # uniquely defined and needs to be setup only once. - funcaddr = capi.c_get_function_address(self.space, self.scope, self.index) + funcaddr = capi.c_function_address_from_index(self.space, self.scope, self.index) if funcaddr and cppthis: # methods only for now state = self.space.fromcache(ffitypes.State) @@ -373,7 +383,10 @@ conv.free_argument(self.space, rffi.cast(capi.C_OBJECT, arg_j), loc_j) capi.c_deallocate_function_args(self.space, args) raise - return args + stat = rffi.cast(rffi.ULONGP, + lltype.direct_ptradd(rffi.cast(rffi.CCHARP, args), int(len(args_w))*stride)) + stat[0] = rffi.cast(rffi.ULONG, 0) + return args, stat @jit.unroll_safe def finalize_call(self, args, args_w, call_local): @@ -435,7 +448,7 @@ # TODO: might have to specialize for CPPTemplatedCall on CPPMethod/CPPFunction here CPPMethod.__init__(self, space, declaring_scope, method_index, arg_defs, args_required) - def call(self, cppthis, args_w): + def call(self, cppthis, args_w, useffi): assert lltype.typeOf(cppthis) == capi.C_OBJECT for i in range(len(args_w)): try: @@ -447,10 +460,10 @@ raise oefmt(self.space.w_TypeError, "non-matching template (got %s where %s expected)", s, self.templ_args[i]) - return W_CPPBoundMethod(cppthis, self) + return W_CPPBoundMethod(cppthis, self, useffi) - def bound_call(self, cppthis, args_w): - return CPPMethod.call(self, cppthis, args_w) + def bound_call(self, cppthis, args_w, useffi): + return CPPMethod.call(self, cppthis, args_w, useffi) def __repr__(self): return "CPPTemplatedCall: %s" % self.prototype() @@ -468,11 +481,11 @@ def unpack_cppthis(space, w_cppinstance, declaring_scope): return rffi.cast(capi.C_OBJECT, declaring_scope.handle) - def call(self, cppthis, args_w): + def call(self, cppthis, args_w, useffi): # Note: this does not return a wrapped instance, just a pointer to the # new instance; the overload must still wrap it before returning. Also, # cppthis is declaring_scope.handle (as per unpack_cppthis(), above). - return CPPMethod.call(self, cppthis, args_w) + return CPPMethod.call(self, cppthis, args_w, useffi) def __repr__(self): return "CPPConstructor: %s" % self.prototype() @@ -485,7 +498,7 @@ _immutable_ = True - def call(self, cppthis, args_w): + def call(self, cppthis, args_w, useffi): end = len(args_w)-1 if 0 <= end: w_item = args_w[end] @@ -493,7 +506,7 @@ if self.converters is None: self._setup(cppthis) self.executor.set_item(self.space, w_item) # TODO: what about threads? - CPPMethod.call(self, cppthis, args_w) + CPPMethod.call(self, cppthis, args_w, useffi) class W_CPPOverload(W_Root): @@ -501,7 +514,7 @@ collection of (possibly) overloaded methods or functions. It calls these in order and deals with error handling and reporting.""" - _attrs_ = ['space', 'scope', 'functions'] + _attrs_ = ['space', 'scope', 'functions', 'flags'] _immutable_fields_ = ['scope', 'functions[*]'] def __init__(self, space, declaring_scope, functions): @@ -510,6 +523,19 @@ assert len(functions) from rpython.rlib import debug self.functions = debug.make_sure_not_resized(functions) + self.flags = 0 + self.flags |= OVERLOAD_FLAGS_USE_FFI + + # allow user to determine ffi use rules per overload + def fget_useffi(self, space): + return space.newbool(bool(self.flags & OVERLOAD_FLAGS_USE_FFI)) + + @unwrap_spec(value=bool) + def fset_useffi(self, space, value): + if space.is_true(value): + self.flags |= OVERLOAD_FLAGS_USE_FFI + else: + self.flags &= ~OVERLOAD_FLAGS_USE_FFI @jit.elidable_promote() def is_static(self): @@ -540,7 +566,7 @@ for i in range(len(self.functions)): cppyyfunc = self.functions[i] try: - return cppyyfunc.call(cppthis, args_w) + return cppyyfunc.call(cppthis, args_w, self.flags & OVERLOAD_FLAGS_USE_FFI) except Exception: pass @@ -553,7 +579,7 @@ for i in range(len(self.functions)): cppyyfunc = self.functions[i] try: - return cppyyfunc.call(cppthis, args_w) + return cppyyfunc.call(cppthis, args_w, self.flags & OVERLOAD_FLAGS_USE_FFI) except OperationError as e: # special case if there's just one function, to prevent clogging the error message if len(self.functions) == 1: @@ -588,6 +614,7 @@ 'CPPOverload', is_static = interp2app(W_CPPOverload.is_static), call = interp2app(W_CPPOverload.call), + __useffi__ = GetSetProperty(W_CPPOverload.fget_useffi, W_CPPOverload.fset_useffi), prototype = interp2app(W_CPPOverload.prototype), ) @@ -611,7 +638,7 @@ self.scope.name) w_result = W_CPPOverload.call(self, w_cppinstance, args_w) newthis = rffi.cast(capi.C_OBJECT, self.space.uint_w(w_result)) - cppinstance = self.space.interp_w(W_CPPClass, w_cppinstance, can_be_None=True) + cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=True) if cppinstance is not None: cppinstance._rawobject = newthis memory_regulator.register(cppinstance) @@ -642,17 +669,18 @@ class W_CPPBoundMethod(W_Root): - _attrs_ = ['cppthis', 'method'] + _attrs_ = ['cppthis', 'method', 'useffi'] - def __init__(self, cppthis, method): + def __init__(self, cppthis, method, useffi): self.cppthis = cppthis self.method = method + self.useffi = useffi def __call__(self, args_w): - return self.method.bound_call(self.cppthis, args_w) + return self.method.bound_call(self.cppthis, args_w, self.useffi) def __repr__(self): - return "W_CPPBoundMethod(%s)" % [f.prototype() for f in self.functions] + return "W_CPPBoundMethod(%s)" % self.method.prototype() W_CPPBoundMethod.typedef = TypeDef( 'CPPBoundMethod', @@ -682,7 +710,7 @@ return offset def get(self, w_cppinstance, w_pycppclass): - cppinstance = self.space.interp_w(W_CPPClass, w_cppinstance, can_be_None=True) + cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=True) if not cppinstance: raise oefmt(self.space.w_AttributeError, "attribute access requires an instance") @@ -690,7 +718,7 @@ return self.converter.from_memory(self.space, w_cppinstance, w_pycppclass, offset) def set(self, w_cppinstance, w_value): - cppinstance = self.space.interp_w(W_CPPClass, w_cppinstance, can_be_None=True) + cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=True) if not cppinstance: raise oefmt(self.space.w_AttributeError, "attribute access requires an instance") @@ -845,24 +873,11 @@ return self.space.w_True def ns__dir__(self): - # Collect a list of everything (currently) available in the namespace. - # The backend can filter by returning empty strings. Special care is - # taken for functions, which need not be unique (overloading). - alldir = [] - for i in range(capi.c_num_scopes(self.space, self)): - sname = capi.c_scope_name(self.space, self, i) - if sname: alldir.append(self.space.newtext(sname)) - allmeth = {} - for i in range(capi.c_num_methods(self.space, self)): - idx = capi.c_method_index_at(self.space, self, i) - mname = capi.c_method_name(self.space, self, idx) - if mname: allmeth.setdefault(mname, 0) - for m in allmeth.keys(): - alldir.append(self.space.newtext(m)) - for i in range(capi.c_num_datamembers(self.space, self)): - dname = capi.c_datamember_name(self.space, self, i) - if dname: alldir.append(self.space.newtext(dname)) - return self.space.newlist(alldir) + alldir = capi.c_get_all_cpp_names(self.space, self) + w_alldir = self.space.newlist([]) + for name in alldir: + w_alldir.append(self.space.newtext(name)) + return w_alldir def missing_attribute_error(self, name): return oefmt(self.space.w_AttributeError, @@ -890,8 +905,7 @@ def _build_methods(self): assert len(self.methods) == 0 methods_temp = {} - for i in range(capi.c_num_methods(self.space, self)): - idx = capi.c_method_index_at(self.space, self, i) + for idx in range(capi.c_num_methods(self.space, self)): if capi.c_is_constructor(self.space, self, idx): pyname = '__init__' else: @@ -1029,7 +1043,7 @@ W_CPPComplexClassDecl.typedef.acceptable_as_base_class = False -class W_CPPClass(W_Root): +class W_CPPInstance(W_Root): _attrs_ = ['space', 'clsdecl', '_rawobject', 'flags', 'finalizer_registered'] _immutable_fields_ = ['clsdecl'] @@ -1109,8 +1123,8 @@ # find a global overload in gbl, in __gnu_cxx (for iterators), or in the # scopes of the argument classes (TODO: implement that last option) try: - # TODO: expecting w_other to be an W_CPPClass is too limiting - other = self.space.interp_w(W_CPPClass, w_other, can_be_None=False) + # TODO: expecting w_other to be an W_CPPInstance is too limiting + other = self.space.interp_w(W_CPPInstance, w_other, can_be_None=False) for name in ["", "__gnu_cxx", "__1"]: nss = scope_byname(self.space, name) meth_idx = capi.c_get_global_operator( @@ -1132,7 +1146,7 @@ # fallback 2: direct pointer comparison (the class comparison is needed since # the first data member in a struct and the struct have the same address) - other = self.space.interp_w(W_CPPClass, w_other, can_be_None=False) # TODO: factor out + other = self.space.interp_w(W_CPPInstance, w_other, can_be_None=False) # TODO: factor out iseq = (self._rawobject == other._rawobject) and (self.clsdecl == other.clsdecl) return self.space.newbool(iseq) @@ -1176,19 +1190,19 @@ if self.flags & INSTANCE_FLAGS_PYTHON_OWNS: self.destruct() -W_CPPClass.typedef = TypeDef( - 'CPPClass', - __python_owns__ = GetSetProperty(W_CPPClass.fget_python_owns, W_CPPClass.fset_python_owns), - __init__ = interp2app(W_CPPClass.instance__init__), - __eq__ = interp2app(W_CPPClass.instance__eq__), - __ne__ = interp2app(W_CPPClass.instance__ne__), - __nonzero__ = interp2app(W_CPPClass.instance__nonzero__), - __len__ = interp2app(W_CPPClass.instance__len__), - __cmp__ = interp2app(W_CPPClass.instance__cmp__), - __repr__ = interp2app(W_CPPClass.instance__repr__), - __destruct__ = interp2app(W_CPPClass.destruct), +W_CPPInstance.typedef = TypeDef( + 'CPPInstance', + __python_owns__ = GetSetProperty(W_CPPInstance.fget_python_owns, W_CPPInstance.fset_python_owns), + __init__ = interp2app(W_CPPInstance.instance__init__), + __eq__ = interp2app(W_CPPInstance.instance__eq__), + __ne__ = interp2app(W_CPPInstance.instance__ne__), + __nonzero__ = interp2app(W_CPPInstance.instance__nonzero__), + __len__ = interp2app(W_CPPInstance.instance__len__), + __cmp__ = interp2app(W_CPPInstance.instance__cmp__), + __repr__ = interp2app(W_CPPInstance.instance__repr__), + __destruct__ = interp2app(W_CPPInstance.destruct), ) -W_CPPClass.typedef.acceptable_as_base_class = True +W_CPPInstance.typedef.acceptable_as_base_class = True class MemoryRegulator: @@ -1200,7 +1214,7 @@ # Note that for now, the associated test carries an m_padding to make # a difference in the addresses. def __init__(self): - self.objects = rweakref.RWeakValueDictionary(int, W_CPPClass) + self.objects = rweakref.RWeakValueDictionary(int, W_CPPInstance) def register(self, obj): if not obj._rawobject: @@ -1266,8 +1280,8 @@ return obj # fresh creation - w_cppinstance = space.allocate_instance(W_CPPClass, w_pycppclass) - cppinstance = space.interp_w(W_CPPClass, w_cppinstance, can_be_None=False) + w_cppinstance = space.allocate_instance(W_CPPInstance, w_pycppclass) + cppinstance = space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=False) cppinstance.__init__(space, clsdecl, rawobject, is_ref, python_owns) memory_regulator.register(cppinstance) return w_cppinstance @@ -1311,7 +1325,7 @@ def move(space, w_obj): """Casts the given instance into an C++-style rvalue.""" - obj = space.interp_w(W_CPPClass, w_obj, can_be_None=True) + obj = space.interp_w(W_CPPInstance, w_obj, can_be_None=True) if obj: obj.flags |= INSTANCE_FLAGS_IS_R_VALUE return w_obj diff --git a/pypy/module/_cppyy/pythonify.py b/pypy/module/_cppyy/pythonify.py --- a/pypy/module/_cppyy/pythonify.py +++ b/pypy/module/_cppyy/pythonify.py @@ -7,7 +7,7 @@ # Metaclasses are needed to store C++ static data members as properties. Since # the interp-level does not support metaclasses, they are created at app-level. # These are the metaclass base classes: -class CPPMetaScope(type): +class CPPScope(type): def __getattr__(self, name): try: return get_scoped_pycppitem(self, name) # will cache on self @@ -15,11 +15,11 @@ raise AttributeError("%s object has no attribute '%s' (details: %s)" % (self, name, str(e))) -class CPPMetaNamespace(CPPMetaScope): +class CPPMetaNamespace(CPPScope): def __dir__(self): return self.__cppdecl__.__dir__() -class CPPMetaClass(CPPMetaScope): +class CPPClass(CPPScope): pass # namespace base class (class base class defined in _init_pythonify) @@ -173,7 +173,7 @@ # get a list of base classes for class creation bases = [get_pycppclass(base) for base in decl.get_base_names()] if not bases: - bases = [CPPClass,] + bases = [CPPInstance,] else: # it's possible that the required class now has been built if one of # the base classes uses it in e.g. a function interface @@ -214,7 +214,7 @@ # create a metaclass to allow properties (for static data write access) metabases = [type(base) for base in bases] - metacpp = type(CPPMetaScope)(cl_name+'_meta', _drop_cycles(metabases), d_meta) + metacpp = type(CPPScope)(cl_name+'_meta', _drop_cycles(metabases), d_meta) # create the python-side C++ class pycls = metacpp(cl_name, _drop_cycles(bases), d_class) @@ -412,11 +412,11 @@ # at pypy-c startup, rather than on the "import _cppyy" statement import _cppyy - # root of all proxy classes: CPPClass in pythonify exists to combine the - # CPPMetaScope metaclass with the interp-level CPPClassBase - global CPPClass - class CPPClass(_cppyy.CPPClassBase): - __metaclass__ = CPPMetaScope + # root of all proxy classes: CPPInstance in pythonify exists to combine + # the CPPScope metaclass with the interp-level CPPInstanceBase + global CPPInstance + class CPPInstance(_cppyy.CPPInstanceBase): + __metaclass__ = CPPScope pass # class generator callback @@ -438,9 +438,8 @@ gbl.std.move = _cppyy.move # install a type for enums to refer to - # TODO: this is correct for C++98, not for C++11 and in general there will - # be the same issue for all typedef'd builtin types setattr(gbl, 'internal_enum_type_t', int) + setattr(gbl, 'unsigned int', int) # if resolved # install for user access _cppyy.gbl = gbl diff --git a/pypy/module/_cppyy/src/dummy_backend.cxx b/pypy/module/_cppyy/src/dummy_backend.cxx --- a/pypy/module/_cppyy/src/dummy_backend.cxx +++ b/pypy/module/_cppyy/src/dummy_backend.cxx @@ -407,11 +407,9 @@ /* name to opaque C++ scope representation -------------------------------- */ -int cppyy_num_scopes(cppyy_scope_t handle) { - return 0; -} - char* cppyy_resolve_name(const char* cppitem_name) { + if (cppyy_is_enum(cppitem_name)) + return cppstring_to_cstring("internal_enum_type_t"); return cppstring_to_cstring(cppitem_name); } @@ -851,10 +849,13 @@ return (cppyy_object_t)result; } -cppyy_funcaddr_t cppyy_get_function_address(cppyy_scope_t /* scope */, cppyy_index_t /* idx */) { +cppyy_funcaddr_t cppyy_function_address_from_index(cppyy_scope_t /* scope */, cppyy_index_t /* idx */) { return (cppyy_funcaddr_t)0; } +cppyy_funcaddr_t cppyy_function_address_from_method(cppyy_method_t /* method */) { + return (cppyy_funcaddr_t)0; +} /* handling of function argument buffer ----------------------------------- */ void* cppyy_allocate_function_args(int nargs) { @@ -926,10 +927,6 @@ return s_scopes[handle].m_methods.size(); } -cppyy_index_t cppyy_method_index_at(cppyy_scope_t /* scope */, int imeth) { - return (cppyy_index_t)imeth; -} - char* cppyy_method_name(cppyy_scope_t handle, cppyy_index_t method_index) { return cppstring_to_cstring(s_scopes[handle].m_methods[(int)method_index].m_name); } @@ -978,8 +975,17 @@ return (cppyy_method_t)0; } +cppyy_index_t cppyy_get_global_operator(cppyy_scope_t /* scope */, + cppyy_scope_t /* lc */, cppyy_scope_t /* rc */, const char* /* op */) { + return (cppyy_index_t)-1; +} + /* method properties ----------------------------------------------------- */ +int cppyy_is_publicmethod(cppyy_type_t /* handle */, cppyy_index_t /* method_index */) { + return 1; +} + int cppyy_is_constructor(cppyy_type_t handle, cppyy_index_t method_index) { if (s_scopes.find(handle) != s_scopes.end()) return s_scopes[handle].m_methods[method_index].m_type == kConstructor; @@ -987,6 +993,10 @@ return 0; } +int cppyy_is_destructor(cppyy_type_t /* handle */, cppyy_index_t /* method_index */) { + return 0; +} + int cppyy_is_staticmethod(cppyy_type_t handle, cppyy_index_t method_index) { if (s_scopes.find(handle) != s_scopes.end()) return s_scopes[handle].m_methods[method_index].m_type == kStatic; @@ -1014,14 +1024,22 @@ /* data member properties ------------------------------------------------ */ -int cppyy_is_publicdata(cppyy_scope_t handle, int idatambr) { +int cppyy_is_publicdata(cppyy_scope_t /* handle */, cppyy_index_t /* idatambr */) { return 1; } -int cppyy_is_staticdata(cppyy_scope_t handle, int idatambr) { +int cppyy_is_staticdata(cppyy_scope_t handle, cppyy_index_t idatambr) { return s_scopes[handle].m_datambrs[idatambr].m_isstatic; } +int cppyy_is_const_data(cppyy_scope_t /* handle */, cppyy_index_t /* idatambr */) { + return 0; +} + +int cppyy_is_enum_data(cppyy_scope_t /* handle */, cppyy_index_t /* idatambr */) { + return 0; +} + /* misc helpers ----------------------------------------------------------- */ #if defined(_MSC_VER) diff --git a/pypy/module/_cppyy/test/Makefile b/pypy/module/_cppyy/test/Makefile --- a/pypy/module/_cppyy/test/Makefile +++ b/pypy/module/_cppyy/test/Makefile @@ -14,7 +14,7 @@ HASGENREFLEX:=$(shell command -v genreflex 2> /dev/null) -cppflags=-std=c++11 -O3 -m64 -fPIC -rdynamic +cppflags=-std=c++14 -O3 -m64 -fPIC -rdynamic ifdef HASGENREFLEX genreflex_flags:=$(shell genreflex --cppflags) cppflags+=$(genreflex_flags) diff --git a/pypy/module/_cppyy/test/advancedcpp.cxx b/pypy/module/_cppyy/test/advancedcpp.cxx --- a/pypy/module/_cppyy/test/advancedcpp.cxx +++ b/pypy/module/_cppyy/test/advancedcpp.cxx @@ -1,5 +1,7 @@ #include "advancedcpp.h" +#include + // for testing of default arguments #define IMPLEMENT_DEFAULTER_CLASS(type, tname) \ @@ -112,3 +114,13 @@ std::string overload_the_other_way::gime() { return "aap"; } int overload_the_other_way::gime() const { return 1; } + + +// exception handling testing +void Thrower::throw_anything() { + throw 1; +} + +void Thrower::throw_exception() { + throw std::runtime_error("C++ function failed"); +} diff --git a/pypy/module/_cppyy/test/advancedcpp.h b/pypy/module/_cppyy/test/advancedcpp.h --- a/pypy/module/_cppyy/test/advancedcpp.h +++ b/pypy/module/_cppyy/test/advancedcpp.h @@ -59,7 +59,7 @@ class a_class { // for esoteric inheritance testing public: a_class() { m_a = 1; m_da = 1.1; } - ~a_class() {} + virtual ~a_class() {} virtual int get_value() = 0; public: @@ -221,6 +221,7 @@ //=========================================================================== class some_abstract_class { // to test abstract class handling public: + virtual ~some_abstract_class() {} virtual void a_virtual_method() = 0; }; @@ -399,3 +400,11 @@ std::string gime(); int gime() const; }; + + +//=========================================================================== +class Thrower { // exception handling testing +public: + void throw_anything(); + void throw_exception(); +}; diff --git a/pypy/module/_cppyy/test/advancedcpp.xml b/pypy/module/_cppyy/test/advancedcpp.xml --- a/pypy/module/_cppyy/test/advancedcpp.xml +++ b/pypy/module/_cppyy/test/advancedcpp.xml @@ -57,4 +57,6 @@ + + diff --git a/pypy/module/_cppyy/test/conftest.py b/pypy/module/_cppyy/test/conftest.py --- a/pypy/module/_cppyy/test/conftest.py +++ b/pypy/module/_cppyy/test/conftest.py @@ -55,7 +55,7 @@ separate_module_files=[srcpath.join('dummy_backend.cxx')], include_dirs=[incpath, tstpath, cdir], compile_extra=['-DRPY_EXTERN=RPY_EXPORTED', '-DCPPYY_DUMMY_BACKEND', - '-fno-strict-aliasing', '-std=c++11'], + '-fno-strict-aliasing', '-std=c++14'], use_cpp_linker=True, ) @@ -65,7 +65,7 @@ outputfilename='libcppyy_dummy_backend', standalone=False) except CompilationError as e: - if '-std=c++11' in str(e): + if '-std=c++14' in str(e): global disabled disabled = str(e) return diff --git a/pypy/module/_cppyy/test/fragile.h b/pypy/module/_cppyy/test/fragile.h --- a/pypy/module/_cppyy/test/fragile.h +++ b/pypy/module/_cppyy/test/fragile.h @@ -30,6 +30,7 @@ void overload(int, no_such_class* p = 0) {} }; + static const int dummy_location = 0xdead; class E { @@ -105,6 +106,7 @@ class M { public: + virtual ~M() {} enum E1 { kOnce=42 }; enum E2 { kTwice=12 }; }; diff --git a/pypy/module/_cppyy/test/test_advancedcpp.py b/pypy/module/_cppyy/test/test_advancedcpp.py --- a/pypy/module/_cppyy/test/test_advancedcpp.py +++ b/pypy/module/_cppyy/test/test_advancedcpp.py @@ -656,3 +656,22 @@ # TODO: currently fails b/c double** not understood as &double* #assert cppyy.gbl.my_global_ptr[0] == 1234. + def test22_exceptions(self): + """Catching of C++ exceptions""" + + import _cppyy as cppyy + Thrower = cppyy.gbl.Thrower + + # TODO: clean up this interface: + Thrower.__cppdecl__.get_overload('throw_anything').__useffi__ = False + Thrower.__cppdecl__.get_overload('throw_exception').__useffi__ = False + + t = Thrower() + + assert raises(Exception, t.throw_anything) + assert raises(Exception, t.throw_exception) + + try: + t.throw_exception() + except Exception, e: + "C++ function failed" in str(e) diff --git a/pypy/module/_cppyy/test/test_overloads.py b/pypy/module/_cppyy/test/test_overloads.py --- a/pypy/module/_cppyy/test/test_overloads.py +++ b/pypy/module/_cppyy/test/test_overloads.py @@ -15,7 +15,6 @@ spaceconfig = dict(usemodules=['_cppyy', '_rawffi', 'itertools']) def setup_class(cls): - env = os.environ cls.w_test_dct = cls.space.newtext(test_dct) cls.w_overloads = cls.space.appexec([], """(): import ctypes diff --git a/pypy/module/_cppyy/test/test_zjit.py b/pypy/module/_cppyy/test/test_zjit.py --- a/pypy/module/_cppyy/test/test_zjit.py +++ b/pypy/module/_cppyy/test/test_zjit.py @@ -137,6 +137,7 @@ executor.get_executor(self, 'int').__class__.c_stubcall = staticmethod(c_call_i) self.w_AttributeError = FakeException(self, "AttributeError") + self.w_Exception = FakeException(self, "Exception") self.w_KeyError = FakeException(self, "KeyError") self.w_NotImplementedError = FakeException(self, "NotImplementedError") self.w_ReferenceError = FakeException(self, "ReferenceError") @@ -282,7 +283,7 @@ inst = interp_cppyy._bind_object(space, FakeInt(0), cls, True) cls.get_overload("__init__").call(inst, [FakeInt(0)]) cppmethod = cls.get_overload(method_name) - assert isinstance(inst, interp_cppyy.W_CPPClass) + assert isinstance(inst, interp_cppyy.W_CPPInstance) i = 10 while i > 0: drv.jit_merge_point(inst=inst, cppmethod=cppmethod, i=i) diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py --- a/pypy/module/cpyext/cdatetime.py +++ b/pypy/module/cpyext/cdatetime.py @@ -124,7 +124,7 @@ # app level datetime.date. If a c-extension class uses datetime.date for its # base class and defines a tp_dealloc, we will get this: # c_class->tp_dealloc == tp_dealloc_func - # c_class->tp_base == datetime.date, + # c_class->tp_base == datetime.date, # datetime.date->tp_dealloc = _PyPy_subtype_dealloc # datetime.date->tp_base = W_DateTime_Date # W_DateTime_Date->tp_dealloc = _PyPy_subtype_dealloc diff --git a/pypy/module/cpyext/test/test_datetime.py b/pypy/module/cpyext/test/test_datetime.py --- a/pypy/module/cpyext/test/test_datetime.py +++ b/pypy/module/cpyext/test/test_datetime.py @@ -1,3 +1,5 @@ +import pytest + from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.cdatetime import * @@ -82,6 +84,14 @@ date = datetime.datetime.fromtimestamp(0) assert space.unwrap(space.str(w_date)) == str(date) + @pytest.mark.parametrize('name', ['Time', 'DateTime', 'Date', 'Delta']) + def test_basicsize(self, space, name): + datetime = _PyDateTime_Import(space) + py_size = getattr(datetime, "c_%sType" % name).c_tp_basicsize + c_size = rffi.sizeof(cts.gettype("PyDateTime_%s" % name)) + assert py_size == c_size + + class AppTestDatetime(AppTestCpythonExtensionBase): def test_CAPI(self): module = self.import_extension('foo', [ @@ -271,9 +281,9 @@ 6, 6, 6, 6, args, PyDateTimeAPI->TimeType); """), ("datetime_with_tzinfo", "METH_O", - """ + """ PyObject * obj; - int tzrefcnt = args->ob_refcnt; + int tzrefcnt = args->ob_refcnt; PyDateTime_IMPORT; obj = PyDateTimeAPI->DateTime_FromDateAndTime( 2000, 6, 6, 6, 6, 6, 6, args, @@ -291,7 +301,7 @@ return NULL; } return obj; - + """), ], prologue='#include "datetime.h"\n') from datetime import tzinfo, datetime, timedelta, time @@ -339,4 +349,4 @@ assert module.checks(o) == (True,) * 3 + (False,) * 7 # isinstance(datetime, date) o = tzinfo() assert module.checks(o) == (False,) * 8 + (True,) * 2 - + diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py --- a/pypy/objspace/std/listobject.py +++ b/pypy/objspace/std/listobject.py @@ -968,6 +968,14 @@ def setslice(self, w_list, start, step, slicelength, w_other): strategy = w_other.strategy + if step != 1: + len2 = strategy.length(w_other) + if len2 == 0: + return + else: + raise oefmt(self.space.w_ValueError, + "attempt to assign sequence of size %d to extended " + "slice of size %d", len2, 0) storage = strategy.getstorage_copy(w_other) w_list.strategy = strategy w_list.lstorage = storage diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py --- a/pypy/objspace/std/test/test_listobject.py +++ b/pypy/objspace/std/test/test_listobject.py @@ -1078,6 +1078,15 @@ l[::3] = ('a', 'b') assert l == ['a', 1.1, 2.2, 'b', 4.4, 5.5] + l_int = [5]; l_int.pop() # IntListStrategy + l_empty = [] # EmptyListStrategy + raises(ValueError, "l_int[::-1] = [42]") + raises(ValueError, "l_int[::7] = [42]") + raises(ValueError, "l_empty[::-1] = [42]") + raises(ValueError, "l_empty[::7] = [42]") + l_int[::1] = [42]; assert l_int == [42] + l_empty[::1] = [42]; assert l_empty == [42] + def test_setslice_with_self(self): l = [1,2,3,4] l[:] = l From pypy.commits at gmail.com Tue Apr 24 02:07:43 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 23 Apr 2018 23:07:43 -0700 (PDT) Subject: [pypy-commit] pypy default: Added tag release-pypy2.7-v6.0.0 for changeset ab0b9caf307d Message-ID: <5adec9af.41711c0a.65e68.0439@mx.google.com> Author: Matti Picus Branch: Changeset: r94441:e1aef8bbd9e7 Date: 2018-04-24 09:06 +0300 http://bitbucket.org/pypy/pypy/changeset/e1aef8bbd9e7/ Log: Added tag release-pypy2.7-v6.0.0 for changeset ab0b9caf307d diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -51,3 +51,4 @@ 0000000000000000000000000000000000000000 release-pypy3.5-v5.10.0 09f9160b643e3f02ccb8c843b2fbb4e5cbf54082 release-pypy3.5-v5.10.0 3f6eaa010fce78cc7973bdc1dfdb95970f08fed2 release-pypy3.5-v5.10.1 +ab0b9caf307db6592905a80b8faffd69b39005b8 release-pypy2.7-v6.0.0 From pypy.commits at gmail.com Tue Apr 24 02:07:45 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 23 Apr 2018 23:07:45 -0700 (PDT) Subject: [pypy-commit] pypy default: Added tag release-pypy3.5-v6.0.0 for changeset fdd60ed87e94 Message-ID: <5adec9b1.1c69fb81.1d53.f5c6@mx.google.com> Author: Matti Picus Branch: Changeset: r94442:9303f16ac3b2 Date: 2018-04-24 09:06 +0300 http://bitbucket.org/pypy/pypy/changeset/9303f16ac3b2/ Log: Added tag release-pypy3.5-v6.0.0 for changeset fdd60ed87e94 diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -52,3 +52,4 @@ 09f9160b643e3f02ccb8c843b2fbb4e5cbf54082 release-pypy3.5-v5.10.0 3f6eaa010fce78cc7973bdc1dfdb95970f08fed2 release-pypy3.5-v5.10.1 ab0b9caf307db6592905a80b8faffd69b39005b8 release-pypy2.7-v6.0.0 +fdd60ed87e941677e8ea11acf9f1819466521bf2 release-pypy3.5-v6.0.0 From pypy.commits at gmail.com Tue Apr 24 03:54:23 2018 From: pypy.commits at gmail.com (arigo) Date: Tue, 24 Apr 2018 00:54:23 -0700 (PDT) Subject: [pypy-commit] cffi default: Merged in jdufresne/cffi (pull request #86) Message-ID: <5adee2af.d3371c0a.30c66.e2e4@mx.google.com> Author: Armin Rigo Branch: Changeset: r3119:0eac9cf29182 Date: 2018-04-24 07:54 +0000 http://bitbucket.org/cffi/cffi/changeset/0eac9cf29182/ Log: Merged in jdufresne/cffi (pull request #86) Include license file in the generated wheel package diff --git a/setup.cfg b/setup.cfg new file mode 100644 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[metadata] +license_file = LICENSE From pypy.commits at gmail.com Tue Apr 24 03:54:26 2018 From: pypy.commits at gmail.com (jdufresne) Date: Tue, 24 Apr 2018 00:54:26 -0700 (PDT) Subject: [pypy-commit] cffi default: Include license file in the generated wheel package Message-ID: <5adee2b2.8d6d1c0a.bcf2b.2340@mx.google.com> Author: Jon Dufresne Branch: Changeset: r3118:b93a527ad3b1 Date: 2018-04-10 20:22 -0700 http://bitbucket.org/cffi/cffi/changeset/b93a527ad3b1/ Log: Include license file in the generated wheel package The wheel package format supports including the license file. This is done using the [metadata] section in the setup.cfg file. For additional information on this feature, see: https://wheel.readthedocs.io/en/stable/index.html#including-the- license-in-the-generated-wheel-file Helps project comply with its own license: > The above copyright notice and this permission notice shall be included > in all copies or substantial portions of the Software. diff --git a/setup.cfg b/setup.cfg new file mode 100644 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[metadata] +license_file = LICENSE From pypy.commits at gmail.com Tue Apr 24 04:00:42 2018 From: pypy.commits at gmail.com (arigo) Date: Tue, 24 Apr 2018 01:00:42 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Fix _PyLong_Sign() to accept any app-level 'int' object Message-ID: <5adee42a.01ed1c0a.10b55.8133@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r94443:a07f07034d28 Date: 2018-04-24 10:00 +0200 http://bitbucket.org/pypy/pypy/changeset/a07f07034d28/ Log: Fix _PyLong_Sign() to accept any app-level 'int' object diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py --- a/pypy/module/cpyext/longobject.py +++ b/pypy/module/cpyext/longobject.py @@ -2,7 +2,6 @@ from pypy.module.cpyext.api import ( cpython_api, PyObject, build_type_checkers_flags, Py_ssize_t, CONST_STRING, ADDR, CANNOT_FAIL) -from pypy.objspace.std.longobject import W_LongObject from pypy.interpreter.error import OperationError, oefmt from rpython.rlib.rbigint import rbigint, InvalidSignednessError @@ -234,8 +233,8 @@ @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def _PyLong_Sign(space, w_long): - assert isinstance(w_long, W_LongObject) - return w_long.num.sign + bigint = space.bigint_w(w_long) + return bigint.sign CONST_UCHARP = lltype.Ptr(lltype.Array(rffi.UCHAR, hints={'nolength': True, 'render_as_const': True})) diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test/test_longobject.py --- a/pypy/module/cpyext/test/test_longobject.py +++ b/pypy/module/cpyext/test/test_longobject.py @@ -136,6 +136,9 @@ assert api._PyLong_Sign(space.wraplong(0L)) == 0 assert api._PyLong_Sign(space.wraplong(2L)) == 1 assert api._PyLong_Sign(space.wraplong(-2L)) == -1 + assert api._PyLong_Sign(space.wrap(0)) == 0 + assert api._PyLong_Sign(space.wrap(42)) == 1 + assert api._PyLong_Sign(space.wrap(-42)) == -1 assert api._PyLong_NumBits(space.wrap(0)) == 0 assert api._PyLong_NumBits(space.wrap(1)) == 1 From pypy.commits at gmail.com Tue Apr 24 14:01:45 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 24 Apr 2018 11:01:45 -0700 (PDT) Subject: [pypy-commit] pypy default: mention antocuni wheels and --no-build-isolation in release notes Message-ID: <5adf7109.1c69fb81.950f5.d123@mx.google.com> Author: Matti Picus Branch: Changeset: r94444:ad394955cf52 Date: 2018-04-24 21:00 +0300 http://bitbucket.org/pypy/pypy/changeset/ad394955cf52/ Log: mention antocuni wheels and --no-build-isolation in release notes diff --git a/pypy/doc/release-v6.0.0.rst b/pypy/doc/release-v6.0.0.rst --- a/pypy/doc/release-v6.0.0.rst +++ b/pypy/doc/release-v6.0.0.rst @@ -14,6 +14,11 @@ affect the included python development header files, all c-extension modules must be recompiled for this version. +Until we can work with downstream providers to distruibute builds with PyPy, we +have made packages for some common packages `available as wheels`_. You may +compile yourself using ``pip install --no-build-isolation ``, the +``no-build-isolation`` is currently needed for pip v10. + First-time python users are often stumped by silly typos and emissions when getting started writing code. We have improved our parser to emit more friendly `syntax errors`_, making PyPy not only faster but more friendly. @@ -60,6 +65,7 @@ .. _`hooks`: gc_info.html#gc-hooks .. _`cffi`: http://cffi.readthedocs.io .. _`cppyy`: https://cppyy.readthedocs.io +.. _`available as wheels`: https://github.com/antocuni/pypy-wheels What is PyPy? ============= From pypy.commits at gmail.com Tue Apr 24 14:03:58 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 24 Apr 2018 11:03:58 -0700 (PDT) Subject: [pypy-commit] pypy.org extradoc: update for release, do not regenerate yet Message-ID: <5adf718e.0c6c1c0a.a9991.dfff@mx.google.com> Author: Matti Picus Branch: extradoc Changeset: r923:bec9ffcc234e Date: 2018-04-24 21:03 +0300 http://bitbucket.org/pypy/pypy.org/changeset/bec9ffcc234e/ Log: update for release, do not regenerate yet diff --git a/source/download.txt b/source/download.txt --- a/source/download.txt +++ b/source/download.txt @@ -15,14 +15,14 @@ We provide binaries for x86, ARM, PPC and s390x running on different operating systems such as Linux, Mac OS X and Windows: -* the Python2.7 compatible release — **PyPy2.7 v5.10.0** — (`what's new in PyPy2.7?`_) +* the Python2.7 compatible release — **PyPy2.7 v6.0** — (`what's new in PyPy2.7?`_) -* the Python3.5 compatible release — **PyPy3.5 v5.10.1** — (`what's new in PyPy3.5?`_). +* the Python3.5 compatible release — **PyPy3.5 v6.0** — (`what's new in PyPy3.5?`_). * the Python2.7 Software Transactional Memory special release — **PyPy-STM 2.5.1** (Linux x86-64 only) -.. _what's new in PyPy2.7?: http://doc.pypy.org/en/latest/release-v5.10.0.html -.. _what's new in PyPy3.5?: http://doc.pypy.org/en/latest/release-v5.10.1.html +.. _what's new in PyPy2.7?: http://doc.pypy.org/en/latest/release-v6.0.0.html +.. _what's new in PyPy3.5?: http://doc.pypy.org/en/latest/release-v6.0.1.html .. class:: download_menu @@ -79,14 +79,13 @@ .. _release: -Python2.7 compatible PyPy 5.10.0 --------------------------------- +Python2.7 compatible PyPy 6.0.0 +------------------------------- * `Linux x86 binary (32bit, tar.bz2 built on Ubuntu 12.04 - 16.04)`__ (see ``[1]`` below) * `Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 16.04)`__ (see ``[1]`` below) * `ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Raspbian)`__ (see ``[1]`` below) * `ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Ubuntu Raring)`__ (see ``[1]`` below) -* `ARM Softfloat Linux binary (ARMEL/gnueabi, tar.bz2, Ubuntu Precise)`__ (see ``[1]`` below) * `Mac OS X binary (64bit)`__ * FreeBSD x86 and x86_64: see FreshPorts_ * `Windows binary (32bit)`__ (you might need the VS 2008 runtime library @@ -98,24 +97,23 @@ * `All our downloads,`__ including previous versions. We also have a mirror_, but please use only if you have troubles accessing the links above -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.10.0-linux32.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.10.0-linux64.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.10.0-linux-armhf-raspbian.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.10.0-linux-armhf-raring.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.10.0-linux-armel.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.10.0-osx64.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.10.0-win32.zip +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v6.0.0-linux32.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v6.0.0-linux64.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v6.0.0-linux-armhf-raspbian.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v6.0.0-linux-armel.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v6.0.0-osx64.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v6.0.0-win32.zip .. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.10.0-ppc64.tar.bz2 .. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.10.0-ppc64le.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.10.0-s390x.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.10.0-src.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.10.0-src.zip +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v6.0.0-s390x.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v6.0.0-src.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v6.0.0-src.zip .. _`vcredist_x86.exe`: http://www.microsoft.com/en-us/download/details.aspx?id=5582 .. __: https://bitbucket.org/pypy/pypy/downloads .. _mirror: http://buildbot.pypy.org/mirror/ .. _FreshPorts: http://www.freshports.org/lang/pypy -Python 3.5.3 compatible PyPy3.5 v5.10.1 +Python 3.5.3 compatible PyPy3.5 v6.0.1 --------------------------------------- .. class:: download_menu @@ -133,16 +131,16 @@ * `All our downloads,`__ including previous versions. We also have a mirror_, but please use only if you have troubles accessing the links above -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.10.1-linux32.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.10.1-linux64.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.10.1-linux-armhf-raspbian.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.10.1-linux-armel.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.10.1-osx64.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v6.0.0-linux32.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v6.0.0-linux64.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v6.0.0-linux-armhf-raspbian.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v6.0.0-linux-armel.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v6.0.0-osx64.tar.bz2 .. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.10.0-osx64-2.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.10.1-win32.zip -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.10.1-s390x.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.10.1-src.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.10.1-src.zip +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v6.0.0-win32.zip +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v6.0.0-s390x.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v6.0.0-src.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v6.0.0-src.zip .. __: https://bitbucket.org/pypy/pypy/downloads If your CPU is really, really old, it may be a x86-32 without SSE2. @@ -208,7 +206,7 @@ uncompressed, they run in-place. For now you can uncompress them either somewhere in your home directory or, say, in ``/opt``, and if you want, put a symlink from somewhere like -``/usr/local/bin/pypy`` to ``/path/to/pypy2-5.10.0/bin/pypy``. Do +``/usr/local/bin/pypy`` to ``/path/to/pypy_expanded/bin/pypy``. Do not move or copy the executable ``pypy`` outside the tree --- put a symlink to it, otherwise it will not find its libraries. @@ -309,11 +307,11 @@ Alternatively, get one of the following smaller packages for the source at the same revision as the above binaries: - * `pypy2-v5.10.0-src.tar.bz2`__ (sources, PyPy 2 only) - * `pypy3-v5.10.1-src.tar.bz2`__ (sources, PyPy 3 only) + * `pypy2-v6.0.1-src.tar.bz2`__ (sources, PyPy 2 only) + * `pypy3-v6.0.1-src.tar.bz2`__ (sources, PyPy 3 only) - .. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.10.0-src.tar.bz2 - .. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.10.1-src.tar.bz2 + .. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v6.0.0-src.tar.bz2 + .. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v6.0.0-src.tar.bz2 2. Make sure you **installed the dependencies.** See the list here__. @@ -472,17 +470,7 @@ pypy 3.5-v5.10.0 sha256:: - 529bc3b11edbdcdd676d90c805b8f607f6eedd5f0ec457a31bbe09c03f5bebfe pypy3-v5.10.0-linux32.tar.bz2 - aa4fb52fb858d973dd838dcf8d74f30705e5afdf1150acb8e056eb99353dfe77 pypy3-v5.10.0-linux64.tar.bz2 - c2cc529befb3e1f2ef8bd4e96af4a823c52ef2d180b0b3bd87511c5b47d59210 pypy3-v5.10.0-linux-armel.tar.bz2 - 4e902e0e79f62f2a9049c1c71310ff4fc801011bec4d25082edb5c537d3f15c9 pypy3-v5.10.0-linux-armhf-raspbian.tar.bz2 - 7e389a103f560de1eead1271ec3a2df9424c6ccffe7cbae8e95e6e81ae811a16 pypy3-v5.10.0-osx64.tar.bz2 f5ced20934fff78e55c72aa82a4703954349a5a8099b94e77d74b96a94326a2c pypy3-v5.10.0-osx64-2.tar.bz2 - e0ffec9d033002eb61af488b1f66c319380da8408abd14a3bc202ded4705dc9a pypy3-v5.10.0-s390x.tar.bz2 - a6e4cffde71e3f08b6e1befa5c0352a9bcc5f4e9f5cbf395001e0763a1a0d9e3 pypy3-v5.10.0-src.tar.bz2 - 96cf354fb410599cd5acd21732855e25e742e13eac7dc079c0c02b0625908cb9 pypy3-v5.10.0-src.zip - 2d93bf2bd7b1d031b96331d3fde6cacdda95673ce6875d6d1669c4c0ea2a52bc pypy3-v5.10.0-win32.zip - pypy 3.5-v5.10.1 sha256:: @@ -495,3 +483,16 @@ 182378d7aab395ee6cf539fb011ec0e384624282834aaaed4a663972a5aa8797 pypy3-v5.10.1-src.zip 4edf4f021689a529e5a631c5cca72a1a9dc19a6ea2091e64289cdd5b60eaf929 pypy3-v5.10.1-win32.zip 9ce98481cddede40a3357f7462f2c894bb96f178e2e8715d04feda1476ec1563 pypy3-v5.10.1-s390x.tar.bz2 + +pypy 3.5-v6.0.0 sha256:: + + b04eeee5160e6cb5f8962de80f077ea1dc7be34e77d74bf075519c23603f5ff9 pypy3-v6.0.0-linux32.tar.bz2 + 4cfffa292b9ef34bb6ba39cdbaa196c5c5cbbc5aa3faaa157cf45d7e34027048 pypy3-v6.0.0-linux64.tar.bz2 + 6a6888a55192f58594838b8b3d2e7daaad43d3bf4293afab3dd8987d0bbd1124 pypy3-v6.0.0-linux-armel.tar.bz2 + 8a0420dda23413925400538bbfc0cff2bbb2ab0de984eef6faaeab6d3309cbcc pypy3-v6.0.0-linux-armhf-raspbian.tar.bz2 + 938b8034e30f5f5060d2a079070c56c3be5559bc7ae9cc0c8395fe6fc45cfe4c pypy3-v6.0.0-osx64.tar.bz2 + f0a3097cf74d6b1fb3ae2a060384e72b7868ca76cd04584336b417e9982ec0a5 pypy3-v6.0.0-s390x.tar.bz2 + ed8005202b46d6fc6831df1d13a4613bc40084bfa42f275068edadf8954034a3 pypy3-v6.0.0-src.tar.bz2 + 8cd3cc2ef362e774edf9c7a6b79dbe42fff75217c5ed96b235a0a792e4421dc4 pypy3-v6.0.0-src.zip + 72dddb3746a51f7672c77d619c818e27efe899e08ae82762448e50dbfdc2f5f3 pypy3-v6.0.0-win32.zip + From pypy.commits at gmail.com Wed Apr 25 08:57:43 2018 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 25 Apr 2018 05:57:43 -0700 (PDT) Subject: [pypy-commit] pypy default: fix typos (thanks lesshaste and __pv) Message-ID: <5ae07b47.1c69fb81.1d53.43b9@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r94445:a614238235c7 Date: 2018-04-25 14:57 +0200 http://bitbucket.org/pypy/pypy/changeset/a614238235c7/ Log: fix typos (thanks lesshaste and __pv) diff --git a/pypy/doc/release-v6.0.0.rst b/pypy/doc/release-v6.0.0.rst --- a/pypy/doc/release-v6.0.0.rst +++ b/pypy/doc/release-v6.0.0.rst @@ -8,18 +8,18 @@ the dual release. This release is a feature release following our previous 5.10 incremental -release in late December 2017. Our C-API compatability layer ``cpyext`` is +release in late December 2017. Our C-API compatibility layer ``cpyext`` is now much faster (see the `blog post`_) as well as more complete. We have made many other improvements in speed and CPython compatibility. Since the changes affect the included python development header files, all c-extension modules must be recompiled for this version. -Until we can work with downstream providers to distruibute builds with PyPy, we +Until we can work with downstream providers to distribute builds with PyPy, we have made packages for some common packages `available as wheels`_. You may compile yourself using ``pip install --no-build-isolation ``, the ``no-build-isolation`` is currently needed for pip v10. -First-time python users are often stumped by silly typos and emissions when +First-time python users are often stumped by silly typos and omissions when getting started writing code. We have improved our parser to emit more friendly `syntax errors`_, making PyPy not only faster but more friendly. From pypy.commits at gmail.com Wed Apr 25 10:34:17 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 25 Apr 2018 07:34:17 -0700 (PDT) Subject: [pypy-commit] pypy.org extradoc: update sha256 for pypy2.7 Message-ID: <5ae091e9.1c69fb81.83f65.062c@mx.google.com> Author: Matti Picus Branch: extradoc Changeset: r924:a1da300b39d3 Date: 2018-04-25 17:33 +0300 http://bitbucket.org/pypy/pypy.org/changeset/a1da300b39d3/ Log: update sha256 for pypy2.7 diff --git a/source/download.txt b/source/download.txt --- a/source/download.txt +++ b/source/download.txt @@ -82,17 +82,17 @@ Python2.7 compatible PyPy 6.0.0 ------------------------------- -* `Linux x86 binary (32bit, tar.bz2 built on Ubuntu 12.04 - 16.04)`__ (see ``[1]`` below) -* `Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 16.04)`__ (see ``[1]`` below) -* `ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Raspbian)`__ (see ``[1]`` below) -* `ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Ubuntu Raring)`__ (see ``[1]`` below) +* `Linux x86 binary (32bit, built on Ubuntu 12.04 - 16.04)`__ (see ``[1]`` below) +* `Linux x86-64 binary (64bit, built on Ubuntu 12.04 - 16.04)`__ (see ``[1]`` below) +* `ARM Hardfloat Linux binary (ARMHF/gnueabihf, Raspbian)`__ (see ``[1]`` below) +* `ARM Hardfloat Linux binary (ARMHF/gnueabihf, Ubuntu Raring)`__ (see ``[1]`` below) * `Mac OS X binary (64bit)`__ * FreeBSD x86 and x86_64: see FreshPorts_ * `Windows binary (32bit)`__ (you might need the VS 2008 runtime library installer `vcredist_x86.exe`_.) * `PowerPC PPC64 Linux binary (64bit big-endian, Fedora 20)`__ (see ``[1]`` below) * `PowerPC PPC64le Linux binary (64bit little-endian, Fedora 21)`__ (see ``[1]`` below) -* `s390x Linux binary (tar.bz2 built on Redhat Linux 7.2)`__ (see ``[1]`` below) +* `s390x Linux binary (built on Redhat Linux 7.2)`__ (see ``[1]`` below) * `Source (tar.bz2)`__; `Source (zip)`__. See below for more about the sources. * `All our downloads,`__ including previous versions. We also have a mirror_, but please use only if you have troubles accessing the links above @@ -118,15 +118,14 @@ .. class:: download_menu -* `Linux x86 binary (32bit, tar.bz2 built on Ubuntu 12.04 - 16.04)`__ (see ``[1]`` below) -* `Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 16.04)`__ (see ``[1]`` below) -* `ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Raspbian)`__ (see ``[1]`` below) -* `ARM Softfloat Linux binary (ARMEL/gnueabi, tar.bz2, Ubuntu Precise)`__ (see ``[1]`` below) -* `Mac OS X binary (64bit)`__ (High Sierra) -* `Mac OS X binary (64bit) for 5.10.0 (2)`__ (Sierra and below) +* `Linux x86 binary (32bit, built on Ubuntu 12.04 - 16.04)`__ (see ``[1]`` below) +* `Linux x86-64 binary (64bit, built on Ubuntu 12.04 - 16.04)`__ (see ``[1]`` below) +* `ARM Hardfloat Linux binary (ARMHF/gnueabihf, Raspbian)`__ (see ``[1]`` below) +* `ARM Softfloat Linux binary (ARMEL/gnueabi, Ubuntu Raring)`__ (see ``[1]`` below) +* `Mac OS X binary (64bit)`__ (High Sierra, not for Sierra and below) * `Windows binary (32bit)`__ **BETA** (you might need the VS 2008 runtime library installer `vcredist_x86.exe`_.) -* `s390x Linux binary for 5.10.0 (tar.bz2 built on Redhat Linux 7.2)`__ (see ``[1]`` below) +* `s390x Linux binary (built on Redhat Linux 7.2)`__ (see ``[1]`` below) * `Source (tar.bz2)`__; `Source (zip)`__. See below for more about the sources. * `All our downloads,`__ including previous versions. We also have a mirror_, but please use only if you have troubles accessing the links above @@ -136,7 +135,6 @@ .. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v6.0.0-linux-armhf-raspbian.tar.bz2 .. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v6.0.0-linux-armel.tar.bz2 .. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v6.0.0-osx64.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.10.0-osx64-2.tar.bz2 .. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v6.0.0-win32.zip .. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v6.0.0-s390x.tar.bz2 .. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v6.0.0-src.tar.bz2 @@ -441,19 +439,6 @@ Here are the checksums for each of the downloads -pypy2.7-5.9.0 sha256:: - - a2431a9e4ef879da1a2b56b111013b4a6efb87d4173a37bf650de47834ac5fe4 pypy2-v5.9.0-linux32.tar.bz2 - 790febd4f09e22d6e2f81154efc7dc4b2feec72712aaf4f82aa91b550abb4b48 pypy2-v5.9.0-linux64.tar.bz2 - ac0676d91dfb388c799ec5c2845f42018a666423376f52f3ae13d61fd2e6f87d pypy2-v5.9.0-linux-armel.tar.bz2 - 2597b7b21acdef4f2b81074a594157c9450363c74a17f005548c6b102f93cff4 pypy2-v5.9.0-linux-armhf-raring.tar.bz2 - b8a20042a3f34486a372ff7c751539b2e16859c0a7ea69d5a73af92f0fdcb25a pypy2-v5.9.0-linux-armhf-raspbian.tar.bz2 - 94de50ed80c7f6392ed356c03fd54cdc84858df43ad21e9e971d1b6da0f6b867 pypy2-v5.9.0-osx64.tar.bz2 - 36d6b5158801c8aa4ef5b9d8990ca0a3782a38a04916be5644a33c2a82465101 pypy2-v5.9.0-s390x.tar.bz2 - de4bf05df47f1349dbac97233d9277bbaf1ef3331663ea2557fd5da3dbcfd0a7 pypy2-v5.9.0-src.tar.bz2 - db42dbed029eeac2da1dfe9bc71d63c934106acbed6bfad8910d2dabb557d9c2 pypy2-v5.9.0-src.zip - b61081e24e05b83d8110da1262be19f0094532c6cacc293e318a1c186d926533 pypy2-v5.9.0-win32.zip - pypy2.7-5.10.0 sha256:: ee1980467ac8cc9fa9d609f7da93c5282503e59a548781248fe1914a7199d540 pypy2-v5.10.0-linux32.tar.bz2 @@ -468,6 +453,18 @@ 9afa1a36a5fc55ebc3e80576f05f44294f2b0de279862286fe00f5ee139965b1 pypy2-v5.10.0-ppc64.tar.bz2 2c32ccfa80e3e2ec56b4cc848526046d7b0de1f2f1a92b0cedeb414ec76745ab pypy2-v5.10.0-ppc64le.tar.bz2 +pypy2.7-6.0.0 sha256:: + + ad1082d4328ae8f32617b14628648583b82b6d29df3aa42b97bd1853c08c4bc8 pypy2-v6.0.0-linux32.tar.bz2 + 6cbf942ba7c90f504d8d6a2e45d4244e3bf146c8722d64e9410b85eac6b5af67 pypy2-v6.0.0-linux64.tar.bz2 + 924ca3f90aa28e8961859508c25752c95253b842318a0f267267ffe90f56a916 pypy2-v6.0.0-linux-armel.tar.bz2 + 6506ce739e31981e5596d3cc2e2c7f5b086ee77bb4d97773082b62b2f283eef2 pypy2-v6.0.0-linux-armhf-raspbian.tar.bz2 + d7dc443e6bb9a45212e8d8f5a63e9f6ce23f1d88c50709efea1c75b76c8bc186 pypy2-v6.0.0-osx64.tar.bz2 + bf155c8ac2f757d24361591080a9f4f95424a07e30f943f7d751d96442e0f36a pypy2-v6.0.0-s390x.tar.bz2 + 6097ec5ee23d0d34d8cd27a1072bed041c8a080ad48731190a03a2223029212d pypy2-v6.0.0-src.tar.bz2 + 3553b19447cdb627919cc37d76979e15dc755b085e979f5ffa9b25933ec343b3 pypy2-v6.0.0-src.zip + 6e2210dae1ae721ed4eb9cba19f15453514b64111511c84f24843c4fdefdaf7f pypy2-v6.0.0-win32.zip + pypy 3.5-v5.10.0 sha256:: f5ced20934fff78e55c72aa82a4703954349a5a8099b94e77d74b96a94326a2c pypy3-v5.10.0-osx64-2.tar.bz2 From pypy.commits at gmail.com Wed Apr 25 15:06:51 2018 From: pypy.commits at gmail.com (arigo) Date: Wed, 25 Apr 2018 12:06:51 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Merged in hroncok/pypy-3/hroncok/fix-multiprocessing-regression-on-newer--1524656522151 (pull request #607) Message-ID: <5ae0d1cb.8b281c0a.1fbb9.a383@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r94447:19216449b343 Date: 2018-04-25 19:06 +0000 http://bitbucket.org/pypy/pypy/changeset/19216449b343/ Log: Merged in hroncok/pypy-3/hroncok/fix-multiprocessing-regression-on- newer--1524656522151 (pull request #607) Fix multiprocessing regression on newer glibcs diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -379,10 +379,10 @@ for w_signum in space.unpackiterable(self.w_signals): signum = space.int_w(w_signum) check_signum_in_range(space, signum) - err = c_sigaddset(self.mask, signum) - if err: - raise oefmt(space.w_ValueError, - "signal number %d out of range", signum) + # bpo-33329: ignore c_sigaddset() return value as it can fail + # for some reserved signals, but we want the `range(1, NSIG)` + # idiom to allow selecting all valid signals. + c_sigaddset(self.mask, signum) return self.mask def __exit__(self, *args): From pypy.commits at gmail.com Wed Apr 25 15:07:01 2018 From: pypy.commits at gmail.com (hroncok) Date: Wed, 25 Apr 2018 12:07:01 -0700 (PDT) Subject: [pypy-commit] pypy hroncok/fix-multiprocessing-regression-on-newer--1524656522151: Fix multiprocessing regression on newer glibcs Message-ID: <5ae0d1d5.1c69fb81.4dbb1.3fe6@mx.google.com> Author: Miro Hrončok Branch: hroncok/fix-multiprocessing-regression-on-newer--1524656522151 Changeset: r94446:610fde79e505 Date: 2018-04-25 11:46 +0000 http://bitbucket.org/pypy/pypy/changeset/610fde79e505/ Log: Fix multiprocessing regression on newer glibcs Starting with glibc 2.27.9000-xxx, sigaddset() can return EINVAL for some reserved signal numbers between 1 and NSIG. The `range(1, NSIG)` idiom is commonly used to select all signals for blocking with `pthread_sigmask`. So we ignore the sigaddset() return value until we expose sigfillset() to provide a better idiom. Co-authored-by: Antoine Pitrou diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -379,10 +379,10 @@ for w_signum in space.unpackiterable(self.w_signals): signum = space.int_w(w_signum) check_signum_in_range(space, signum) - err = c_sigaddset(self.mask, signum) - if err: - raise oefmt(space.w_ValueError, - "signal number %d out of range", signum) + # bpo-33329: ignore c_sigaddset() return value as it can fail + # for some reserved signals, but we want the `range(1, NSIG)` + # idiom to allow selecting all valid signals. + c_sigaddset(self.mask, signum) return self.mask def __exit__(self, *args): From pypy.commits at gmail.com Wed Apr 25 19:36:23 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 25 Apr 2018 16:36:23 -0700 (PDT) Subject: [pypy-commit] pypy.org extradoc: regenerate web site Message-ID: <5ae110f7.1c69fb81.f3b16.53fc@mx.google.com> Author: Matti Picus Branch: extradoc Changeset: r925:4ce1878652ff Date: 2018-04-26 02:35 +0300 http://bitbucket.org/pypy/pypy.org/changeset/4ce1878652ff/ Log: regenerate web site diff --git a/download.html b/download.html --- a/download.html +++ b/download.html @@ -68,8 +68,8 @@

    We provide binaries for x86, ARM, PPC and s390x running on different operating systems such as Linux, Mac OS X and Windows:

      @@ -108,39 +108,37 @@
    • or translate your own PyPy.
    -
    -

    Python2.7 compatible PyPy 5.10.0

    +
    +

    Python2.7 compatible PyPy 6.0.0

    -
    -

    Python 3.5.3 compatible PyPy3.5 v5.10.1

    +
    +

    Python 3.5.3 compatible PyPy3.5 v6.0.1

    @@ -190,7 +188,7 @@ uncompressed, they run in-place. For now you can uncompress them either somewhere in your home directory or, say, in /opt, and if you want, put a symlink from somewhere like -/usr/local/bin/pypy to /path/to/pypy2-5.10.0/bin/pypy. Do +/usr/local/bin/pypy to /path/to/pypy_expanded/bin/pypy. Do not move or copy the executable pypy outside the tree – put a symlink to it, otherwise it will not find its libraries.

    @@ -269,8 +267,8 @@

    Alternatively, get one of the following smaller packages for the source at the same revision as the above binaries:

  • Make sure you installed the dependencies. See the list here.

    @@ -385,19 +383,6 @@

    Checksums

    Here are the checksums for each of the downloads

    -

    pypy2.7-5.9.0 sha256:

    -
    -a2431a9e4ef879da1a2b56b111013b4a6efb87d4173a37bf650de47834ac5fe4  pypy2-v5.9.0-linux32.tar.bz2
-790febd4f09e22d6e2f81154efc7dc4b2feec72712aaf4f82aa91b550abb4b48  pypy2-v5.9.0-linux64.tar.bz2
-ac0676d91dfb388c799ec5c2845f42018a666423376f52f3ae13d61fd2e6f87d  pypy2-v5.9.0-linux-armel.tar.bz2
-2597b7b21acdef4f2b81074a594157c9450363c74a17f005548c6b102f93cff4  pypy2-v5.9.0-linux-armhf-raring.tar.bz2
-b8a20042a3f34486a372ff7c751539b2e16859c0a7ea69d5a73af92f0fdcb25a  pypy2-v5.9.0-linux-armhf-raspbian.tar.bz2
-94de50ed80c7f6392ed356c03fd54cdc84858df43ad21e9e971d1b6da0f6b867  pypy2-v5.9.0-osx64.tar.bz2
-36d6b5158801c8aa4ef5b9d8990ca0a3782a38a04916be5644a33c2a82465101  pypy2-v5.9.0-s390x.tar.bz2
-de4bf05df47f1349dbac97233d9277bbaf1ef3331663ea2557fd5da3dbcfd0a7  pypy2-v5.9.0-src.tar.bz2
-db42dbed029eeac2da1dfe9bc71d63c934106acbed6bfad8910d2dabb557d9c2  pypy2-v5.9.0-src.zip
-b61081e24e05b83d8110da1262be19f0094532c6cacc293e318a1c186d926533  pypy2-v5.9.0-win32.zip
-

    pypy2.7-5.10.0 sha256:

     ee1980467ac8cc9fa9d609f7da93c5282503e59a548781248fe1914a7199d540  pypy2-v5.10.0-linux32.tar.bz2
@@ -412,18 +397,21 @@
 9afa1a36a5fc55ebc3e80576f05f44294f2b0de279862286fe00f5ee139965b1  pypy2-v5.10.0-ppc64.tar.bz2
 2c32ccfa80e3e2ec56b4cc848526046d7b0de1f2f1a92b0cedeb414ec76745ab  pypy2-v5.10.0-ppc64le.tar.bz2
    +

    pypy2.7-6.0.0 sha256:

    +
    +ad1082d4328ae8f32617b14628648583b82b6d29df3aa42b97bd1853c08c4bc8  pypy2-v6.0.0-linux32.tar.bz2
+6cbf942ba7c90f504d8d6a2e45d4244e3bf146c8722d64e9410b85eac6b5af67  pypy2-v6.0.0-linux64.tar.bz2
+924ca3f90aa28e8961859508c25752c95253b842318a0f267267ffe90f56a916  pypy2-v6.0.0-linux-armel.tar.bz2
+6506ce739e31981e5596d3cc2e2c7f5b086ee77bb4d97773082b62b2f283eef2  pypy2-v6.0.0-linux-armhf-raspbian.tar.bz2
+d7dc443e6bb9a45212e8d8f5a63e9f6ce23f1d88c50709efea1c75b76c8bc186  pypy2-v6.0.0-osx64.tar.bz2
+bf155c8ac2f757d24361591080a9f4f95424a07e30f943f7d751d96442e0f36a  pypy2-v6.0.0-s390x.tar.bz2
+6097ec5ee23d0d34d8cd27a1072bed041c8a080ad48731190a03a2223029212d  pypy2-v6.0.0-src.tar.bz2
+3553b19447cdb627919cc37d76979e15dc755b085e979f5ffa9b25933ec343b3  pypy2-v6.0.0-src.zip
+6e2210dae1ae721ed4eb9cba19f15453514b64111511c84f24843c4fdefdaf7f  pypy2-v6.0.0-win32.zip
+

    pypy 3.5-v5.10.0 sha256:

    -529bc3b11edbdcdd676d90c805b8f607f6eedd5f0ec457a31bbe09c03f5bebfe  pypy3-v5.10.0-linux32.tar.bz2
-aa4fb52fb858d973dd838dcf8d74f30705e5afdf1150acb8e056eb99353dfe77  pypy3-v5.10.0-linux64.tar.bz2
-c2cc529befb3e1f2ef8bd4e96af4a823c52ef2d180b0b3bd87511c5b47d59210  pypy3-v5.10.0-linux-armel.tar.bz2
-4e902e0e79f62f2a9049c1c71310ff4fc801011bec4d25082edb5c537d3f15c9  pypy3-v5.10.0-linux-armhf-raspbian.tar.bz2
-7e389a103f560de1eead1271ec3a2df9424c6ccffe7cbae8e95e6e81ae811a16  pypy3-v5.10.0-osx64.tar.bz2
 f5ced20934fff78e55c72aa82a4703954349a5a8099b94e77d74b96a94326a2c  pypy3-v5.10.0-osx64-2.tar.bz2
-e0ffec9d033002eb61af488b1f66c319380da8408abd14a3bc202ded4705dc9a  pypy3-v5.10.0-s390x.tar.bz2
-a6e4cffde71e3f08b6e1befa5c0352a9bcc5f4e9f5cbf395001e0763a1a0d9e3  pypy3-v5.10.0-src.tar.bz2
-96cf354fb410599cd5acd21732855e25e742e13eac7dc079c0c02b0625908cb9  pypy3-v5.10.0-src.zip
-2d93bf2bd7b1d031b96331d3fde6cacdda95673ce6875d6d1669c4c0ea2a52bc  pypy3-v5.10.0-win32.zip

    pypy 3.5-v5.10.1 sha256:

    @@ -437,6 +425,18 @@
 4edf4f021689a529e5a631c5cca72a1a9dc19a6ea2091e64289cdd5b60eaf929  pypy3-v5.10.1-win32.zip
 9ce98481cddede40a3357f7462f2c894bb96f178e2e8715d04feda1476ec1563  pypy3-v5.10.1-s390x.tar.bz2
    +

    pypy 3.5-v6.0.0 sha256:

    +
    +b04eeee5160e6cb5f8962de80f077ea1dc7be34e77d74bf075519c23603f5ff9  pypy3-v6.0.0-linux32.tar.bz2
+4cfffa292b9ef34bb6ba39cdbaa196c5c5cbbc5aa3faaa157cf45d7e34027048  pypy3-v6.0.0-linux64.tar.bz2
+6a6888a55192f58594838b8b3d2e7daaad43d3bf4293afab3dd8987d0bbd1124  pypy3-v6.0.0-linux-armel.tar.bz2
+8a0420dda23413925400538bbfc0cff2bbb2ab0de984eef6faaeab6d3309cbcc  pypy3-v6.0.0-linux-armhf-raspbian.tar.bz2
+938b8034e30f5f5060d2a079070c56c3be5559bc7ae9cc0c8395fe6fc45cfe4c  pypy3-v6.0.0-osx64.tar.bz2
+f0a3097cf74d6b1fb3ae2a060384e72b7868ca76cd04584336b417e9982ec0a5  pypy3-v6.0.0-s390x.tar.bz2
+ed8005202b46d6fc6831df1d13a4613bc40084bfa42f275068edadf8954034a3  pypy3-v6.0.0-src.tar.bz2
+8cd3cc2ef362e774edf9c7a6b79dbe42fff75217c5ed96b235a0a792e4421dc4  pypy3-v6.0.0-src.zip
+72dddb3746a51f7672c77d619c818e27efe899e08ae82762448e50dbfdc2f5f3  pypy3-v6.0.0-win32.zip
+

    • Python2.7 compatible PyPy 6.0.0

    -
      +
      • Linux x86 binary (32bit, built on Ubuntu 12.04 - 16.04) (see [1] below)
      • Linux x86-64 binary (64bit, built on Ubuntu 12.04 - 16.04) (see [1] below)
      • ARM Hardfloat Linux binary (ARMHF/gnueabihf, Raspbian) (see [1] below)
        • @@ -135,8 +135,7 @@
      • ARM Hardfloat Linux binary (ARMHF/gnueabihf, Raspbian) (see [1] below)
      • ARM Softfloat Linux binary (ARMEL/gnueabi, Ubuntu Raring) (see [1] below)
      • Mac OS X binary (64bit) (High Sierra, not for Sierra and below)
        • -
      • Windows binary (32bit) BETA (you might need the VS 2008 runtime library -installer vcredist_x86.exe.)
        • +
      • Windows binary (32bit) BETA
      • s390x Linux binary (built on Redhat Linux 7.2) (see [1] below)
      • Source (tar.bz2); Source (zip). See below for more about the sources.
      • All our downloads, including previous versions. We also have a diff --git a/source/download.txt b/source/download.txt --- a/source/download.txt +++ b/source/download.txt @@ -82,6 +82,8 @@ Python2.7 compatible PyPy 6.0.0 ------------------------------- +.. class:: download_menu + * `Linux x86 binary (32bit, built on Ubuntu 12.04 - 16.04)`__ (see ``[1]`` below) * `Linux x86-64 binary (64bit, built on Ubuntu 12.04 - 16.04)`__ (see ``[1]`` below) * `ARM Hardfloat Linux binary (ARMHF/gnueabihf, Raspbian)`__ (see ``[1]`` below) @@ -123,8 +125,7 @@ * `ARM Hardfloat Linux binary (ARMHF/gnueabihf, Raspbian)`__ (see ``[1]`` below) * `ARM Softfloat Linux binary (ARMEL/gnueabi, Ubuntu Raring)`__ (see ``[1]`` below) * `Mac OS X binary (64bit)`__ (High Sierra, not for Sierra and below) -* `Windows binary (32bit)`__ **BETA** (you might need the VS 2008 runtime library - installer `vcredist_x86.exe`_.) +* `Windows binary (32bit)`__ **BETA** * `s390x Linux binary (built on Redhat Linux 7.2)`__ (see ``[1]`` below) * `Source (tar.bz2)`__; `Source (zip)`__. See below for more about the sources. * `All our downloads,`__ including previous versions. We also have a From pypy.commits at gmail.com Thu Apr 26 07:29:14 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 26 Apr 2018 04:29:14 -0700 (PDT) Subject: [pypy-commit] pypy.org extradoc: typo (nedbat) Message-ID: <5ae1b80a.0e941c0a.69d4f.011e@mx.google.com> Author: Matti Picus Branch: extradoc Changeset: r927:ad1d49d4c6ef Date: 2018-04-26 14:28 +0300 http://bitbucket.org/pypy/pypy.org/changeset/ad1d49d4c6ef/ Log: typo (nedbat) diff --git a/download.html b/download.html --- a/download.html +++ b/download.html @@ -69,7 +69,7 @@ Linux, Mac OS X and Windows:

        • the Python2.7 compatible release — PyPy2.7 v6.0 — (what's new in PyPy2.7?)
          • -
        • the Python3.5 compatible release — PyPy3.5 v6.0 — (what's new in PyPy3.5?).
          • +
        • the Python3.5 compatible release — PyPy3.5 v6.0 — (what's new in PyPy3.5?).
        • the Python2.7 Software Transactional Memory special release — PyPy-STM 2.5.1 (Linux x86-64 only)
          @@ -127,8 +127,8 @@ mirror, but please use only if you have troubles accessing the links above
    -
    -

    Python 3.5.3 compatible PyPy3.5 v6.0.1

    +
    +

    Python 3.5.3 compatible PyPy3.5 v6.0.0

    • Linux x86 binary (32bit, built on Ubuntu 12.04 - 16.04) (see [1] below)
    • Linux x86-64 binary (64bit, built on Ubuntu 12.04 - 16.04) (see [1] below)
      • @@ -266,8 +266,8 @@

      Alternatively, get one of the following smaller packages for the source at the same revision as the above binaries:

    • Make sure you installed the dependencies. See the list here.

      diff --git a/source/download.txt b/source/download.txt --- a/source/download.txt +++ b/source/download.txt @@ -22,7 +22,7 @@ * the Python2.7 Software Transactional Memory special release — **PyPy-STM 2.5.1** (Linux x86-64 only) .. _what's new in PyPy2.7?: http://doc.pypy.org/en/latest/release-v6.0.0.html -.. _what's new in PyPy3.5?: http://doc.pypy.org/en/latest/release-v6.0.1.html +.. _what's new in PyPy3.5?: http://doc.pypy.org/en/latest/release-v6.0.0.html .. class:: download_menu @@ -115,7 +115,7 @@ .. _mirror: http://buildbot.pypy.org/mirror/ .. _FreshPorts: http://www.freshports.org/lang/pypy -Python 3.5.3 compatible PyPy3.5 v6.0.1 +Python 3.5.3 compatible PyPy3.5 v6.0.0 --------------------------------------- .. class:: download_menu @@ -306,8 +306,8 @@ Alternatively, get one of the following smaller packages for the source at the same revision as the above binaries: - * `pypy2-v6.0.1-src.tar.bz2`__ (sources, PyPy 2 only) - * `pypy3-v6.0.1-src.tar.bz2`__ (sources, PyPy 3 only) + * `pypy2-v6.0.0-src.tar.bz2`__ (sources, PyPy 2 only) + * `pypy3-v6.0.0-src.tar.bz2`__ (sources, PyPy 3 only) .. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v6.0.0-src.tar.bz2 .. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v6.0.0-src.tar.bz2 From pypy.commits at gmail.com Thu Apr 26 13:34:05 2018 From: pypy.commits at gmail.com (wlav) Date: Thu, 26 Apr 2018 10:34:05 -0700 (PDT) Subject: [pypy-commit] pypy cppyy-packaging: const correctness for data members and associated tests Message-ID: <5ae20d8d.1c69fb81.d4a11.a110@mx.google.com> Author: Wim Lavrijsen Branch: cppyy-packaging Changeset: r94449:e7331182c14c Date: 2018-04-24 13:39 -0700 http://bitbucket.org/pypy/pypy/changeset/e7331182c14c/ Log: const correctness for data members and associated tests diff --git a/pypy/module/_cppyy/__init__.py b/pypy/module/_cppyy/__init__.py --- a/pypy/module/_cppyy/__init__.py +++ b/pypy/module/_cppyy/__init__.py @@ -7,6 +7,7 @@ interpleveldefs = { '_resolve_name' : 'interp_cppyy.resolve_name', '_scope_byname' : 'interp_cppyy.scope_byname', + '_is_static_data' : 'interp_cppyy.is_static_data', '_is_template' : 'interp_cppyy.is_template', '_std_string_name' : 'interp_cppyy.std_string_name', '_set_class_generator' : 'interp_cppyy.set_class_generator', diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py --- a/pypy/module/_cppyy/interp_cppyy.py +++ b/pypy/module/_cppyy/interp_cppyy.py @@ -149,6 +149,24 @@ W_CPPLibrary.typedef.acceptable_as_base_class = True +#----- +# Classes involved with methods and functions: +# +# CPPMethod: base class wrapping a single function or method +# CPPConstructor: specialization for allocating a new object +# CPPFunction: specialization for free and static functions +# CPPSetItem: specialization for Python's __setitem__ +# CPPTemplatedCall: trampoline to instantiate and bind templated functions +# W_CPPOverload, W_CPPConstructorOverload, W_CPPTemplateOverload: +# user-facing, app-level, collection of overloads, with specializations +# for constructors and templates +# W_CPPBoundMethod: instantiated template method +# +# All methods/functions derive from CPPMethod and are collected as overload +# candidates in user-facing overload classes. Templated methods are a two-step +# process, where first the template is instantiated (or selected if already +# available), which returns a callable object that is the actual bound method. + class CPPMethod(object): """Dispatcher of methods. Checks the arguments, find the corresponding FFI function if available, makes the call, and returns the wrapped result. It @@ -688,6 +706,18 @@ ) +#----- +# Classes for data members: +# +# W_CPPDataMember: instance data members +# W_CPPConstDataMember: specialization for const data members +# W_CPPStaticData: class-level and global/static data +# W_CPPConstStaticData: specialization for const global/static data +# +# Data is represented by an offset which is either a global pointer (static data) +# or an offset from the start of an instance (data members). The "const" +# specializations raise when attempting to set their value. + class W_CPPDataMember(W_Root): _attrs_ = ['space', 'scope', 'converter', 'offset'] _immutable_fields = ['scope', 'converter', 'offset'] @@ -698,9 +728,6 @@ self.converter = converter.get_converter(self.space, type_name, '') self.offset = offset - def is_static(self): - return self.space.w_False - def _get_offset(self, cppinstance): if cppinstance: assert lltype.typeOf(cppinstance.clsdecl.handle) == lltype.typeOf(self.scope.handle) @@ -728,16 +755,25 @@ W_CPPDataMember.typedef = TypeDef( 'CPPDataMember', - is_static = interp2app(W_CPPDataMember.is_static), __get__ = interp2app(W_CPPDataMember.get), __set__ = interp2app(W_CPPDataMember.set), ) W_CPPDataMember.typedef.acceptable_as_base_class = False + +class W_CPPConstDataMember(W_CPPDataMember): + def set(self, w_cppinstance, w_value): + raise oefmt(self.space.w_TypeError, "assignment to const data not allowed") + +W_CPPConstDataMember.typedef = TypeDef( + 'CPPConstDataMember', + __get__ = interp2app(W_CPPDataMember.get), + __set__ = interp2app(W_CPPConstDataMember.set), +) +W_CPPConstDataMember.typedef.acceptable_as_base_class = False + + class W_CPPStaticData(W_CPPDataMember): - def is_static(self): - return self.space.w_True - @jit.elidable_promote() def _get_offset(self, cppinstance): return self.offset @@ -751,19 +787,34 @@ W_CPPStaticData.typedef = TypeDef( 'CPPStaticData', - is_static = interp2app(W_CPPStaticData.is_static), __get__ = interp2app(W_CPPStaticData.get), __set__ = interp2app(W_CPPStaticData.set), ) W_CPPStaticData.typedef.acceptable_as_base_class = False -def is_static(space, w_obj): + +class W_CPPConstStaticData(W_CPPStaticData): + def set(self, w_cppinstance, w_value): + raise oefmt(self.space.w_TypeError, "assignment to const data not allowed") + +W_CPPConstStaticData.typedef = TypeDef( + 'CPPConstStaticData', + __get__ = interp2app(W_CPPConstStaticData.get), + __set__ = interp2app(W_CPPConstStaticData.set), +) +W_CPPConstStaticData.typedef.acceptable_as_base_class = False + + +def is_static_data(space, w_obj): try: space.interp_w(W_CPPStaticData, w_obj, can_be_None=False) return space.w_True except Exception: return space.w_False +#----- + + class W_CPPScopeDecl(W_Root): _attrs_ = ['space', 'handle', 'name', 'methods', 'datamembers'] _immutable_fields_ = ['handle', 'name'] @@ -847,7 +898,10 @@ offset = capi.c_datamember_offset(self.space, self, dm_idx) if offset == -1: raise self.missing_attribute_error(dm_name) - datamember = W_CPPStaticData(self.space, self, type_name, offset) + if capi.c_is_const_data(self.space, self, dm_idx): + datamember = W_CPPConstStaticData(self.space, self, type_name, offset) + else: + datamember = W_CPPStaticData(self.space, self, type_name, offset) self.datamembers[dm_name] = datamember return datamember @@ -967,8 +1021,13 @@ if offset == -1: continue # dictionary problem; raises AttributeError on use is_static = bool(capi.c_is_staticdata(self.space, self, i)) - if is_static: + is_const = bool(capi.c_is_const_data(self.space, self, i)) + if is_static and is_const: + datamember = W_CPPConstStaticData(self.space, self, type_name, offset) + elif is_static: datamember = W_CPPStaticData(self.space, self, type_name, offset) + elif is_const: + datamember = W_CPPConstDataMember(self.space, self, type_name, offset) else: datamember = W_CPPDataMember(self.space, self, type_name, offset) self.datamembers[datamember_name] = datamember diff --git a/pypy/module/_cppyy/pythonify.py b/pypy/module/_cppyy/pythonify.py --- a/pypy/module/_cppyy/pythonify.py +++ b/pypy/module/_cppyy/pythonify.py @@ -169,6 +169,7 @@ return method def make_cppclass(scope, cl_name, decl): + import _cppyy # get a list of base classes for class creation bases = [get_pycppclass(base) for base in decl.get_base_names()] @@ -209,7 +210,7 @@ for d_name in decl.get_datamember_names(): cppdm = decl.get_datamember(d_name) d_class[d_name] = cppdm - if cppdm.is_static(): + if _cppyy._is_static_data(cppdm): d_meta[d_name] = cppdm # create a metaclass to allow properties (for static data write access) @@ -278,7 +279,7 @@ try: cppdm = scope.__cppdecl__.get_datamember(name) setattr(scope, name, cppdm) - if cppdm.is_static(): + if _cppyy._is_static_data(cppdm): setattr(scope.__class__, name, cppdm) pycppitem = getattr(scope, name) # gets actual property value except AttributeError: diff --git a/pypy/module/_cppyy/test/datatypes.cxx b/pypy/module/_cppyy/test/datatypes.cxx --- a/pypy/module/_cppyy/test/datatypes.cxx +++ b/pypy/module/_cppyy/test/datatypes.cxx @@ -6,7 +6,7 @@ //=========================================================================== -CppyyTestData::CppyyTestData() : m_owns_arrays(false) +CppyyTestData::CppyyTestData() : m_const_int(17), m_owns_arrays(false) { m_bool = false; m_char = 'a'; @@ -333,3 +333,17 @@ CppyyTestPod* get_null_pod() { return (CppyyTestPod*)0; } + + +//= function pointer passing ================================================ +int sum_of_int(int i1, int i2) { + return i1+i2; +} + +double sum_of_double(double d1, double d2) { + return d1+d2; +} + +double call_double_double(double (*d)(double, double), double d1, double d2) { + return d(d1, d2); +} diff --git a/pypy/module/_cppyy/test/datatypes.h b/pypy/module/_cppyy/test/datatypes.h --- a/pypy/module/_cppyy/test/datatypes.h +++ b/pypy/module/_cppyy/test/datatypes.h @@ -1,5 +1,5 @@ // copied from RtypesCore.h ... -#if defined(R__WIN32) +#if defined(R__WIN32) && !defined(__CINT__) typedef __int64 Long64_t; //Portable signed long integer 8 bytes typedef unsigned __int64 ULong64_t; //Portable unsigned long integer 8 bytes #else @@ -26,8 +26,13 @@ //=========================================================================== namespace EnumSpace { - enum E {E1 = 1, E2}; -}; + enum E {E1 = 1, E2}; + class EnumClass { + public: + enum {E1 = -1}; + enum EE {E2 = -1}; + }; +} //=========================================================================== @@ -243,6 +248,7 @@ short m_short; unsigned short m_ushort; int m_int; + const int m_const_int; // special case: const testing unsigned int m_uint; long m_long; unsigned long m_ulong; @@ -364,3 +370,9 @@ void set_global_pod(CppyyTestPod* t); CppyyTestPod* get_global_pod(); CppyyTestPod* get_null_pod(); + + +//= function pointer passing ================================================ +int sum_of_int(int i1, int i2); +double sum_of_double(double d1, double d2); +double call_double_double(double (*d)(double, double), double d1, double d2); diff --git a/pypy/module/_cppyy/test/datatypes.xml b/pypy/module/_cppyy/test/datatypes.xml --- a/pypy/module/_cppyy/test/datatypes.xml +++ b/pypy/module/_cppyy/test/datatypes.xml @@ -4,6 +4,8 @@ + + @@ -14,4 +16,8 @@ + + + + diff --git a/pypy/module/_cppyy/test/test_datatypes.py b/pypy/module/_cppyy/test/test_datatypes.py --- a/pypy/module/_cppyy/test/test_datatypes.py +++ b/pypy/module/_cppyy/test/test_datatypes.py @@ -191,6 +191,10 @@ for i in range(self.N): assert eval('c.m_%s_array2[i]' % names[j]) == b[i] + # can not write to constant data + assert c.m_const_int == 17 + raises(TypeError, setattr, c, 'm_const_int', 71) + c.__destruct__() def test03_array_passing(self): @@ -464,6 +468,10 @@ assert gbl.kBanana == 29 assert gbl.kCitrus == 34 + assert gbl.EnumSpace.E + assert gbl.EnumSpace.EnumClass.E1 == -1 # anonymous + assert gbl.EnumSpace.EnumClass.E2 == -1 # named type + def test11_string_passing(self): """Test passing/returning of a const char*""" @@ -741,3 +749,19 @@ c.s_voidp = c2 address_equality_test(c.s_voidp, c2) + + def test21_function_pointers(self): + """Function pointer passing""" + + import _cppyy as cppyy + + f1 = cppyy.gbl.sum_of_int + f2 = cppyy.gbl.sum_of_double + f3 = cppyy.gbl.call_double_double + + assert 5 == f1(2, 3) + assert 5. == f2(5., 0.) + + raises(TypeError, f3, f1, 2, 3) + + assert 5. == f3(f2, 5., 0.) From pypy.commits at gmail.com Thu Apr 26 13:34:11 2018 From: pypy.commits at gmail.com (wlav) Date: Thu, 26 Apr 2018 10:34:11 -0700 (PDT) Subject: [pypy-commit] pypy cppyy-packaging: anotator fixes Message-ID: <5ae20d93.1c69fb81.2b752.06e8@mx.google.com> Author: Wim Lavrijsen Branch: cppyy-packaging Changeset: r94452:ae0244400329 Date: 2018-04-26 10:08 -0700 http://bitbucket.org/pypy/pypy/changeset/ae0244400329/ Log: anotator fixes diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py --- a/pypy/module/_cppyy/converter.py +++ b/pypy/module/_cppyy/converter.py @@ -687,6 +687,8 @@ class FunctionPointerConverter(TypeConverter): + _immutable_fields_ = ['signature'] + def __init__(self, space, signature): self.signature = signature @@ -780,7 +782,9 @@ return _converters["internal_enum_type_t"](space, default) elif "(*)" in name or "::*)" in name: # function pointer - return FunctionPointerConverter(space, name[name.find("*)")+2:]) + pos = name.find("*)") + if pos > 0: + return FunctionPointerConverter(space, name[pos+2:]) # 5) void* or void converter (which fails on use) if 0 <= compound.find('*'): From pypy.commits at gmail.com Thu Apr 26 13:34:07 2018 From: pypy.commits at gmail.com (wlav) Date: Thu, 26 Apr 2018 10:34:07 -0700 (PDT) Subject: [pypy-commit] pypy cppyy-packaging: support anonymous enums Message-ID: <5ae20d8f.08c41c0a.c892.4cbb@mx.google.com> Author: Wim Lavrijsen Branch: cppyy-packaging Changeset: r94450:682a43ef7a89 Date: 2018-04-24 14:28 -0700 http://bitbucket.org/pypy/pypy/changeset/682a43ef7a89/ Log: support anonymous enums diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py --- a/pypy/module/_cppyy/converter.py +++ b/pypy/module/_cppyy/converter.py @@ -749,6 +749,9 @@ return InstancePtrPtrConverter(space, clsdecl) elif compound == "": return InstanceConverter(space, clsdecl) + elif "(anonymous)" in name: + # special case: enum w/o a type name + return _converters["internal_enum_type_t"](space, default) # 5) void* or void converter (which fails on use) if 0 <= compound.find('*'): diff --git a/pypy/module/_cppyy/executor.py b/pypy/module/_cppyy/executor.py --- a/pypy/module/_cppyy/executor.py +++ b/pypy/module/_cppyy/executor.py @@ -289,6 +289,9 @@ return InstancePtrExecutor(space, cppclass) elif compound == '**' or compound == '*&': return InstancePtrPtrExecutor(space, cppclass) + elif "(anonymous)" in name: + # special case: enum w/o a type name + return _executors["internal_enum_type_t"](space, None) # 4) additional special cases if compound == '*': From pypy.commits at gmail.com Thu Apr 26 13:34:13 2018 From: pypy.commits at gmail.com (wlav) Date: Thu, 26 Apr 2018 10:34:13 -0700 (PDT) Subject: [pypy-commit] pypy cppyy-packaging: add extra info in dummy_backend for const_int test Message-ID: <5ae20d95.530a1c0a.32aa7.0b8d@mx.google.com> Author: Wim Lavrijsen Branch: cppyy-packaging Changeset: r94453:b762bdca4513 Date: 2018-04-26 10:16 -0700 http://bitbucket.org/pypy/pypy/changeset/b762bdca4513/ Log: add extra info in dummy_backend for const_int test diff --git a/pypy/module/_cppyy/src/dummy_backend.cxx b/pypy/module/_cppyy/src/dummy_backend.cxx --- a/pypy/module/_cppyy/src/dummy_backend.cxx +++ b/pypy/module/_cppyy/src/dummy_backend.cxx @@ -348,6 +348,7 @@ PUBLIC_CPPYY_DATA3(short, short, h); PUBLIC_CPPYY_DATA3(ushort, unsigned short, H); PUBLIC_CPPYY_DATA3(int, int, i); + PUBLIC_CPPYY_DATA (const_int, const int); PUBLIC_CPPYY_DATA3(uint, unsigned int, I); PUBLIC_CPPYY_DATA3(long, long, l); PUBLIC_CPPYY_DATA3(ulong, unsigned long, L); @@ -1032,7 +1033,9 @@ return s_scopes[handle].m_datambrs[idatambr].m_isstatic; } -int cppyy_is_const_data(cppyy_scope_t /* handle */, cppyy_index_t /* idatambr */) { +int cppyy_is_const_data(cppyy_scope_t handle, cppyy_index_t idatambr) { + if (s_scopes[handle].m_datambrs[idatambr].m_name == "m_const_int") + return 1; return 0; } From pypy.commits at gmail.com Thu Apr 26 13:34:09 2018 From: pypy.commits at gmail.com (wlav) Date: Thu, 26 Apr 2018 10:34:09 -0700 (PDT) Subject: [pypy-commit] pypy cppyy-packaging: initial support for function pointer arguments Message-ID: <5ae20d91.481a1c0a.90358.8d56@mx.google.com> Author: Wim Lavrijsen Branch: cppyy-packaging Changeset: r94451:f8f27990a737 Date: 2018-04-25 16:59 -0700 http://bitbucket.org/pypy/pypy/changeset/f8f27990a737/ Log: initial support for function pointer arguments diff --git a/pypy/module/_cppyy/capi/loadable_capi.py b/pypy/module/_cppyy/capi/loadable_capi.py --- a/pypy/module/_cppyy/capi/loadable_capi.py +++ b/pypy/module/_cppyy/capi/loadable_capi.py @@ -421,7 +421,7 @@ _cdata_to_ptr(space, call_capi(space, 'function_address_from_index', args))) def c_function_address_from_method(space, cppmethod): return rffi.cast(C_FUNC_PTR, - _cdata_to_ptr(space, call_capi(space, 'function_address_from_method', _ArgH(cppmethod)))) + _cdata_to_ptr(space, call_capi(space, 'function_address_from_method', [_ArgH(cppmethod)]))) # handling of function argument buffer --------------------------------------- def c_allocate_function_args(space, size): diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py --- a/pypy/module/_cppyy/converter.py +++ b/pypy/module/_cppyy/converter.py @@ -686,6 +686,32 @@ decref(space, rffi.cast(PyObject, rffi.cast(rffi.VOIDPP, arg)[0])) +class FunctionPointerConverter(TypeConverter): + def __init__(self, space, signature): + self.signature = signature + + def convert_argument(self, space, w_obj, address, call_local): + # TODO: atm, does not actually get an overload, but a staticmethod + from pypy.module._cppyy.interp_cppyy import W_CPPOverload + cppol = space.interp_w(W_CPPOverload, w_obj) + + # find the function with matching signature + for i in range(len(cppol.functions)): + m = cppol.functions[i] + if m.signature(False) == self.signature: + x = rffi.cast(rffi.VOIDPP, address) + x[0] = rffi.cast(rffi.VOIDP, + capi.c_function_address_from_method(space, m.cppmethod)) + address = rffi.cast(capi.C_OBJECT, address) + ba = rffi.cast(rffi.CCHARP, address) + ba[capi.c_function_arg_typeoffset(space)] = 'p' + return + + # lookup failed + raise oefmt(space.w_TypeError, + "no overload found matching %s", self.signature) + + class MacroConverter(TypeConverter): def from_memory(self, space, w_obj, w_pycppclass, offset): # TODO: get the actual type info from somewhere ... @@ -752,6 +778,9 @@ elif "(anonymous)" in name: # special case: enum w/o a type name return _converters["internal_enum_type_t"](space, default) + elif "(*)" in name or "::*)" in name: + # function pointer + return FunctionPointerConverter(space, name[name.find("*)")+2:]) # 5) void* or void converter (which fails on use) if 0 <= compound.find('*'): diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py --- a/pypy/module/_cppyy/interp_cppyy.py +++ b/pypy/module/_cppyy/interp_cppyy.py @@ -128,7 +128,7 @@ def register_class(space, w_pycppclass): w_cppclass = space.findattr(w_pycppclass, space.newtext("__cppdecl__")) - cppclass = space.interp_w(W_CPPClassDecl, w_cppclass, can_be_None=False) + cppclass = space.interp_w(W_CPPClassDecl, w_cppclass) # add back-end specific method pythonizations (doing this on the wrapped # class allows simple aliasing of methods) capi.pythonize(space, cppclass.name, w_pycppclass) @@ -195,7 +195,7 @@ @staticmethod def unpack_cppthis(space, w_cppinstance, declaring_scope): - cppinstance = space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=False) + cppinstance = space.interp_w(W_CPPInstance, w_cppinstance) cppinstance._nullcheck() return cppinstance.get_cppthis(declaring_scope) @@ -442,7 +442,7 @@ class CPPFunction(CPPMethod): - """Global (namespaced) function dispatcher.""" + """Global (namespaced) / static function dispatcher.""" _immutable_ = True @@ -807,7 +807,7 @@ def is_static_data(space, w_obj): try: - space.interp_w(W_CPPStaticData, w_obj, can_be_None=False) + space.interp_w(W_CPPStaticData, w_obj) return space.w_True except Exception: return space.w_False @@ -1183,7 +1183,7 @@ # scopes of the argument classes (TODO: implement that last option) try: # TODO: expecting w_other to be an W_CPPInstance is too limiting - other = self.space.interp_w(W_CPPInstance, w_other, can_be_None=False) + other = self.space.interp_w(W_CPPInstance, w_other) for name in ["", "__gnu_cxx", "__1"]: nss = scope_byname(self.space, name) meth_idx = capi.c_get_global_operator( @@ -1205,7 +1205,7 @@ # fallback 2: direct pointer comparison (the class comparison is needed since # the first data member in a struct and the struct have the same address) - other = self.space.interp_w(W_CPPInstance, w_other, can_be_None=False) # TODO: factor out + other = self.space.interp_w(W_CPPInstance, w_other) # TODO: factor out iseq = (self._rawobject == other._rawobject) and (self.clsdecl == other.clsdecl) return self.space.newbool(iseq) @@ -1322,7 +1322,7 @@ offset = capi.c_base_offset1(space, actual, clsdecl, rawobject, -1) rawobject = capi.direct_ptradd(rawobject, offset) w_cppdecl = space.findattr(w_pycppclass, space.newtext("__cppdecl__")) - clsdecl = space.interp_w(W_CPPClassDecl, w_cppdecl, can_be_None=False) + clsdecl = space.interp_w(W_CPPClassDecl, w_cppdecl) except Exception: # failed to locate/build the derived class, so stick to the base (note # that only get_pythonized_cppclass is expected to raise, so none of @@ -1340,7 +1340,7 @@ # fresh creation w_cppinstance = space.allocate_instance(W_CPPInstance, w_pycppclass) - cppinstance = space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=False) + cppinstance = space.interp_w(W_CPPInstance, w_cppinstance) cppinstance.__init__(space, clsdecl, rawobject, is_ref, python_owns) memory_regulator.register(cppinstance) return w_cppinstance @@ -1368,7 +1368,7 @@ except Exception: # accept integer value as address rawobject = rffi.cast(capi.C_OBJECT, space.uint_w(w_obj)) - decl = space.interp_w(W_CPPClassDecl, w_clsdecl, can_be_None=False) + decl = space.interp_w(W_CPPClassDecl, w_clsdecl) return wrap_cppinstance(space, rawobject, decl, python_owns=owns, do_cast=cast) @unwrap_spec(owns=bool, cast=bool) @@ -1384,7 +1384,7 @@ def move(space, w_obj): """Casts the given instance into an C++-style rvalue.""" - obj = space.interp_w(W_CPPInstance, w_obj, can_be_None=True) + obj = space.interp_w(W_CPPInstance, w_obj) if obj: obj.flags |= INSTANCE_FLAGS_IS_R_VALUE return w_obj diff --git a/pypy/module/_cppyy/test/test_datatypes.py b/pypy/module/_cppyy/test/test_datatypes.py --- a/pypy/module/_cppyy/test/test_datatypes.py +++ b/pypy/module/_cppyy/test/test_datatypes.py @@ -764,4 +764,7 @@ raises(TypeError, f3, f1, 2, 3) + # TODO: get straightforward access to the overload type + f2 = cppyy.gbl.__cppdecl__.get_overload('sum_of_double') + assert 5. == f3(f2, 5., 0.) From pypy.commits at gmail.com Fri Apr 27 22:25:36 2018 From: pypy.commits at gmail.com (arigo) Date: Fri, 27 Apr 2018 19:25:36 -0700 (PDT) Subject: [pypy-commit] pypy default: Issue #2813 Message-ID: <5ae3dba0.06581c0a.3abc4.967b@mx.google.com> Author: Armin Rigo Branch: Changeset: r94456:4a06e23782cf Date: 2018-04-28 04:24 +0200 http://bitbucket.org/pypy/pypy/changeset/4a06e23782cf/ Log: Issue #2813 Fix for ctypes: doing some operations before setting the _fields_ of a Structure would make and cache an _ffiargtype that says "opaque". diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py --- a/lib_pypy/_ctypes/array.py +++ b/lib_pypy/_ctypes/array.py @@ -82,8 +82,11 @@ def _CData_output(self, resarray, base=None, index=-1): from _rawffi.alt import types # If a char_p or unichar_p is received, skip the string interpretation - if base._ffiargtype != types.Pointer(types.char_p) and \ - base._ffiargtype != types.Pointer(types.unichar_p): + try: + deref = type(base)._deref_ffiargtype() + except AttributeError: + deref = None + if deref != types.char_p and deref != types.unichar_p: # this seems to be a string if we're array of char, surprise! from ctypes import c_char, c_wchar if self._type_ is c_char: @@ -120,6 +123,12 @@ value = self(*value) return _CDataMeta.from_param(self, value) + def _build_ffiargtype(self): + return _ffi.types.Pointer(self._type_.get_ffi_argtype()) + + def _deref_ffiargtype(self): + return self._type_.get_ffi_argtype() + def array_get_slice_params(self, index): if hasattr(self, '_length_'): start, stop, step = index.indices(self._length_) @@ -248,6 +257,5 @@ _type_ = base ) cls = ArrayMeta(name, (Array,), tpdict) - cls._ffiargtype = _ffi.types.Pointer(base.get_ffi_argtype()) ARRAY_CACHE[key] = cls return cls diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py --- a/lib_pypy/_ctypes/basics.py +++ b/lib_pypy/_ctypes/basics.py @@ -49,10 +49,13 @@ else: return self.from_param(as_parameter) + def _build_ffiargtype(self): + return _shape_to_ffi_type(self._ffiargshape_) + def get_ffi_argtype(self): if self._ffiargtype: return self._ffiargtype - self._ffiargtype = _shape_to_ffi_type(self._ffiargshape_) + self._ffiargtype = self._build_ffiargtype() return self._ffiargtype def _CData_output(self, resbuffer, base=None, index=-1): diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py --- a/lib_pypy/_ctypes/pointer.py +++ b/lib_pypy/_ctypes/pointer.py @@ -70,7 +70,12 @@ self._ffiarray = ffiarray self.__init__ = __init__ self._type_ = TP - self._ffiargtype = _ffi.types.Pointer(TP.get_ffi_argtype()) + + def _build_ffiargtype(self): + return _ffi.types.Pointer(self._type_.get_ffi_argtype()) + + def _deref_ffiargtype(self): + return self._type_.get_ffi_argtype() from_address = cdata_from_address diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py --- a/lib_pypy/_ctypes/structure.py +++ b/lib_pypy/_ctypes/structure.py @@ -160,6 +160,10 @@ raise AttributeError("_fields_ is final") if self in [f[1] for f in value]: raise AttributeError("Structure or union cannot contain itself") + if self._ffiargtype is not None: + raise NotImplementedError("Too late to set _fields_: we already " + "said to libffi that the structure type %s is opaque" + % (self,)) names_and_fields( self, value, self.__bases__[0], diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py b/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py @@ -272,3 +272,18 @@ base = cast(d, c_void_p).value for i in [0, 1, 4, 1444, -10293]: assert cast(byref(c, i), c_void_p).value == base + i + + def test_issue2813_fix(self): + class C(Structure): + pass + POINTER(C) + C._fields_ = [('x', c_int)] + ffitype = C.get_ffi_argtype() + assert C.get_ffi_argtype() is ffitype + assert ffitype.sizeof() == sizeof(c_int) + + def test_issue2813_cant_change_fields_after_get_ffi_argtype(self): + class C(Structure): + pass + ffitype = C.get_ffi_argtype() + raises(NotImplementedError, "C._fields_ = [('x', c_int)]") From pypy.commits at gmail.com Mon Apr 30 12:27:58 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 30 Apr 2018 09:27:58 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: merge default into py3.5 Message-ID: <5ae7440e.45931c0a.7853.1e29@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r94457:8a3e586cc749 Date: 2018-04-29 20:56 +0300 http://bitbucket.org/pypy/pypy/changeset/8a3e586cc749/ Log: merge default into py3.5 diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -51,3 +51,5 @@ 0000000000000000000000000000000000000000 release-pypy3.5-v5.10.0 09f9160b643e3f02ccb8c843b2fbb4e5cbf54082 release-pypy3.5-v5.10.0 3f6eaa010fce78cc7973bdc1dfdb95970f08fed2 release-pypy3.5-v5.10.1 +ab0b9caf307db6592905a80b8faffd69b39005b8 release-pypy2.7-v6.0.0 +fdd60ed87e941677e8ea11acf9f1819466521bf2 release-pypy3.5-v6.0.0 diff --git a/dotviewer/font/NOTICE b/dotviewer/font/COPYING.txt rename from dotviewer/font/NOTICE rename to dotviewer/font/COPYING.txt diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py --- a/lib_pypy/_ctypes/array.py +++ b/lib_pypy/_ctypes/array.py @@ -81,8 +81,11 @@ def _CData_output(self, resarray, base=None, index=-1): from _rawffi.alt import types # If a char_p or unichar_p is received, skip the string interpretation - if base._ffiargtype != types.Pointer(types.char_p) and \ - base._ffiargtype != types.Pointer(types.unichar_p): + try: + deref = type(base)._deref_ffiargtype() + except AttributeError: + deref = None + if deref != types.char_p and deref != types.unichar_p: # this seems to be a string if we're array of char, surprise! from ctypes import c_char, c_wchar if self._type_ is c_char: @@ -127,6 +130,12 @@ value = self(*value) return _CDataMeta.from_param(self, value) + def _build_ffiargtype(self): + return _ffi.types.Pointer(self._type_.get_ffi_argtype()) + + def _deref_ffiargtype(self): + return self._type_.get_ffi_argtype() + def array_get_slice_params(self, index): if hasattr(self, '_length_'): start, stop, step = index.indices(self._length_) @@ -254,6 +263,5 @@ _type_ = base ) cls = ArrayMeta(name, (Array,), tpdict) - cls._ffiargtype = _ffi.types.Pointer(base.get_ffi_argtype()) ARRAY_CACHE[key] = cls return cls diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py --- a/lib_pypy/_ctypes/basics.py +++ b/lib_pypy/_ctypes/basics.py @@ -49,10 +49,13 @@ else: return self.from_param(as_parameter) + def _build_ffiargtype(self): + return _shape_to_ffi_type(self._ffiargshape_) + def get_ffi_argtype(self): if self._ffiargtype: return self._ffiargtype - self._ffiargtype = _shape_to_ffi_type(self._ffiargshape_) + self._ffiargtype = self._build_ffiargtype() return self._ffiargtype def _CData_output(self, resbuffer, base=None, index=-1): diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py --- a/lib_pypy/_ctypes/pointer.py +++ b/lib_pypy/_ctypes/pointer.py @@ -70,7 +70,12 @@ self._ffiarray = ffiarray self.__init__ = __init__ self._type_ = TP - self._ffiargtype = _ffi.types.Pointer(TP.get_ffi_argtype()) + + def _build_ffiargtype(self): + return _ffi.types.Pointer(self._type_.get_ffi_argtype()) + + def _deref_ffiargtype(self): + return self._type_.get_ffi_argtype() from_address = cdata_from_address diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py --- a/lib_pypy/_ctypes/structure.py +++ b/lib_pypy/_ctypes/structure.py @@ -160,6 +160,10 @@ raise AttributeError("_fields_ is final") if self in [f[1] for f in value]: raise AttributeError("Structure or union cannot contain itself") + if self._ffiargtype is not None: + raise NotImplementedError("Too late to set _fields_: we already " + "said to libffi that the structure type %s is opaque" + % (self,)) names_and_fields( self, value, self.__bases__[0], diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst --- a/pypy/doc/gc_info.rst +++ b/pypy/doc/gc_info.rst @@ -152,7 +152,7 @@ to wait until it reaches a point in which the interpreter is in a known state and calling user-defined code is harmless. It might happen that multiple events occur before the hook is invoked: in this case, you can inspect the -value ``stats.count`` to know how many times the event occured since the last +value ``stats.count`` to know how many times the event occurred since the last time the hook was called. Similarly, ``stats.duration`` contains the **total** time spent by the GC for this specific event since the last time the hook was called. @@ -163,7 +163,7 @@ The attributes for ``GcMinorStats`` are: ``count`` - The number of minor collections occured since the last hook call. + The number of minor collections occurred since the last hook call. ``duration`` The total time spent inside minor collections since the last hook diff --git a/pypy/doc/release-v6.0.0.rst b/pypy/doc/release-v6.0.0.rst --- a/pypy/doc/release-v6.0.0.rst +++ b/pypy/doc/release-v6.0.0.rst @@ -8,13 +8,18 @@ the dual release. This release is a feature release following our previous 5.10 incremental -release in late December 2017. Our C-API compatability layer ``cpyext`` is +release in late December 2017. Our C-API compatibility layer ``cpyext`` is now much faster (see the `blog post`_) as well as more complete. We have made many other improvements in speed and CPython compatibility. Since the changes affect the included python development header files, all c-extension modules must be recompiled for this version. -First-time python users are often stumped by silly typos and emissions when +Until we can work with downstream providers to distribute builds with PyPy, we +have made packages for some common packages `available as wheels`_. You may +compile yourself using ``pip install --no-build-isolation ``, the +``no-build-isolation`` is currently needed for pip v10. + +First-time python users are often stumped by silly typos and omissions when getting started writing code. We have improved our parser to emit more friendly `syntax errors`_, making PyPy not only faster but more friendly. @@ -60,6 +65,7 @@ .. _`hooks`: gc_info.html#gc-hooks .. _`cffi`: http://cffi.readthedocs.io .. _`cppyy`: https://cppyy.readthedocs.io +.. _`available as wheels`: https://github.com/antocuni/pypy-wheels What is PyPy? ============= @@ -110,7 +116,7 @@ * Fix JIT bugs exposed in the sre module * Improve speed of Python parser, improve ParseError messages and SyntaxError * Handle JIT hooks more efficiently -* Fix a rare GC bug exposed by intensive use of cpyext `Buffer` s +* Fix a rare GC bug exposed by intensive use of cpyext ``Buffer`` s We also refactored many parts of the JIT bridge optimizations, as well as cpyext internals, and together with new contributors fixed issues, added new diff --git a/pypy/module/__builtin__/operation.py b/pypy/module/__builtin__/operation.py --- a/pypy/module/__builtin__/operation.py +++ b/pypy/module/__builtin__/operation.py @@ -137,7 +137,7 @@ """iter(collection) -> iterator over the elements of the collection. iter(callable, sentinel) -> iterator calling callable() until it returns - the sentinal. + the sentinel. """ if w_sentinel is None: return space.iter(w_collection_or_callable) diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -556,7 +556,7 @@ to the default encoding. errors may be given to set a different error handling scheme. Default is 'strict' meaning that encoding errors raise a ValueError. Other possible values are 'ignore' and 'replace' - as well as any other name registerd with codecs.register_error that is + as well as any other name registered with codecs.register_error that is able to handle ValueErrors. """ if w_encoding is None: diff --git a/pypy/module/_cppyy/test/test_datatypes.py b/pypy/module/_cppyy/test/test_datatypes.py --- a/pypy/module/_cppyy/test/test_datatypes.py +++ b/pypy/module/_cppyy/test/test_datatypes.py @@ -476,7 +476,7 @@ assert c.get_valid_string('aap') == 'aap' #assert c.get_invalid_string() == '' - def test12_copy_contructor(self): + def test12_copy_constructor(self): """Test copy constructor""" import _cppyy as cppyy diff --git a/pypy/module/itertools/interp_itertools.py b/pypy/module/itertools/interp_itertools.py --- a/pypy/module/itertools/interp_itertools.py +++ b/pypy/module/itertools/interp_itertools.py @@ -538,7 +538,7 @@ def chain_from_iterable(space, w_cls, w_arg): """chain.from_iterable(iterable) --> chain object - Alternate chain() contructor taking a single iterable argument + Alternate chain() constructor taking a single iterable argument that evaluates lazily.""" r = space.allocate_instance(W_Chain, w_cls) r.__init__(space, space.iter(w_arg)) diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -820,7 +820,7 @@ if (axis1 < 0 or axis2 < 0 or axis1 >= self.ndims() or axis2 >= self.ndims()): raise oefmt(space.w_ValueError, - "axis1(=%d) and axis2(=%d) must be withing range " + "axis1(=%d) and axis2(=%d) must be within range " "(ndim=%d)", axis1, axis2, self.ndims()) if axis1 == axis2: raise oefmt(space.w_ValueError, diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py b/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py @@ -62,7 +62,7 @@ assert t(h).value == h def test_typeerror(self): - # Only numbers are allowed in the contructor, + # Only numbers are allowed in the constructor, # otherwise TypeError is raised for t in signed_types + unsigned_types + float_types: with pytest.raises(TypeError): diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py b/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py @@ -272,3 +272,18 @@ base = cast(d, c_void_p).value for i in [0, 1, 4, 1444, -10293]: assert cast(byref(c, i), c_void_p).value == base + i + + def test_issue2813_fix(self): + class C(Structure): + pass + POINTER(C) + C._fields_ = [('x', c_int)] + ffitype = C.get_ffi_argtype() + assert C.get_ffi_argtype() is ffitype + assert ffitype.sizeof() == sizeof(c_int) + + def test_issue2813_cant_change_fields_after_get_ffi_argtype(self): + class C(Structure): + pass + ffitype = C.get_ffi_argtype() + raises(NotImplementedError, "C._fields_ = [('x', c_int)]") diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -63,7 +63,7 @@ def remember_known_length(self, op, val): self._known_lengths[op] = val - def remember_setarrayitem_occured(self, op, index): + def remember_setarrayitem_occurred(self, op, index): op = self.get_box_replacement(op) try: subs = self._setarrayitems_occurred[op] @@ -456,7 +456,7 @@ array_box = op.getarg(0) index_box = op.getarg(1) if not isinstance(array_box, ConstPtr) and index_box.is_constant(): - self.remember_setarrayitem_occured(array_box, index_box.getint()) + self.remember_setarrayitem_occurred(array_box, index_box.getint()) def clear_varsize_gc_fields(self, kind, descr, result, v_length, opnum): if self.gc_ll_descr.malloc_zero_filled: diff --git a/rpython/jit/backend/zarch/opassembler.py b/rpython/jit/backend/zarch/opassembler.py --- a/rpython/jit/backend/zarch/opassembler.py +++ b/rpython/jit/backend/zarch/opassembler.py @@ -74,7 +74,7 @@ mc.MLGR(lr, l1) mc.LGHI(r.SCRATCH, l.imm(-1)) mc.RISBG(r.SCRATCH, r.SCRATCH, l.imm(0), l.imm(0x80 | 0), l.imm(0)) - # is the value greater than 2**63 ? then an overflow occured + # is the value greater than 2**63 ? then an overflow occurred jmp_xor_lq_overflow = mc.get_relative_pos() mc.reserve_cond_jump() # CLGRJ lq > 0x8000 ... 00 -> (label_overflow) jmp_xor_lr_overflow = mc.get_relative_pos() diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.c b/rpython/rlib/rvmprof/src/shared/vmp_stack.c --- a/rpython/rlib/rvmprof/src/shared/vmp_stack.c +++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.c @@ -214,7 +214,7 @@ ret = unw_getcontext(&uc); if (ret < 0) { // could not initialize lib unwind cursor and context - fprintf(stderr, "WARNING: unw_getcontext did not retreive context, switching to python profiling mode \n"); + fprintf(stderr, "WARNING: unw_getcontext did not retrieve context, switching to python profiling mode \n"); vmp_native_disable(); return vmp_walk_and_record_python_stack_only(frame, result, max_depth, 0, pc); } diff --git a/rpython/rtyper/rpbc.py b/rpython/rtyper/rpbc.py --- a/rpython/rtyper/rpbc.py +++ b/rpython/rtyper/rpbc.py @@ -999,7 +999,7 @@ classdef.has_no_attrs()): # special case for instanciating simple built-in # exceptions: always return the same prebuilt instance, - # and ignore any arguments passed to the contructor. + # and ignore any arguments passed to the constructor. r_instance = rclass.getinstancerepr(hop.rtyper, classdef) example = r_instance.get_reusable_prebuilt_instance() hop.exception_cannot_occur() diff --git a/rpython/translator/c/extfunc.py b/rpython/translator/c/extfunc.py --- a/rpython/translator/c/extfunc.py +++ b/rpython/translator/c/extfunc.py @@ -17,7 +17,7 @@ yield ('RPYTHON_EXCEPTION_MATCH', exceptiondata.fn_exception_match) yield ('RPYTHON_TYPE_OF_EXC_INST', exceptiondata.fn_type_of_exc_inst) - yield ('RPyExceptionOccurred1', exctransformer.rpyexc_occured_ptr.value) + yield ('RPyExceptionOccurred1', exctransformer.rpyexc_occurred_ptr.value) yield ('RPyFetchExceptionType', exctransformer.rpyexc_fetch_type_ptr.value) yield ('RPyFetchExceptionValue', exctransformer.rpyexc_fetch_value_ptr.value) yield ('RPyClearException', exctransformer.rpyexc_clear_ptr.value) diff --git a/rpython/translator/exceptiontransform.py b/rpython/translator/exceptiontransform.py --- a/rpython/translator/exceptiontransform.py +++ b/rpython/translator/exceptiontransform.py @@ -66,7 +66,7 @@ assertion_error_ll_exc_type) self.c_n_i_error_ll_exc_type = constant_value(n_i_error_ll_exc_type) - def rpyexc_occured(): + def rpyexc_occurred(): exc_type = exc_data.exc_type return bool(exc_type) @@ -109,9 +109,9 @@ exc_data.exc_type = ll_inst_type(evalue) exc_data.exc_value = evalue - self.rpyexc_occured_ptr = self.build_func( + self.rpyexc_occurred_ptr = self.build_func( "RPyExceptionOccurred", - rpyexc_occured, + rpyexc_occurred, [], lltype.Bool) self.rpyexc_fetch_type_ptr = self.build_func( From pypy.commits at gmail.com Mon Apr 30 12:28:07 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 30 Apr 2018 09:28:07 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8: fix merge Message-ID: <5ae74417.1c69fb81.12c0.d013@mx.google.com> Author: Matti Picus Branch: unicode-utf8 Changeset: r94459:ab5ac9802e14 Date: 2018-04-29 23:28 +0300 http://bitbucket.org/pypy/pypy/changeset/ab5ac9802e14/ Log: fix merge diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py --- a/pypy/module/_sre/interp_sre.py +++ b/pypy/module/_sre/interp_sre.py @@ -133,7 +133,7 @@ endbytepos = rutf8.codepoint_position_at_index(utf8str, index_storage, endpos) ctx = rsre_utf8.Utf8MatchContext( - self.code, utf8str, bytepos, endbytepos, self.flags) + utf8str, bytepos, endbytepos, self.flags) # xxx we store the w_string on the ctx too, for # W_SRE_Match.bytepos_to_charindex() ctx.w_unicode_obj = w_unicode_obj @@ -159,14 +159,14 @@ def fresh_copy(self, ctx): if isinstance(ctx, rsre_utf8.Utf8MatchContext): result = rsre_utf8.Utf8MatchContext( - ctx.pattern, ctx._utf8, ctx.match_start, ctx.end, ctx.flags) + ctx._utf8, ctx.match_start, ctx.end, ctx.flags) result.w_unicode_obj = ctx.w_unicode_obj elif isinstance(ctx, rsre_core.StrMatchContext): result = self._make_str_match_context( ctx._string, ctx.match_start, ctx.end) elif isinstance(ctx, rsre_core.BufMatchContext): result = rsre_core.BufMatchContext( - ctx.pattern, ctx._buffer, ctx.match_start, ctx.end, ctx.flags) + ctx._buffer, ctx.match_start, ctx.end, ctx.flags) else: raise AssertionError("bad ctx type") result.match_end = ctx.match_end @@ -174,7 +174,7 @@ def _make_str_match_context(self, str, pos, endpos): # for tests to override - return rsre_core.StrMatchContext(self.code, str, + return rsre_core.StrMatchContext(str, pos, endpos, self.flags) def getmatch(self, ctx, found): @@ -319,7 +319,7 @@ n = 0 last_pos = ctx.ZERO while not count or n < count: - pattern = ctx.pattern + pattern = self.code sub_jitdriver.jit_merge_point( self=self, use_builder=use_builder, diff --git a/pypy/module/_sre/test/test_app_sre.py b/pypy/module/_sre/test/test_app_sre.py --- a/pypy/module/_sre/test/test_app_sre.py +++ b/pypy/module/_sre/test/test_app_sre.py @@ -32,7 +32,7 @@ start = support.Position(start) if not isinstance(end, support.Position): end = support.Position(end) - return support.MatchContextForTests(self.code, str, start, end, self.flags) + return support.MatchContextForTests(str, start, end, self.flags) def _bytepos_to_charindex(self, bytepos): if isinstance(self.ctx, support.MatchContextForTests): diff --git a/rpython/rlib/rsre/rsre_core.py b/rpython/rlib/rsre/rsre_core.py --- a/rpython/rlib/rsre/rsre_core.py +++ b/rpython/rlib/rsre/rsre_core.py @@ -55,6 +55,8 @@ specific subclass, calling 'func' is a direct call; if 'ctx' is only known to be of class AbstractMatchContext, calling 'func' is an indirect call. """ + from rpython.rlib.rsre.rsre_utf8 import Utf8MatchContext + assert func.func_code.co_varnames[0] == 'ctx' specname = '_spec_' + func.func_name while specname in _seen_specname: @@ -65,7 +67,9 @@ specialized_methods = [] for prefix, concreteclass in [('buf', BufMatchContext), ('str', StrMatchContext), - ('uni', UnicodeMatchContext)]: + ('uni', UnicodeMatchContext), + ('utf8', Utf8MatchContext), + ]: newfunc = func_with_new_name(func, prefix + specname) assert not hasattr(concreteclass, specname) setattr(concreteclass, specname, newfunc) @@ -83,6 +87,8 @@ def __init__(self, msg): self.msg = msg +class EndOfString(Exception): + pass class CompiledPattern(object): _immutable_fields_ = ['pattern[*]'] @@ -142,6 +148,46 @@ """Similar to str().""" raise NotImplementedError + # The following methods are provided to be overriden in + # Utf8MatchContext. The non-utf8 implementation is provided + # by the FixedMatchContext abstract subclass, in order to use + # the same @not_rpython safety trick as above. + ZERO = 0 + @not_rpython + def next(self, position): + raise NotImplementedError + @not_rpython + def prev(self, position): + raise NotImplementedError + @not_rpython + def next_n(self, position, n): + raise NotImplementedError + @not_rpython + def prev_n(self, position, n, start_position): + raise NotImplementedError + @not_rpython + def debug_check_pos(self, position): + raise NotImplementedError + @not_rpython + def maximum_distance(self, position_low, position_high): + raise NotImplementedError + @not_rpython + def get_single_byte(self, base_position, index): + raise NotImplementedError + + def bytes_difference(self, position1, position2): + return position1 - position2 + def go_forward_by_bytes(self, base_position, index): + return base_position + index + def next_indirect(self, position): + assert position < self.end + return position + 1 # like next(), but can be called indirectly + def prev_indirect(self, position): + position -= 1 # like prev(), but can be called indirectly + if position < 0: + raise EndOfString + return position + def get_mark(self, gid): return find_mark(self.match_marks, gid) @@ -185,13 +231,40 @@ def fresh_copy(self, start): raise NotImplementedError -class BufMatchContext(AbstractMatchContext): +class FixedMatchContext(AbstractMatchContext): + """Abstract subclass to introduce the default implementation for + these position methods. The Utf8MatchContext subclass doesn't + inherit from here.""" + + next = AbstractMatchContext.next_indirect + prev = AbstractMatchContext.prev_indirect + + def next_n(self, position, n, end_position): + position += n + if position > end_position: + raise EndOfString + return position + + def prev_n(self, position, n, start_position): + position -= n + if position < start_position: + raise EndOfString + return position + + def debug_check_pos(self, position): + pass + + def maximum_distance(self, position_low, position_high): + return position_high - position_low + + +class BufMatchContext(FixedMatchContext): """Concrete subclass for matching in a buffer.""" _immutable_fields_ = ["_buffer"] def __init__(self, buf, match_start, end, flags): - AbstractMatchContext.__init__(self, match_start, end, flags) + FixedMatchContext.__init__(self, match_start, end, flags) self._buffer = buf def str(self, index): @@ -206,13 +279,17 @@ return BufMatchContext(self._buffer, start, self.end, self.flags) -class StrMatchContext(AbstractMatchContext): + def get_single_byte(self, base_position, index): + return self.str(base_position + index) + + +class StrMatchContext(FixedMatchContext): """Concrete subclass for matching in a plain string.""" _immutable_fields_ = ["_string"] def __init__(self, string, match_start, end, flags): - AbstractMatchContext.__init__(self, match_start, end, flags) + FixedMatchContext.__init__(self, match_start, end, flags) self._string = string if not we_are_translated() and isinstance(string, unicode): self.flags |= rsre_char.SRE_FLAG_UNICODE # for rsre_re.py @@ -229,13 +306,20 @@ return StrMatchContext(self._string, start, self.end, self.flags) -class UnicodeMatchContext(AbstractMatchContext): + def get_single_byte(self, base_position, index): + return self.str(base_position + index) + + def _real_pos(self, index): + return index # overridden by tests + + +class UnicodeMatchContext(FixedMatchContext): """Concrete subclass for matching in a unicode string.""" _immutable_fields_ = ["_unicodestr"] def __init__(self, unicodestr, match_start, end, flags): - AbstractMatchContext.__init__(self, match_start, end, flags) + FixedMatchContext.__init__(self, match_start, end, flags) self._unicodestr = unicodestr def str(self, index): @@ -250,6 +334,9 @@ return UnicodeMatchContext(self._unicodestr, start, self.end, self.flags) + def get_single_byte(self, base_position, index): + return self.str(base_position + index) + # ____________________________________________________________ class Mark(object): @@ -325,7 +412,10 @@ self=self, ptr=ptr, ctx=ctx, nextppos=nextppos, pattern=pattern) result = sre_match(ctx, pattern, nextppos, ptr, self.start_marks) - ptr -= 1 + try: + ptr = ctx.prev_indirect(ptr) + except EndOfString: + ptr = -1 if result is not None: self.subresult = result self.start_ptr = ptr @@ -336,32 +426,35 @@ class MinRepeatOneMatchResult(MatchResult): install_jitdriver('MinRepeatOne', greens=['nextppos', 'ppos3', 'pattern'], - reds=['ptr', 'self', 'ctx'], + reds=['max_count', 'ptr', 'self', 'ctx'], debugprint=(2, 0)) # indices in 'greens' - def __init__(self, nextppos, ppos3, maxptr, ptr, marks): + def __init__(self, nextppos, ppos3, max_count, ptr, marks): self.nextppos = nextppos self.ppos3 = ppos3 - self.maxptr = maxptr + self.max_count = max_count self.start_ptr = ptr self.start_marks = marks def find_first_result(self, ctx, pattern): ptr = self.start_ptr nextppos = self.nextppos + max_count = self.max_count ppos3 = self.ppos3 - while ptr <= self.maxptr: + while max_count >= 0: ctx.jitdriver_MinRepeatOne.jit_merge_point( self=self, ptr=ptr, ctx=ctx, nextppos=nextppos, ppos3=ppos3, - pattern=pattern) + max_count=max_count, pattern=pattern) result = sre_match(ctx, pattern, nextppos, ptr, self.start_marks) if result is not None: self.subresult = result self.start_ptr = ptr + self.max_count = max_count return self if not self.next_char_ok(ctx, pattern, ptr, ppos3): break - ptr += 1 + ptr = ctx.next_indirect(ptr) + max_count -= 1 def find_next_result(self, ctx, pattern): ptr = self.start_ptr @@ -440,12 +533,12 @@ min = pattern.pat(ppos+1) if enum is not None: # matched one more 'item'. record it and continue. - last_match_length = ctx.match_end - ptr + last_match_zero_length = (ctx.match_end == ptr) self.pending = Pending(ptr, marks, enum, self.pending) self.num_pending += 1 ptr = ctx.match_end marks = ctx.match_marks - if last_match_length == 0 and self.num_pending >= min: + if last_match_zero_length and self.num_pending >= min: # zero-width protection: after an empty match, if there # are enough matches, don't try to match more. Instead, # fall through to trying to match 'tail'. @@ -561,22 +654,25 @@ # if ptr >= ctx.end or rsre_char.is_linebreak(ctx.str(ptr)): return - ptr += 1 + ptr = ctx.next(ptr) elif op == OPCODE_ANY_ALL: # match anything # if ptr >= ctx.end: return - ptr += 1 + ptr = ctx.next(ptr) elif op == OPCODE_ASSERT: # assert subpattern # <0=skip> <1=back> - ptr1 = ptr - pattern.pat(ppos+1) + try: + ptr1 = ctx.prev_n(ptr, pattern.pat(ppos+1), ctx.ZERO) + except EndOfString: + return saved = ctx.fullmatch_only ctx.fullmatch_only = False - stop = ptr1 < 0 or sre_match(ctx, pattern, ppos + 2, ptr1, marks) is None + stop = sre_match(ctx, pattern, ppos + 2, ptr1, marks) is None ctx.fullmatch_only = saved if stop: return @@ -586,14 +682,18 @@ elif op == OPCODE_ASSERT_NOT: # assert not subpattern # <0=skip> <1=back> - ptr1 = ptr - pattern.pat(ppos+1) - saved = ctx.fullmatch_only - ctx.fullmatch_only = False - stop = (ptr1 >= 0 and sre_match(ctx, pattern, ppos + 2, ptr1, marks) - is not None) - ctx.fullmatch_only = saved - if stop: - return + + try: + ptr1 = ctx.prev_n(ptr, pattern.pat(ppos+1), ctx.ZERO) + except EndOfString: + pass + else: + saved = ctx.fullmatch_only + ctx.fullmatch_only = False + stop = sre_match(ctx, pattern, ppos + 2, ptr1, marks) is not None + ctx.fullmatch_only = saved + if stop: + return ppos += pattern.pat(ppos) elif op == OPCODE_AT: @@ -616,36 +716,36 @@ if (ptr == ctx.end or not rsre_char.category_dispatch(pattern.pat(ppos), ctx.str(ptr))): return - ptr += 1 + ptr = ctx.next(ptr) ppos += 1 elif op == OPCODE_GROUPREF: # match backreference # - startptr, length = get_group_ref(marks, pattern.pat(ppos)) - if length < 0: + startptr, length_bytes = get_group_ref(ctx, marks, pattern.pat(ppos)) + if length_bytes < 0: return # group was not previously defined - if not match_repeated(ctx, ptr, startptr, length): + if not match_repeated(ctx, ptr, startptr, length_bytes): return # no match - ptr += length + ptr = ctx.go_forward_by_bytes(ptr, length_bytes) ppos += 1 elif op == OPCODE_GROUPREF_IGNORE: # match backreference # - startptr, length = get_group_ref(marks, pattern.pat(ppos)) - if length < 0: + startptr, length_bytes = get_group_ref(ctx, marks, pattern.pat(ppos)) + if length_bytes < 0: return # group was not previously defined - if not match_repeated_ignore(ctx, ptr, startptr, length): + if not match_repeated_ignore(ctx, ptr, startptr, length_bytes): return # no match - ptr += length + ptr = ctx.go_forward_by_bytes(ptr, length_bytes) ppos += 1 elif op == OPCODE_GROUPREF_EXISTS: # conditional match depending on the existence of a group # codeyes codeno ... - _, length = get_group_ref(marks, pattern.pat(ppos)) - if length >= 0: + _, length_bytes = get_group_ref(ctx, marks, pattern.pat(ppos)) + if length_bytes >= 0: ppos += 2 # jump to 'codeyes' else: ppos += pattern.pat(ppos+1) # jump to 'codeno' @@ -657,7 +757,7 @@ ctx.str(ptr)): return ppos += pattern.pat(ppos) - ptr += 1 + ptr = ctx.next(ptr) elif op == OPCODE_IN_IGNORE: # match set member (or non_member), ignoring case @@ -666,12 +766,12 @@ ctx.lowstr(ptr)): return ppos += pattern.pat(ppos) - ptr += 1 + ptr = ctx.next(ptr) elif op == OPCODE_INFO: # optimization info block # <0=skip> <1=flags> <2=min> ... - if (ctx.end - ptr) < pattern.pat(ppos+2): + if ctx.maximum_distance(ptr, ctx.end) < pattern.pat(ppos+2): return ppos += pattern.pat(ppos) @@ -684,7 +784,7 @@ if ptr >= ctx.end or ctx.str(ptr) != pattern.pat(ppos): return ppos += 1 - ptr += 1 + ptr = ctx.next(ptr) elif op == OPCODE_LITERAL_IGNORE: # match literal string, ignoring case @@ -692,7 +792,7 @@ if ptr >= ctx.end or ctx.lowstr(ptr) != pattern.pat(ppos): return ppos += 1 - ptr += 1 + ptr = ctx.next(ptr) elif op == OPCODE_MARK: # set mark @@ -707,7 +807,7 @@ if ptr >= ctx.end or ctx.str(ptr) == pattern.pat(ppos): return ppos += 1 - ptr += 1 + ptr = ctx.next(ptr) elif op == OPCODE_NOT_LITERAL_IGNORE: # match if it's not a literal string, ignoring case @@ -715,7 +815,7 @@ if ptr >= ctx.end or ctx.lowstr(ptr) == pattern.pat(ppos): return ppos += 1 - ptr += 1 + ptr = ctx.next(ptr) elif op == OPCODE_REPEAT: # general repeat. in this version of the re module, all the work @@ -753,8 +853,10 @@ # use the MAX_REPEAT operator. # <1=min> <2=max> item tail start = ptr - minptr = start + pattern.pat(ppos+1) - if minptr > ctx.end: + + try: + minptr = ctx.next_n(start, pattern.pat(ppos+1), ctx.end) + except EndOfString: return # cannot match ptr = find_repetition_end(ctx, pattern, ppos+3, start, pattern.pat(ppos+2), @@ -776,22 +878,22 @@ start = ptr min = pattern.pat(ppos+1) if min > 0: - minptr = ptr + min - if minptr > ctx.end: - return # cannot match + try: + minptr = ctx.next_n(ptr, min, ctx.end) + except EndOfString: + return # cannot match # count using pattern min as the maximum ptr = find_repetition_end(ctx, pattern, ppos+3, ptr, min, marks) if ptr < minptr: return # did not match minimum number of times - maxptr = ctx.end + max_count = sys.maxint max = pattern.pat(ppos+2) if max != rsre_char.MAXREPEAT: - maxptr1 = start + max - if maxptr1 <= maxptr: - maxptr = maxptr1 + max_count = max - min + assert max_count >= 0 nextppos = ppos + pattern.pat(ppos) - result = MinRepeatOneMatchResult(nextppos, ppos+3, maxptr, + result = MinRepeatOneMatchResult(nextppos, ppos+3, max_count, ptr, marks) return result.find_first_result(ctx, pattern) @@ -799,40 +901,43 @@ raise Error("bad pattern code %d" % op) -def get_group_ref(marks, groupnum): +def get_group_ref(ctx, marks, groupnum): gid = groupnum * 2 startptr = find_mark(marks, gid) - if startptr < 0: + if startptr < ctx.ZERO: return 0, -1 endptr = find_mark(marks, gid + 1) - length = endptr - startptr # < 0 if endptr < startptr (or if endptr=-1) - return startptr, length + length_bytes = ctx.bytes_difference(endptr, startptr) + return startptr, length_bytes @specializectx -def match_repeated(ctx, ptr, oldptr, length): - if ptr + length > ctx.end: +def match_repeated(ctx, ptr, oldptr, length_bytes): + if ctx.bytes_difference(ctx.end, ptr) < length_bytes: return False - for i in range(length): - if ctx.str(ptr + i) != ctx.str(oldptr + i): + for i in range(length_bytes): + if ctx.get_single_byte(ptr, i) != ctx.get_single_byte(oldptr, i): return False return True @specializectx -def match_repeated_ignore(ctx, ptr, oldptr, length): - if ptr + length > ctx.end: - return False - for i in range(length): - if ctx.lowstr(ptr + i) != ctx.lowstr(oldptr + i): - return False - return True +def match_repeated_ignore(ctx, ptr, oldptr, length_bytes): + oldend = ctx.go_forward_by_bytes(oldptr, length_bytes) + while oldptr < oldend: + if ptr >= ctx.end: + return -1 + if ctx.lowstr(ptr) != ctx.lowstr(oldptr): + return -1 + ptr = ctx.next(ptr) + oldptr = ctx.next(oldptr) + return ptr @specializectx def find_repetition_end(ctx, pattern, ppos, ptr, maxcount, marks): end = ctx.end - ptrp1 = ptr + 1 # First get rid of the cases where we don't have room for any match. - if maxcount <= 0 or ptrp1 > end: + if maxcount <= 0 or ptr >= end: return ptr + ptrp1 = ctx.next(ptr) # Check the first character directly. If it doesn't match, we are done. # The idea is to be fast for cases like re.search("b+"), where we expect # the common case to be a non-match. It's much faster with the JIT to @@ -854,9 +959,10 @@ # Else we really need to count how many times it matches. if maxcount != rsre_char.MAXREPEAT: # adjust end - end1 = ptr + maxcount - if end1 <= end: - end = end1 + try: + end = ctx.next_n(ptr, maxcount, end) + except EndOfString: + pass op = pattern.pat(ppos) for op1, fre in unroll_fre_checker: if op1 == op: @@ -873,7 +979,7 @@ if end1 <= end: end = end1 while ptr < end and sre_match(ctx, patern, ppos, ptr, marks) is not None: - ptr += 1 + ptr = ctx.next(ptr) return ptr @specializectx @@ -916,7 +1022,7 @@ end=end, ppos=ppos, pattern=pattern) if ptr < end and checkerfn(ctx, pattern, ptr, ppos): - ptr += 1 + ptr = ctx.next(ptr) else: return ptr elif checkerfn == match_IN_IGNORE: @@ -931,7 +1037,7 @@ end=end, ppos=ppos, pattern=pattern) if ptr < end and checkerfn(ctx, pattern, ptr, ppos): - ptr += 1 + ptr = ctx.next(ptr) else: return ptr else: @@ -940,7 +1046,7 @@ @specializectx def fre(ctx, pattern, ptr, end, ppos): while ptr < end and checkerfn(ctx, pattern, ptr, ppos): - ptr += 1 + ptr = ctx.next(ptr) return ptr fre = func_with_new_name(fre, 'fre_' + checkerfn.__name__) return fre @@ -980,11 +1086,14 @@ def sre_at(ctx, atcode, ptr): if (atcode == AT_BEGINNING or atcode == AT_BEGINNING_STRING): - return ptr == 0 + return ptr == ctx.ZERO elif atcode == AT_BEGINNING_LINE: - prevptr = ptr - 1 - return prevptr < 0 or rsre_char.is_linebreak(ctx.str(prevptr)) + try: + prevptr = ctx.prev(ptr) + except EndOfString: + return True + return rsre_char.is_linebreak(ctx.str(prevptr)) elif atcode == AT_BOUNDARY: return at_boundary(ctx, ptr) @@ -993,9 +1102,8 @@ return at_non_boundary(ctx, ptr) elif atcode == AT_END: - remaining_chars = ctx.end - ptr - return remaining_chars <= 0 or ( - remaining_chars == 1 and rsre_char.is_linebreak(ctx.str(ptr))) + return (ptr == ctx.end or + (ctx.next(ptr) == ctx.end and rsre_char.is_linebreak(ctx.str(ptr)))) elif atcode == AT_END_LINE: return ptr == ctx.end or rsre_char.is_linebreak(ctx.str(ptr)) @@ -1020,18 +1128,26 @@ def _make_boundary(word_checker): @specializectx def at_boundary(ctx, ptr): - if ctx.end == 0: + if ctx.end == ctx.ZERO: return False - prevptr = ptr - 1 - that = prevptr >= 0 and word_checker(ctx.str(prevptr)) + try: + prevptr = ctx.prev(ptr) + except EndOfString: + that = False + else: + that = word_checker(ctx.str(prevptr)) this = ptr < ctx.end and word_checker(ctx.str(ptr)) return this != that @specializectx def at_non_boundary(ctx, ptr): - if ctx.end == 0: + if ctx.end == ctx.ZERO: return False - prevptr = ptr - 1 - that = prevptr >= 0 and word_checker(ctx.str(prevptr)) + try: + prevptr = ctx.prev(ptr) + except EndOfString: + that = False + else: + that = word_checker(ctx.str(prevptr)) this = ptr < ctx.end and word_checker(ctx.str(ptr)) return this == that return at_boundary, at_non_boundary @@ -1109,13 +1225,15 @@ def regular_search(ctx, pattern, base): start = ctx.match_start - while start <= ctx.end: - ctx.jitdriver_RegularSearch.jit_merge_point(ctx=ctx, start=start, - base=base, pattern=pattern) + while True: + ctx.jitdriver_RegularSearch.jit_merge_point(ctx=ctx, pattern=pattern, + start=start, base=base) if sre_match(ctx, pattern, base, start, None) is not None: ctx.match_start = start return True - start += 1 + if start >= ctx.end: + break + start = ctx.next_indirect(start) return False install_jitdriver_spec("LiteralSearch", @@ -1132,11 +1250,12 @@ while start < ctx.end: ctx.jitdriver_LiteralSearch.jit_merge_point(ctx=ctx, start=start, base=base, character=character, pattern=pattern) + start1 = ctx.next(start) if ctx.str(start) == character: - if sre_match(ctx, pattern, base, start + 1, None) is not None: + if sre_match(ctx, pattern, base, start1, None) is not None: ctx.match_start = start return True - start += 1 + start = start1 return False install_jitdriver_spec("CharsetSearch", @@ -1154,7 +1273,7 @@ if sre_match(ctx, pattern, base, start, None) is not None: ctx.match_start = start return True - start += 1 + start = ctx.next(start) return False install_jitdriver_spec('FastSearch', @@ -1186,11 +1305,14 @@ else: i += 1 if i == prefix_len: - # found a potential match - start = string_position + 1 - prefix_len - assert start >= 0 + # start = string_position + 1 - prefix_len: computed later + ptr = string_position prefix_skip = pattern.pat(6) - ptr = start + prefix_skip + if prefix_skip == prefix_len: + ptr = ctx.next(ptr) + else: + assert prefix_skip < prefix_len + ptr = ctx.prev_n(ptr, prefix_len-1 - prefix_skip, ctx.ZERO) #flags = pattern.pat(2) #if flags & rsre_char.SRE_INFO_LITERAL: # # matched all of pure literal pattern @@ -1201,10 +1323,11 @@ pattern_offset = pattern.pat(1) + 1 ppos_start = pattern_offset + 2 * prefix_skip if sre_match(ctx, pattern, ppos_start, ptr, None) is not None: + start = ctx.prev_n(ptr, prefix_skip, ctx.ZERO) ctx.match_start = start return True overlap_offset = prefix_len + (7 - 1) i = pattern.pat(overlap_offset + i) - string_position += 1 + string_position = ctx.next(string_position) if string_position >= ctx.end: return False diff --git a/rpython/rlib/rsre/rsre_utf8.py b/rpython/rlib/rsre/rsre_utf8.py --- a/rpython/rlib/rsre/rsre_utf8.py +++ b/rpython/rlib/rsre/rsre_utf8.py @@ -12,8 +12,8 @@ by this class are expressed in *bytes*, not in characters. """ - def __init__(self, pattern, utf8string, match_start, end, flags): - AbstractMatchContext.__init__(self, pattern, match_start, end, flags) + def __init__(self, utf8string, match_start, end, flags): + AbstractMatchContext.__init__(self, match_start, end, flags) self._utf8 = utf8string def str(self, index): From pypy.commits at gmail.com Mon Apr 30 12:28:05 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 30 Apr 2018 09:28:05 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8: merge default into branch Message-ID: <5ae74415.1c69fb81.12c0.d010@mx.google.com> Author: Matti Picus Branch: unicode-utf8 Changeset: r94458:d4baff192be4 Date: 2018-04-29 21:18 +0300 http://bitbucket.org/pypy/pypy/changeset/d4baff192be4/ Log: merge default into branch diff too long, truncating to 2000 out of 11084 lines diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -51,3 +51,5 @@ 0000000000000000000000000000000000000000 release-pypy3.5-v5.10.0 09f9160b643e3f02ccb8c843b2fbb4e5cbf54082 release-pypy3.5-v5.10.0 3f6eaa010fce78cc7973bdc1dfdb95970f08fed2 release-pypy3.5-v5.10.1 +ab0b9caf307db6592905a80b8faffd69b39005b8 release-pypy2.7-v6.0.0 +fdd60ed87e941677e8ea11acf9f1819466521bf2 release-pypy3.5-v6.0.0 diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -6,36 +6,36 @@ Except when otherwise stated (look for LICENSE files in directories or information at the beginning of each file) all software and documentation in the 'rpython', 'pypy', 'ctype_configure', 'dotviewer', 'demo', 'lib_pypy', -'py', and '_pytest' directories is licensed as follows: +'py', and '_pytest' directories is licensed as follows: The MIT License - Permission is hereby granted, free of charge, to any person - obtaining a copy of this software and associated documentation - files (the "Software"), to deal in the Software without - restriction, including without limitation the rights to use, - copy, modify, merge, publish, distribute, sublicense, and/or - sell copies of the Software, and to permit persons to whom the + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation + files (the "Software"), to deal in the Software without + restriction, including without limitation the rights to use, + copy, modify, merge, publish, distribute, sublicense, and/or + sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included + The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. PyPy Copyright holders 2003-2018 ------------------------------------ +-------------------------------- Except when otherwise stated (look for LICENSE files or information at the beginning of each file) the files in the 'pypy' directory are each -copyrighted by one or more of the following people and organizations: +copyrighted by one or more of the following people and organizations: Armin Rigo Maciej Fijalkowski @@ -89,13 +89,13 @@ Niko Matsakis Alexander Hesse Ludovic Aubry + stian Jacob Hallen Jason Creighton Mark Young Alex Martelli Spenser Bauman Michal Bendowski - stian Jan de Mooij Tyler Wade Vincent Legoll @@ -123,10 +123,10 @@ Wenzhu Man Konstantin Lopuhin John Witulski + Jeremy Thurgood Greg Price Ivan Sichmann Freitas Dario Bertini - Jeremy Thurgood Mark Pearse Simon Cross Tobias Pape @@ -145,18 +145,19 @@ Adrian Kuhn tav Georg Brandl + Joannah Nanjekye Bert Freudenberg Stian Andreassen Wanja Saatkamp Mike Blume - Joannah Nanjekye Gerald Klix Oscar Nierstrasz Rami Chowdhury Stefan H. Muller + Dodan Mihai Tim Felgentreff Eugene Oden - Dodan Mihai + Colin Valliant Jeff Terrace Henry Mason Vasily Kuznetsov @@ -225,12 +226,14 @@ Vaibhav Sood Reuben Cummings Attila Gobi + Floris Bruynooghe Christopher Pope Tristan Arthur Christian Tismer Dan Stromberg Carl Meyer Florin Papa + Arianna Avanzini Jens-Uwe Mager Valentina Mukhamedzhanova Stefano Parmesan @@ -244,15 +247,18 @@ Lukas Vacek Omer Katz Jacek Generowicz + Tomasz Dziopa Sylvain Thenault Jakub Stasiak Andrew Dalke Alejandro J. Cura Vladimir Kryachko Gabriel + Thomas Hisch Mark Williams Kunal Grover Nathan Taylor + Barry Hart Travis Francis Athougies Yasir Suhail Sergey Kishchenko @@ -260,6 +266,7 @@ Lutz Paelike Ian Foote Philipp Rustemeuer + Logan Chien Catalin Gabriel Manciu Jacob Oscarson Ryan Gonzalez @@ -295,19 +302,20 @@ Akira Li Gustavo Niemeyer Rafał Gałczyński - Logan Chien Lucas Stadler roberto at goyle Matt Bogosian Yury V. Zaytsev florinpapa Anders Sigfridsson + Matt Jackson Nikolay Zinov rafalgalczynski at gmail.com Joshua Gilbert Anna Katrina Dominguez Kim Jin Su Amber Brown + Miro Hrončok Anthony Sottile Nate Bragg Ben Darnell @@ -315,7 +323,6 @@ Godefroid Chappelle Julian Berman Michael Hudson-Doyle - Floris Bruynooghe Stephan Busemann Dan Colish timo @@ -357,6 +364,7 @@ Michael Chermside Anna Ravencroft remarkablerocket + Pauli Virtanen Petre Vijiac Berker Peksag Christian Muirhead @@ -381,6 +389,7 @@ Graham Markall Dan Loewenherz werat + Andrew Stepanov Niclas Olofsson Chris Pressey Tobias Diaz @@ -395,14 +404,14 @@ m at funkyhat.org Stefan Marr - Heinrich-Heine University, Germany + Heinrich-Heine University, Germany Open End AB (formerly AB Strakt), Sweden - merlinux GmbH, Germany - tismerysoft GmbH, Germany - Logilab Paris, France - DFKI GmbH, Germany + merlinux GmbH, Germany + tismerysoft GmbH, Germany + Logilab Paris, France + DFKI GmbH, Germany Impara, Germany - Change Maker, Sweden + Change Maker, Sweden University of California Berkeley, USA Google Inc. King's College London @@ -410,14 +419,14 @@ The PyPy Logo as used by http://speed.pypy.org and others was created by Samuel Reis and is distributed on terms of Creative Commons Share Alike License. - -License for 'lib-python/2.7' -============================ + +License for 'lib-python/2.7, lib-python/3' +========================================== Except when otherwise stated (look for LICENSE files or copyright/license -information at the beginning of each file) the files in the 'lib-python/2.7' +information at the beginning of each file) the files in the 'lib-python' directory are all copyrighted by the Python Software Foundation and licensed -under the terms that you can find here: https://docs.python.org/2/license.html +under the terms that you can find here: https://docs.python.org/3/license.html License for 'pypy/module/unicodedata/' ====================================== @@ -441,9 +450,9 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/dotviewer/font/NOTICE b/dotviewer/font/COPYING.txt rename from dotviewer/font/NOTICE rename to dotviewer/font/COPYING.txt diff --git a/lib-python/2.7/re.py b/lib-python/2.7/re.py --- a/lib-python/2.7/re.py +++ b/lib-python/2.7/re.py @@ -225,7 +225,7 @@ _pattern_type = type(sre_compile.compile("", 0)) -_MAXCACHE = 100 +_MAXCACHE = 1000 def _compile(*key): # internal: compile pattern diff --git a/lib-python/2.7/test/test_eof.py b/lib-python/2.7/test/test_eof.py --- a/lib-python/2.7/test/test_eof.py +++ b/lib-python/2.7/test/test_eof.py @@ -5,7 +5,7 @@ class EOFTestCase(unittest.TestCase): def test_EOFC(self): - expect = "EOL while scanning string literal (, line 1)" + expect = "end of line (EOL) while scanning string literal (, line 1)" try: eval("""'this is a test\ """) @@ -15,7 +15,7 @@ raise test_support.TestFailed def test_EOFS(self): - expect = ("EOF while scanning triple-quoted string literal " + expect = ("end of file (EOF) while scanning triple-quoted string literal " "(, line 1)") try: eval("""'''this is a test""") diff --git a/lib-python/2.7/test/test_generators.py b/lib-python/2.7/test/test_generators.py --- a/lib-python/2.7/test/test_generators.py +++ b/lib-python/2.7/test/test_generators.py @@ -398,7 +398,10 @@ 0 >>> type(i.gi_frame) ->>> i.gi_running = 42 + +PyPy prints "readonly attribute 'gi_running'" so ignore the exception detail + +>>> i.gi_running = 42 # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ... TypeError: readonly attribute diff --git a/lib-python/2.7/test/test_genexps.py b/lib-python/2.7/test/test_genexps.py --- a/lib-python/2.7/test/test_genexps.py +++ b/lib-python/2.7/test/test_genexps.py @@ -87,7 +87,7 @@ >>> dict(a = i for i in xrange(10)) Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: invalid syntax (expected ')') Verify that parenthesis are required when used as a keyword argument value diff --git a/lib-python/2.7/test/test_traceback.py b/lib-python/2.7/test/test_traceback.py --- a/lib-python/2.7/test/test_traceback.py +++ b/lib-python/2.7/test/test_traceback.py @@ -123,10 +123,7 @@ self.assertEqual(len(err), 4) self.assertEqual(err[1].strip(), "print(2)") self.assertIn("^", err[2]) - if check_impl_detail(): - self.assertEqual(err[1].find("p"), err[2].find("^")) - if check_impl_detail(pypy=True): - self.assertEqual(err[1].find("2)") + 1, err[2].find("^")) + self.assertEqual(err[1].find("p"), err[2].find("^")) def test_base_exception(self): # Test that exceptions derived from BaseException are formatted right diff --git a/lib-python/2.7/threading.py b/lib-python/2.7/threading.py --- a/lib-python/2.7/threading.py +++ b/lib-python/2.7/threading.py @@ -351,6 +351,21 @@ # forward-compatibility reasons we do the same. waiter.acquire() gotit = True + except AttributeError: + # someone patched the 'waiter' class, probably. + # Fall back to the standard CPython logic. + # See the CPython lib for the comments about it... + endtime = _time() + timeout + delay = 0.0005 # 500 us -> initial delay of 1 ms + while True: + gotit = waiter.acquire(0) + if gotit: + break + remaining = endtime - _time() + if remaining <= 0: + break + delay = min(delay * 2, remaining, .05) + _sleep(delay) else: gotit = waiter.acquire(False) if not gotit: diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py --- a/lib_pypy/_ctypes/array.py +++ b/lib_pypy/_ctypes/array.py @@ -82,8 +82,11 @@ def _CData_output(self, resarray, base=None, index=-1): from _rawffi.alt import types # If a char_p or unichar_p is received, skip the string interpretation - if base._ffiargtype != types.Pointer(types.char_p) and \ - base._ffiargtype != types.Pointer(types.unichar_p): + try: + deref = type(base)._deref_ffiargtype() + except AttributeError: + deref = None + if deref != types.char_p and deref != types.unichar_p: # this seems to be a string if we're array of char, surprise! from ctypes import c_char, c_wchar if self._type_ is c_char: @@ -120,6 +123,12 @@ value = self(*value) return _CDataMeta.from_param(self, value) + def _build_ffiargtype(self): + return _ffi.types.Pointer(self._type_.get_ffi_argtype()) + + def _deref_ffiargtype(self): + return self._type_.get_ffi_argtype() + def array_get_slice_params(self, index): if hasattr(self, '_length_'): start, stop, step = index.indices(self._length_) @@ -248,6 +257,5 @@ _type_ = base ) cls = ArrayMeta(name, (Array,), tpdict) - cls._ffiargtype = _ffi.types.Pointer(base.get_ffi_argtype()) ARRAY_CACHE[key] = cls return cls diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py --- a/lib_pypy/_ctypes/basics.py +++ b/lib_pypy/_ctypes/basics.py @@ -49,10 +49,13 @@ else: return self.from_param(as_parameter) + def _build_ffiargtype(self): + return _shape_to_ffi_type(self._ffiargshape_) + def get_ffi_argtype(self): if self._ffiargtype: return self._ffiargtype - self._ffiargtype = _shape_to_ffi_type(self._ffiargshape_) + self._ffiargtype = self._build_ffiargtype() return self._ffiargtype def _CData_output(self, resbuffer, base=None, index=-1): diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py --- a/lib_pypy/_ctypes/pointer.py +++ b/lib_pypy/_ctypes/pointer.py @@ -70,7 +70,12 @@ self._ffiarray = ffiarray self.__init__ = __init__ self._type_ = TP - self._ffiargtype = _ffi.types.Pointer(TP.get_ffi_argtype()) + + def _build_ffiargtype(self): + return _ffi.types.Pointer(self._type_.get_ffi_argtype()) + + def _deref_ffiargtype(self): + return self._type_.get_ffi_argtype() from_address = cdata_from_address diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py --- a/lib_pypy/_ctypes/structure.py +++ b/lib_pypy/_ctypes/structure.py @@ -160,6 +160,10 @@ raise AttributeError("_fields_ is final") if self in [f[1] for f in value]: raise AttributeError("Structure or union cannot contain itself") + if self._ffiargtype is not None: + raise NotImplementedError("Too late to set _fields_: we already " + "said to libffi that the structure type %s is opaque" + % (self,)) names_and_fields( self, value, self.__bases__[0], diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -56,13 +56,13 @@ Niko Matsakis Alexander Hesse Ludovic Aubry + stian Jacob Hallen Jason Creighton Mark Young Alex Martelli Spenser Bauman Michal Bendowski - stian Jan de Mooij Tyler Wade Vincent Legoll @@ -90,10 +90,10 @@ Wenzhu Man Konstantin Lopuhin John Witulski + Jeremy Thurgood Greg Price Ivan Sichmann Freitas Dario Bertini - Jeremy Thurgood Mark Pearse Simon Cross Tobias Pape @@ -112,18 +112,19 @@ Adrian Kuhn tav Georg Brandl + Joannah Nanjekye Bert Freudenberg Stian Andreassen Wanja Saatkamp Mike Blume - Joannah Nanjekye Gerald Klix Oscar Nierstrasz Rami Chowdhury Stefan H. Muller + Dodan Mihai Tim Felgentreff Eugene Oden - Dodan Mihai + Colin Valliant Jeff Terrace Henry Mason Vasily Kuznetsov @@ -192,12 +193,14 @@ Vaibhav Sood Reuben Cummings Attila Gobi + Floris Bruynooghe Christopher Pope Tristan Arthur Christian Tismer Dan Stromberg Carl Meyer Florin Papa + Arianna Avanzini Jens-Uwe Mager Valentina Mukhamedzhanova Stefano Parmesan @@ -211,6 +214,7 @@ Lukas Vacek Omer Katz Jacek Generowicz + Tomasz Dziopa Sylvain Thenault Jakub Stasiak Andrew Dalke @@ -221,6 +225,7 @@ Mark Williams Kunal Grover Nathan Taylor + Barry Hart Travis Francis Athougies Yasir Suhail Sergey Kishchenko @@ -228,6 +233,7 @@ Lutz Paelike Ian Foote Philipp Rustemeuer + Logan Chien Catalin Gabriel Manciu Jacob Oscarson Ryan Gonzalez @@ -263,19 +269,20 @@ Akira Li Gustavo Niemeyer Rafał Gałczyński - Logan Chien Lucas Stadler roberto at goyle Matt Bogosian Yury V. Zaytsev florinpapa Anders Sigfridsson + Matt Jackson Nikolay Zinov rafalgalczynski at gmail.com Joshua Gilbert Anna Katrina Dominguez Kim Jin Su Amber Brown + Miro Hrončok Anthony Sottile Nate Bragg Ben Darnell @@ -283,7 +290,6 @@ Godefroid Chappelle Julian Berman Michael Hudson-Doyle - Floris Bruynooghe Stephan Busemann Dan Colish timo @@ -325,6 +331,7 @@ Michael Chermside Anna Ravencroft remarkablerocket + Pauli Virtanen Petre Vijiac Berker Peksag Christian Muirhead @@ -349,6 +356,7 @@ Graham Markall Dan Loewenherz werat + Andrew Stepanov Niclas Olofsson Chris Pressey Tobias Diaz diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -10,89 +10,6 @@ PyPy. -.. _extension-modules: - -Extension modules ------------------ - -List of extension modules that we support: - -* Supported as built-in modules (in :source:`pypy/module/`): - - __builtin__ - :doc:`__pypy__ <__pypy__-module>` - _ast - _codecs - _collections - :doc:`_continuation ` - :doc:`_ffi ` - _hashlib - _io - _locale - _lsprof - _md5 - :doc:`_minimal_curses ` - _multiprocessing - _random - :doc:`_rawffi ` - _sha - _socket - _sre - _ssl - _warnings - _weakref - _winreg - array - binascii - bz2 - cStringIO - cmath - `cpyext`_ - crypt - errno - exceptions - fcntl - gc - imp - itertools - marshal - math - mmap - operator - parser - posix - pyexpat - select - signal - struct - symbol - sys - termios - thread - time - token - unicodedata - zipimport - zlib - - When translated on Windows, a few Unix-only modules are skipped, - and the following module is built instead: - - _winreg - -* Supported by being rewritten in pure Python (possibly using ``cffi``): - see the :source:`lib_pypy/` directory. Examples of modules that we - support this way: ``ctypes``, ``cPickle``, ``cmath``, ``dbm``, ``datetime``... - Note that some modules are both in there and in the list above; - by default, the built-in module is used (but can be disabled - at translation time). - -The extension modules (i.e. modules written in C, in the standard CPython) -that are neither mentioned above nor in :source:`lib_pypy/` are not available in PyPy. -(You may have a chance to use them anyway with `cpyext`_.) - -.. _cpyext: http://morepypy.blogspot.com/2010/04/using-cpython-extension-modules-with.html - Differences related to garbage collection strategies ---------------------------------------------------- @@ -559,7 +476,96 @@ environment variable. CPython searches for ``vcvarsall.bat`` somewhere **above** that value. +* SyntaxError_ s try harder to give details about the cause of the failure, so + the error messages are not the same as in CPython + + +.. _extension-modules: + +Extension modules +----------------- + +List of extension modules that we support: + +* Supported as built-in modules (in :source:`pypy/module/`): + + __builtin__ + :doc:`__pypy__ <__pypy__-module>` + _ast + _codecs + _collections + :doc:`_continuation ` + :doc:`_ffi ` + _hashlib + _io + _locale + _lsprof + _md5 + :doc:`_minimal_curses ` + _multiprocessing + _random + :doc:`_rawffi ` + _sha + _socket + _sre + _ssl + _warnings + _weakref + _winreg + array + binascii + bz2 + cStringIO + cmath + `cpyext`_ + crypt + errno + exceptions + fcntl + gc + imp + itertools + marshal + math + mmap + operator + parser + posix + pyexpat + select + signal + struct + symbol + sys + termios + thread + time + token + unicodedata + zipimport + zlib + + When translated on Windows, a few Unix-only modules are skipped, + and the following module is built instead: + + _winreg + +* Supported by being rewritten in pure Python (possibly using ``cffi``): + see the :source:`lib_pypy/` directory. Examples of modules that we + support this way: ``ctypes``, ``cPickle``, ``cmath``, ``dbm``, ``datetime``... + Note that some modules are both in there and in the list above; + by default, the built-in module is used (but can be disabled + at translation time). + +The extension modules (i.e. modules written in C, in the standard CPython) +that are neither mentioned above nor in :source:`lib_pypy/` are not available in PyPy. +(You may have a chance to use them anyway with `cpyext`_.) + +.. _cpyext: http://morepypy.blogspot.com/2010/04/using-cpython-extension-modules-with.html + + .. _`is ignored in PyPy`: http://bugs.python.org/issue14621 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/ .. _`issue #2653`: https://bitbucket.org/pypy/pypy/issues/2653/ +.. _SyntaxError: https://morepypy.blogspot.co.il/2018/04/improving-syntaxerror-in-pypy.html diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst --- a/pypy/doc/gc_info.rst +++ b/pypy/doc/gc_info.rst @@ -121,6 +121,166 @@ alive by GC objects, but not accounted in the GC +GC Hooks +-------- + +GC hooks are user-defined functions which are called whenever a specific GC +event occur, and can be used to monitor GC activity and pauses. You can +install the hooks by setting the following attributes: + +``gc.hook.on_gc_minor`` + Called whenever a minor collection occurs. It corresponds to + ``gc-minor`` sections inside ``PYPYLOG``. + +``gc.hook.on_gc_collect_step`` + Called whenever an incremental step of a major collection occurs. It + corresponds to ``gc-collect-step`` sections inside ``PYPYLOG``. + +``gc.hook.on_gc_collect`` + Called after the last incremental step, when a major collection is fully + done. It corresponds to ``gc-collect-done`` sections inside ``PYPYLOG``. + +To uninstall a hook, simply set the corresponding attribute to ``None``. To +install all hooks at once, you can call ``gc.hooks.set(obj)``, which will look +for methods ``on_gc_*`` on ``obj``. To uninstall all the hooks at once, you +can call ``gc.hooks.reset()``. + +The functions called by the hooks receive a single ``stats`` argument, which +contains various statistics about the event. + +Note that PyPy cannot call the hooks immediately after a GC event, but it has +to wait until it reaches a point in which the interpreter is in a known state +and calling user-defined code is harmless. It might happen that multiple +events occur before the hook is invoked: in this case, you can inspect the +value ``stats.count`` to know how many times the event occurred since the last +time the hook was called. Similarly, ``stats.duration`` contains the +**total** time spent by the GC for this specific event since the last time the +hook was called. + +On the other hand, all the other fields of the ``stats`` object are relative +only to the **last** event of the series. + +The attributes for ``GcMinorStats`` are: + +``count`` + The number of minor collections occurred since the last hook call. + +``duration`` + The total time spent inside minor collections since the last hook + call. See below for more information on the unit. + +``duration_min`` + The duration of the fastest minor collection since the last hook call. + +``duration_max`` + The duration of the slowest minor collection since the last hook call. + + ``total_memory_used`` + The amount of memory used at the end of the minor collection, in + bytes. This include the memory used in arenas (for GC-managed memory) and + raw-malloced memory (e.g., the content of numpy arrays). + +``pinned_objects`` + the number of pinned objects. + + +The attributes for ``GcCollectStepStats`` are: + +``count``, ``duration``, ``duration_min``, ``duration_max`` + See above. + +``oldstate``, ``newstate`` + Integers which indicate the state of the GC before and after the step. + +The value of ``oldstate`` and ``newstate`` is one of these constants, defined +inside ``gc.GcCollectStepStats``: ``STATE_SCANNING``, ``STATE_MARKING``, +``STATE_SWEEPING``, ``STATE_FINALIZING``. It is possible to get a string +representation of it by indexing the ``GC_STATS`` tuple. + + +The attributes for ``GcCollectStats`` are: + +``count`` + See above. + +``num_major_collects`` + The total number of major collections which have been done since the + start. Contrarily to ``count``, this is an always-growing counter and it's + not reset between invocations. + +``arenas_count_before``, ``arenas_count_after`` + Number of arenas used before and after the major collection. + +``arenas_bytes`` + Total number of bytes used by GC-managed objects. + +``rawmalloc_bytes_before``, ``rawmalloc_bytes_after`` + Total number of bytes used by raw-malloced objects, before and after the + major collection. + +Note that ``GcCollectStats`` has **not** got a ``duration`` field. This is +because all the GC work is done inside ``gc-collect-step``: +``gc-collect-done`` is used only to give additional stats, but doesn't do any +actual work. + +A note about the ``duration`` field: depending on the architecture and +operating system, PyPy uses different ways to read timestamps, so ``duration`` +is expressed in varying units. It is possible to know which by calling +``__pypy__.debug_get_timestamp_unit()``, which can be one of the following +values: + +``tsc`` + The default on ``x86`` machines: timestamps are expressed in CPU ticks, as + read by the `Time Stamp Counter`_. + +``ns`` + Timestamps are expressed in nanoseconds. + +``QueryPerformanceCounter`` + On Windows, in case for some reason ``tsc`` is not available: timestamps + are read using the win API ``QueryPerformanceCounter()``. + + +Unfortunately, there does not seem to be a reliable standard way for +converting ``tsc`` ticks into nanoseconds, although in practice on modern CPUs +it is enough to divide the ticks by the maximum nominal frequency of the CPU. +For this reason, PyPy gives the raw value, and leaves the job of doing the +conversion to external libraries. + +Here is an example of GC hooks in use:: + + import sys + import gc + + class MyHooks(object): + done = False + + def on_gc_minor(self, stats): + print 'gc-minor: count = %02d, duration = %d' % (stats.count, + stats.duration) + + def on_gc_collect_step(self, stats): + old = gc.GcCollectStepStats.GC_STATES[stats.oldstate] + new = gc.GcCollectStepStats.GC_STATES[stats.newstate] + print 'gc-collect-step: %s --> %s' % (old, new) + print ' count = %02d, duration = %d' % (stats.count, + stats.duration) + + def on_gc_collect(self, stats): + print 'gc-collect-done: count = %02d' % stats.count + self.done = True + + hooks = MyHooks() + gc.hooks.set(hooks) + + # simulate some GC activity + lst = [] + while not hooks.done: + lst = [lst, 1, 2, 3] + + +.. _`Time Stamp Counter`: https://en.wikipedia.org/wiki/Time_Stamp_Counter + .. _minimark-environment-variables: Environment variables diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst --- a/pypy/doc/how-to-release.rst +++ b/pypy/doc/how-to-release.rst @@ -40,6 +40,8 @@ sure things are ported back to the trunk and to the branch as necessary. +* Make sure the RPython builds on the buildbot pass with no failures + * Maybe bump the SOABI number in module/imp/importing. This has many implications, so make sure the PyPy community agrees to the change. diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-v6.0.0.rst release-v5.10.1.rst release-v5.10.0.rst release-v5.9.0.rst diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -7,6 +7,8 @@ .. toctree:: whatsnew-head.rst + whatsnew-pypy2-6.0.0.rst + whatsnew-pypy2-5.10.0.rst whatsnew-pypy2-5.10.0.rst whatsnew-pypy2-5.9.0.rst whatsnew-pypy2-5.8.0.rst diff --git a/pypy/doc/release-v6.0.0.rst b/pypy/doc/release-v6.0.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-v6.0.0.rst @@ -0,0 +1,123 @@ +====================================== +PyPy2.7 and PyPy3.5 v6.0 dual release +====================================== + +The PyPy team is proud to release both PyPy2.7 v6.0 (an interpreter supporting +Python 2.7 syntax), and a PyPy3.5 v6.0 (an interpreter supporting Python +3.5 syntax). The two releases are both based on much the same codebase, thus +the dual release. + +This release is a feature release following our previous 5.10 incremental +release in late December 2017. Our C-API compatibility layer ``cpyext`` is +now much faster (see the `blog post`_) as well as more complete. We have made +many other improvements in speed and CPython compatibility. Since the changes +affect the included python development header files, all c-extension modules must +be recompiled for this version. + +Until we can work with downstream providers to distribute builds with PyPy, we +have made packages for some common packages `available as wheels`_. You may +compile yourself using ``pip install --no-build-isolation ``, the +``no-build-isolation`` is currently needed for pip v10. + +First-time python users are often stumped by silly typos and omissions when +getting started writing code. We have improved our parser to emit more friendly +`syntax errors`_, making PyPy not only faster but more friendly. + +The GC now has `hooks`_ to gain more insights into its performance + +The Matplotlib TkAgg backend now works with PyPy, as do pygame and pygobject_. + +We updated the `cffi`_ module included in PyPy to version 1.11.5, and the +`cppyy`_ backend to 0.6.0. Please use these to wrap your C and C++ code, +respectively, for a JIT friendly experience. + +As always, this release is 100% compatible with the previous one and fixed +several issues and bugs raised by the growing community of PyPy users. +We strongly recommend updating. + +The Windows PyPy3.5 release is still considered beta-quality. There are open +issues with unicode handling especially around system calls and c-extensions. + +The utf8 branch that changes internal representation of unicode to utf8 did not +make it into the release, so there is still more goodness coming. We also +began working on a Python3.6 implementation, help is welcome. + +You can download the v6.0 releases here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project. If PyPy is not quite good enough for your needs, we are available for +direct consulting work. + +We would also like to thank our contributors and encourage new people to join +the project. PyPy has many layers and we need help with all of them: `PyPy`_ +and `RPython`_ documentation improvements, tweaking popular `modules`_ to run +on pypy, or general `help`_ with making RPython's JIT even better. + +.. _`PyPy`: index.html +.. _`RPython`: https://rpython.readthedocs.org +.. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly +.. _`help`: project-ideas.html +.. _`blog post`: https://morepypy.blogspot.it/2017/10/cape-of-good-hope-for-pypy-hello-from.html +.. _pygobject: https://lazka.github.io/posts/2018-04_pypy-pygobject/index.html +.. _`syntax errors`: https://morepypy.blogspot.com/2018/04/improving-syntaxerror-in-pypy.html +.. _`hooks`: gc_info.html#gc-hooks +.. _`cffi`: http://cffi.readthedocs.io +.. _`cppyy`: https://cppyy.readthedocs.io +.. _`available as wheels`: https://github.com/antocuni/pypy-wheels + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7 and CPython 3.5. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +The PyPy release supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html + +Changelog +========= + +* Speed up C-API method calls, and make most Py*_Check calls C macros +* Speed up C-API slot method calls +* Enable TkAgg backend support for matplotlib +* support ``hastzinfo`` and ``tzinfo`` in the C-API ``PyDateTime*`` structures +* datetime.h is now more similar to CPython +* We now support ``PyUnicode_AsUTF{16,32}String``, ``_PyLong_AsByteArray``, + ``_PyLong_AsByteArrayO``, +* PyPy3.5 on Windows is compiled with the Microsoft Visual Compiler v14, like + CPython +* Fix performance of attribute lookup when more than 80 attributes are used +* Improve performance on passing built-in types to C-API C code +* Improve the performance of datetime and timedelta by skipping the consistency + checks of the datetime values (they are correct by construction) +* Improve handling of ``bigint`` s, including fixing ``int_divmod`` +* Improve reporting of GC statistics +* Accept unicode filenames in ``dbm.open()`` +* Improve RPython support for half-floats +* Added missing attributes to C-API ``instancemethod`` on pypy3 +* Store error state in thread-local storage for C-API. +* Fix JIT bugs exposed in the sre module +* Improve speed of Python parser, improve ParseError messages and SyntaxError +* Handle JIT hooks more efficiently +* Fix a rare GC bug exposed by intensive use of cpyext ``Buffer`` s + +We also refactored many parts of the JIT bridge optimizations, as well as cpyext +internals, and together with new contributors fixed issues, added new +documentation, and cleaned up the codebase. diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -1,73 +1,12 @@ -=========================== -What's new in PyPy2.7 5.10+ -=========================== +========================== +What's new in PyPy2.7 6.0+ +========================== -.. this is a revision shortly after release-pypy2.7-v5.10.0 -.. startrev: 6b024edd9d12 +.. this is a revision shortly after release-pypy-6.0.0 +.. startrev: e50e11af23f1 -.. branch: cpyext-avoid-roundtrip -Big refactoring of some cpyext code, which avoids a lot of nonsense when -calling C from Python and vice-versa: the result is a big speedup in -function/method calls, up to 6 times faster. -.. branch: cpyext-datetime2 - -Support ``tzinfo`` field on C-API datetime objects, fixes latest pandas HEAD - - -.. branch: mapdict-size-limit - -Fix a corner case of mapdict: When an instance is used like a dict (using -``setattr`` and ``getattr``, or ``.__dict__``) and a lot of attributes are -added, then the performance using mapdict is linear in the number of -attributes. This is now fixed (by switching to a regular dict after 80 -attributes). - - -.. branch: cpyext-faster-arg-passing - -When using cpyext, improve the speed of passing certain objects from PyPy to C -code, most notably None, True, False, types, all instances of C-defined types. -Before, a dict lookup was needed every time such an object crossed over, now it -is just a field read. - - -.. branch: 2634_datetime_timedelta_performance - -Improve datetime + timedelta performance. - -.. branch: memory-accounting - -Improve way to describe memory - -.. branch: msvc14 - -Allow compilaiton with Visual Studio 2017 compiler suite on windows - -.. branch: refactor-slots - -Refactor cpyext slots. - - -.. branch: call-loopinvariant-into-bridges - -Speed up branchy code that does a lot of function inlining by saving one call -to read the TLS in most bridges. - -.. branch: rpython-sprint - -Refactor in rpython signatures - -.. branch: cpyext-tls-operror2 - -Store error state thread-locally in executioncontext, fixes issue #2764 - -.. branch: cpyext-fast-typecheck - -Optimize `Py*_Check` for `Bool`, `Float`, `Set`. Also refactor and simplify -`W_PyCWrapperObject` which is used to call slots from the C-API, greatly -improving microbenchmarks in https://github.com/antocuni/cpyext-benchmarks .. branch: unicode-utf8-re .. branch: utf8-io diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-pypy2-6.0.0.rst copy from pypy/doc/whatsnew-head.rst copy to pypy/doc/whatsnew-pypy2-6.0.0.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-pypy2-6.0.0.rst @@ -69,8 +69,60 @@ `W_PyCWrapperObject` which is used to call slots from the C-API, greatly improving microbenchmarks in https://github.com/antocuni/cpyext-benchmarks -.. branch: unicode-utf8-re -.. branch: utf8-io -Utf8 handling for unicode +.. branch: fix-sre-problems +Fix two (unrelated) JIT bugs manifesting in the re module: + +- green fields are broken and were thus disabled, plus their usage removed from + the _sre implementation + +- in rare "trace is too long" situations, the JIT could break behaviour + arbitrarily. + +.. branch: jit-hooks-can-be-disabled + +Be more efficient about JIT hooks. Make it possible for the frontend to declare +that jit hooks are currently not enabled at all. in that case, the list of ops +does not have to be created in the case of the on_abort hook (which is +expensive). + + +.. branch: pyparser-improvements + +Improve speed of Python parser, improve ParseError messages slightly. + +.. branch: ioctl-arg-size + +Work around possible bugs in upstream ioctl users, like CPython allocate at +least 1024 bytes for the arg in calls to ``ioctl(fd, request, arg)``. Fixes +issue #2776 + +.. branch: cpyext-subclass-setattr + +Fix for python-level classes that inherit from C-API types, previously the +`w_obj` was not necessarily preserved throughout the lifetime of the `pyobj` +which led to cases where instance attributes were lost. Fixes issue #2793 + + +.. branch: pyparser-improvements-2 + +Improve line offsets that are reported by SyntaxError. Improve error messages +for a few situations, including mismatched parenthesis. + +.. branch: issue2752 + +Fix a rare GC bug that was introduced more than one year ago, but was +not diagnosed before issue #2752. + +.. branch: gc-hooks + +Introduce GC hooks, as documented in doc/gc_info.rst + +.. branch: gc-hook-better-timestamp + +Improve GC hooks + +.. branch: cppyy-packaging + +Update backend to 0.6.0 and support exceptions through wrappers diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -215,6 +215,7 @@ usage = SUPPRESS_USAGE take_options = True + space = None def opt_parser(self, config): parser = to_optparse(config, useoptions=["objspace.*"], @@ -364,15 +365,21 @@ from pypy.module.pypyjit.hooks import pypy_hooks return PyPyJitPolicy(pypy_hooks) + def get_gchooks(self): + from pypy.module.gc.hook import LowLevelGcHooks + if self.space is None: + raise Exception("get_gchooks must be called afeter get_entry_point") + return self.space.fromcache(LowLevelGcHooks) + def get_entry_point(self, config): - space = make_objspace(config) + self.space = make_objspace(config) # manually imports app_main.py filename = os.path.join(pypydir, 'interpreter', 'app_main.py') app = gateway.applevel(open(filename).read(), 'app_main.py', 'app_main') app.hidden_applevel = False - w_dict = app.getwdict(space) - entry_point, _ = create_entry_point(space, w_dict) + w_dict = app.getwdict(self.space) + entry_point, _ = create_entry_point(self.space, w_dict) return entry_point, None, PyPyAnnotatorPolicy() @@ -381,7 +388,7 @@ 'jitpolicy', 'get_entry_point', 'get_additional_config_options']: ns[name] = getattr(self, name) - + ns['get_gchooks'] = self.get_gchooks PyPyTarget().interface(globals()) diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -404,7 +404,7 @@ self._periodic_actions = [] self._nonperiodic_actions = [] self.has_bytecode_counter = False - self.fired_actions = None + self._fired_actions_reset() # the default value is not 100, unlike CPython 2.7, but a much # larger value, because we use a technique that not only allows # but actually *forces* another thread to run whenever the counter @@ -416,13 +416,28 @@ """Request for the action to be run before the next opcode.""" if not action._fired: action._fired = True - if self.fired_actions is None: - self.fired_actions = [] - self.fired_actions.append(action) + self._fired_actions_append(action) # set the ticker to -1 in order to force action_dispatcher() # to run at the next possible bytecode self.reset_ticker(-1) + def _fired_actions_reset(self): + # linked list of actions. We cannot use a normal RPython list because + # we want AsyncAction.fire() to be marked as @rgc.collect: this way, + # we can call it from e.g. GcHooks or cpyext's dealloc_trigger. + self._fired_actions_first = None + self._fired_actions_last = None + + @rgc.no_collect + def _fired_actions_append(self, action): + assert action._next is None + if self._fired_actions_first is None: + self._fired_actions_first = action + self._fired_actions_last = action + else: + self._fired_actions_last._next = action + self._fired_actions_last = action + @not_rpython def register_periodic_action(self, action, use_bytecode_counter): """ @@ -467,19 +482,26 @@ action.perform(ec, frame) # nonperiodic actions - list = self.fired_actions - if list is not None: - self.fired_actions = None + action = self._fired_actions_first + if action: + self._fired_actions_reset() # NB. in case there are several actions, we reset each # 'action._fired' to false only when we're about to call # 'action.perform()'. This means that if # 'action.fire()' happens to be called any time before # the corresponding perform(), the fire() has no # effect---which is the effect we want, because - # perform() will be called anyway. - for action in list: + # perform() will be called anyway. All such pending + # actions with _fired == True are still inside the old + # chained list. As soon as we reset _fired to False, + # we also reset _next to None and we are ready for + # another fire(). + while action is not None: + next_action = action._next + action._next = None action._fired = False action.perform(ec, frame) + action = next_action self.action_dispatcher = action_dispatcher @@ -512,10 +534,12 @@ to occur between two opcodes, not at a completely random time. """ _fired = False + _next = None def __init__(self, space): self.space = space + @rgc.no_collect def fire(self): """Request for the action to be run before the next opcode. The action must have been registered at space initalization time.""" diff --git a/pypy/interpreter/pyparser/error.py b/pypy/interpreter/pyparser/error.py --- a/pypy/interpreter/pyparser/error.py +++ b/pypy/interpreter/pyparser/error.py @@ -6,6 +6,7 @@ lastlineno=0): self.msg = msg self.lineno = lineno + # NB: offset is a 1-based index! self.offset = offset self.text = text self.filename = filename diff --git a/pypy/interpreter/pyparser/metaparser.py b/pypy/interpreter/pyparser/metaparser.py --- a/pypy/interpreter/pyparser/metaparser.py +++ b/pypy/interpreter/pyparser/metaparser.py @@ -147,8 +147,10 @@ for label, next in state.arcs.iteritems(): arcs.append((self.make_label(gram, label), dfa.index(next))) states.append((arcs, state.is_final)) - gram.dfas.append((states, self.make_first(gram, name))) - assert len(gram.dfas) - 1 == gram.symbol_ids[name] - 256 + symbol_id = gram.symbol_ids[name] + dfa = parser.DFA(symbol_id, states, self.make_first(gram, name)) + gram.dfas.append(dfa) + assert len(gram.dfas) - 1 == symbol_id - 256 gram.start = gram.symbol_ids[self.start_symbol] return gram @@ -162,6 +164,13 @@ else: gram.labels.append(gram.symbol_ids[label]) gram.symbol_to_label[label] = label_index + first = self.first[label] + if len(first) == 1: + first, = first + if not first[0].isupper(): + first = first.strip("\"'") + assert label_index not in gram.token_to_error_string + gram.token_to_error_string[label_index] = first return label_index elif label.isupper(): token_index = gram.TOKENS[label] @@ -183,7 +192,7 @@ else: gram.labels.append(gram.KEYWORD_TOKEN) gram.keyword_ids[value] = label_index - return label_index + result = label_index else: try: token_index = gram.OPERATOR_MAP[value] @@ -194,7 +203,10 @@ else: gram.labels.append(token_index) gram.token_ids[token_index] = label_index - return label_index + result = label_index + assert result not in gram.token_to_error_string + gram.token_to_error_string[result] = value + return result def make_first(self, gram, name): original_firsts = self.first[name] diff --git a/pypy/interpreter/pyparser/parser.py b/pypy/interpreter/pyparser/parser.py --- a/pypy/interpreter/pyparser/parser.py +++ b/pypy/interpreter/pyparser/parser.py @@ -1,6 +1,7 @@ """ A CPython inspired RPython parser. """ +from rpython.rlib.objectmodel import not_rpython class Grammar(object): @@ -16,6 +17,7 @@ self.symbol_names = {} self.symbol_to_label = {} self.keyword_ids = {} + self.token_to_error_string = {} self.dfas = [] self.labels = [0] self.token_ids = {} @@ -41,6 +43,27 @@ pass return True +class DFA(object): + def __init__(self, symbol_id, states, first): + self.symbol_id = symbol_id + self.states = states + self.first = self._first_to_string(first) + + def could_match_token(self, label_index): + pos = label_index >> 3 + bit = 1 << (label_index & 0b111) + return bool(ord(self.first[label_index >> 3]) & bit) + + @staticmethod + @not_rpython + def _first_to_string(first): + l = sorted(first.keys()) + b = bytearray(32) + for label_index in l: + pos = label_index >> 3 + bit = 1 << (label_index & 0b111) + b[pos] |= bit + return str(b) class Node(object): @@ -127,14 +150,17 @@ class Nonterminal(AbstractNonterminal): __slots__ = ("_children", ) - def __init__(self, type, children): + def __init__(self, type, children=None): Node.__init__(self, type) + if children is None: + children = [] self._children = children def __repr__(self): return "Nonterminal(type=%s, children=%r)" % (self.type, self._children) def get_child(self, i): + assert self._children is not None return self._children[i] def num_children(self): @@ -168,25 +194,50 @@ class ParseError(Exception): def __init__(self, msg, token_type, value, lineno, column, line, - expected=-1): + expected=-1, expected_str=None): self.msg = msg self.token_type = token_type self.value = value self.lineno = lineno + # this is a 0-based index self.column = column self.line = line self.expected = expected + self.expected_str = expected_str def __str__(self): return "ParserError(%s, %r)" % (self.token_type, self.value) +class StackEntry(object): + def __init__(self, next, dfa, state): + self.next = next + self.dfa = dfa + self.state = state + self.node = None + + def push(self, dfa, state): + return StackEntry(self, dfa, state) + + def pop(self): + return self.next + + def node_append_child(self, child): + node = self.node + if node is None: + self.node = Nonterminal1(self.dfa.symbol_id, child) + elif isinstance(node, Nonterminal1): + newnode = self.node = Nonterminal( + self.dfa.symbol_id, [node._child, child]) + else: + self.node.append_child(child) + + class Parser(object): def __init__(self, grammar): self.grammar = grammar self.root = None - self.stack = None def prepare(self, start=-1): """Setup the parser for parsing. @@ -196,16 +247,15 @@ if start == -1: start = self.grammar.start self.root = None - current_node = Nonterminal(start, []) - self.stack = [] - self.stack.append((self.grammar.dfas[start - 256], 0, current_node)) + self.stack = StackEntry(None, self.grammar.dfas[start - 256], 0) def add_token(self, token_type, value, lineno, column, line): label_index = self.classify(token_type, value, lineno, column, line) sym_id = 0 # for the annotator while True: - dfa, state_index, node = self.stack[-1] - states, first = dfa + dfa = self.stack.dfa + state_index = self.stack.state + states = dfa.states arcs, is_accepting = states[state_index] for i, next_state in arcs: sym_id = self.grammar.labels[i] @@ -217,16 +267,17 @@ # the stack. while state[1] and not state[0]: self.pop() - if not self.stack: + if self.stack is None: # Parsing is done. return True - dfa, state_index, node = self.stack[-1] - state = dfa[0][state_index] + dfa = self.stack.dfa + state_index = self.stack.state + state = dfa.states[state_index] return False elif sym_id >= 256: sub_node_dfa = self.grammar.dfas[sym_id - 256] # Check if this token can start a child node. - if label_index in sub_node_dfa[1]: + if sub_node_dfa.could_match_token(label_index): self.push(sub_node_dfa, next_state, sym_id, lineno, column) break @@ -235,7 +286,7 @@ # state is accepting, it's invalid input. if is_accepting: self.pop() - if not self.stack: + if self.stack is None: raise ParseError("too much input", token_type, value, lineno, column, line) else: @@ -243,10 +294,13 @@ # error. if len(arcs) == 1: expected = sym_id + expected_str = self.grammar.token_to_error_string.get( + arcs[0][0], None) else: expected = -1 + expected_str = None raise ParseError("bad input", token_type, value, lineno, - column, line, expected) + column, line, expected, expected_str) def classify(self, token_type, value, lineno, column, line): """Find the label for a token.""" @@ -262,26 +316,22 @@ def shift(self, next_state, token_type, value, lineno, column): """Shift a non-terminal and prepare for the next state.""" - dfa, state, node = self.stack[-1] new_node = Terminal(token_type, value, lineno, column) - node.append_child(new_node) - self.stack[-1] = (dfa, next_state, node) + self.stack.node_append_child(new_node) + self.stack.state = next_state def push(self, next_dfa, next_state, node_type, lineno, column): """Push a terminal and adjust the current state.""" - dfa, state, node = self.stack[-1] - new_node = Nonterminal(node_type, []) - self.stack[-1] = (dfa, next_state, node) - self.stack.append((next_dfa, 0, new_node)) + self.stack.state = next_state + self.stack = self.stack.push(next_dfa, 0) def pop(self): """Pop an entry off the stack and make its node a child of the last.""" - dfa, state, node = self.stack.pop() + top = self.stack + self.stack = top.pop() + node = top.node + assert node is not None if self.stack: - # we are now done with node, so we can store it more efficiently if - # it has just one child - if node.num_children() == 1: - node = Nonterminal1(node.type, node.get_child(0)) - self.stack[-1][2].append_child(node) + self.stack.node_append_child(node) else: self.root = node diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py --- a/pypy/interpreter/pyparser/pyparse.py +++ b/pypy/interpreter/pyparser/pyparse.py @@ -132,7 +132,11 @@ w_message = space.str(e.get_w_value(space)) raise error.SyntaxError(space.text_w(w_message)) raise + if enc is not None: + compile_info.encoding = enc + return self._parse(textsrc, compile_info) + def _parse(self, textsrc, compile_info): flags = compile_info.flags # The tokenizer is very picky about how it wants its input. @@ -181,13 +185,16 @@ else: new_err = error.SyntaxError msg = "invalid syntax" - raise new_err(msg, e.lineno, e.column, e.line, + if e.expected_str is not None: + msg += " (expected '%s')" % e.expected_str + + # parser.ParseError(...).column is 0-based, but the offsets in the + # exceptions in the error module are 1-based, hence the '+ 1' + raise new_err(msg, e.lineno, e.column + 1, e.line, compile_info.filename) else: tree = self.root finally: # Avoid hanging onto the tree. self.root = None - if enc is not None: - compile_info.encoding = enc return tree diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py --- a/pypy/interpreter/pyparser/pytokenizer.py +++ b/pypy/interpreter/pyparser/pytokenizer.py @@ -73,14 +73,14 @@ logical line; continuation lines are included. """ token_list = [] - lnum = parenlev = continued = 0 + lnum = continued = 0 namechars = NAMECHARS numchars = NUMCHARS contstr, needcont = '', 0 contline = None indents = [0] last_comment = '' - parenlevstart = (0, 0, "") + parenstack = [] # make the annotator happy endDFA = DUMMY_DFA @@ -97,7 +97,7 @@ if contstr: if not line: raise TokenError( - "EOF while scanning triple-quoted string literal", + "end of file (EOF) while scanning triple-quoted string literal", strstart[2], strstart[0], strstart[1]+1, token_list, lnum-1) endmatch = endDFA.recognize(line) @@ -123,7 +123,7 @@ contline = contline + line continue - elif parenlev == 0 and not continued: # new statement + elif not parenstack and not continued: # new statement if not line: break column = 0 while pos < max: # measure leading whitespace @@ -143,21 +143,21 @@ token_list.append((tokens.INDENT, line[:pos], lnum, 0, line)) last_comment = '' while column < indents[-1]: - indents = indents[:-1] + indents.pop() token_list.append((tokens.DEDENT, '', lnum, pos, line)) last_comment = '' if column != indents[-1]: err = "unindent does not match any outer indentation level" - raise TokenIndentationError(err, line, lnum, 0, token_list) + raise TokenIndentationError(err, line, lnum, column+1, token_list) else: # continued statement if not line: - if parenlev > 0: - lnum1, start1, line1 = parenlevstart + if parenstack: + _, lnum1, start1, line1 = parenstack[0] raise TokenError("parenthesis is never closed", line1, lnum1, start1 + 1, token_list, lnum) - raise TokenError("EOF in multi-line statement", line, - lnum, 0, token_list) + raise TokenError("end of file (EOF) in multi-line statement", line, + lnum, 0, token_list) # XXX why is the offset 0 here? continued = 0 while pos < max: @@ -180,7 +180,7 @@ token_list.append((tokens.NUMBER, token, lnum, start, line)) last_comment = '' elif initial in '\r\n': - if parenlev <= 0: + if not parenstack: tok = (tokens.NEWLINE, last_comment, lnum, start, line) token_list.append(tok) last_comment = '' @@ -222,14 +222,22 @@ continued = 1 else: if initial in '([{': - if parenlev == 0: - parenlevstart = (lnum, start, line) - parenlev = parenlev + 1 + parenstack.append((initial, lnum, start, line)) elif initial in ')]}': - parenlev = parenlev - 1 - if parenlev < 0: + if not parenstack: raise TokenError("unmatched '%s'" % initial, line, lnum, start + 1, token_list) + opening, lnum1, start1, line1 = parenstack.pop() + if not ((opening == "(" and initial == ")") or + (opening == "[" and initial == "]") or + (opening == "{" and initial == "}")): + msg = "closing parenthesis '%s' does not match opening parenthesis '%s'" % ( + initial, opening) + + if lnum1 != lnum: + msg += " on line " + str(lnum1) + raise TokenError( + msg, line, lnum, start + 1, token_list) if token in python_opmap: punct = python_opmap[token] else: @@ -241,7 +249,7 @@ if start < 0: start = pos if start", "exec") + parser = pyparse.PythonParser(fakespace) + tree = parser._parse(s, info) + b = time.clock() + print fn, (b-a) + + +def entry_point(argv): + if len(argv) == 2: + fn = argv[1] + else: + fn = "../../../../rpython/rlib/unicodedata/unicodedb_5_2_0.py" + fd = os.open(fn, os.O_RDONLY, 0777) + res = [] + while True: + s = os.read(fd, 4096) + if not s: + break + res.append(s) + os.close(fd) + s = "".join(res) + print len(s) + bench(fn, s) + + return 0 + +# _____ Define and setup target ___ + +def target(*args): + return entry_point, None + +if __name__ == '__main__': + entry_point(sys.argv) diff --git a/pypy/interpreter/pyparser/test/test_metaparser.py b/pypy/interpreter/pyparser/test/test_metaparser.py --- a/pypy/interpreter/pyparser/test/test_metaparser.py +++ b/pypy/interpreter/pyparser/test/test_metaparser.py @@ -34,8 +34,8 @@ assert len(g.dfas) == 1 eval_sym = g.symbol_ids["eval"] assert g.start == eval_sym - states, first = g.dfas[eval_sym - 256] - assert states == [([(1, 1)], False), ([], True)] + dfa = g.dfas[eval_sym - 256] + assert dfa.states == [([(1, 1)], False), ([], True)] assert g.labels[0] == 0 def test_load_python_grammars(self): @@ -51,7 +51,7 @@ def test_items(self): g = self.gram_for("foo: NAME STRING OP '+'") assert len(g.dfas) == 1 - states = g.dfas[g.symbol_ids["foo"] - 256][0] + states = g.dfas[g.symbol_ids["foo"] - 256].states last = states[0][0][0][1] for state in states[1:-1]: assert last < state[0][0][1] diff --git a/pypy/interpreter/pyparser/test/test_parser.py b/pypy/interpreter/pyparser/test/test_parser.py --- a/pypy/interpreter/pyparser/test/test_parser.py +++ b/pypy/interpreter/pyparser/test/test_parser.py @@ -7,6 +7,12 @@ from pypy.interpreter.pyparser.test.test_metaparser import MyGrammar +def test_char_set(): + first = {5: None, 9: None, 100: None, 255:None} + p = parser.DFA(None, None, first) + for i in range(256): + assert p.could_match_token(i) == (i in first) + class SimpleParser(parser.Parser): def parse(self, input): @@ -55,8 +61,7 @@ n = parser.Terminal(tp, value, 0, 0) else: tp = gram.symbol_ids[data[0]] - children = [] - n = parser.Nonterminal(tp, children) + n = parser.Nonterminal(tp) new_indent = count_indent(line) if new_indent >= last_indent: if new_indent == last_indent and node_stack: @@ -291,3 +296,37 @@ NEWLINE ENDMARKER""" assert tree_from_string(expected, gram) == p.parse("hi 42 end") + + + def test_optimized_terminal(self): + gram = """foo: bar baz 'end' NEWLINE ENDMARKER +bar: NAME +baz: NUMBER +""" + p, gram = self.parser_for(gram, False) + expected = """ + foo + bar + NAME "a_name" + baz + NUMBER "42" + NAME "end" + NEWLINE + ENDMARKER""" + input = "a_name 42 end" + tree = p.parse(input) + assert tree_from_string(expected, gram) == tree + assert isinstance(tree, parser.Nonterminal) + assert isinstance(tree.get_child(0), parser.Nonterminal1) + assert isinstance(tree.get_child(1), parser.Nonterminal1) + + + def test_error_string(self): + p, gram = self.parser_for( + "foo: 'if' NUMBER '+' NUMBER" + ) + info = py.test.raises(parser.ParseError, p.parse, "if 42") + info.value.expected_str is None + info = py.test.raises(parser.ParseError, p.parse, "if 42 42") + info.value.expected_str == '+' + diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py --- a/pypy/interpreter/pyparser/test/test_pyparse.py +++ b/pypy/interpreter/pyparser/test/test_pyparse.py @@ -76,14 +76,14 @@ exc = py.test.raises(SyntaxError, parse, "name another for").value assert exc.msg == "invalid syntax" assert exc.lineno == 1 - assert exc.offset == 5 + assert exc.offset == 6 assert exc.text.startswith("name another for") exc = py.test.raises(SyntaxError, parse, "x = \"blah\n\n\n").value - assert exc.msg == "EOL while scanning string literal" + assert exc.msg == "end of line (EOL) while scanning string literal" assert exc.lineno == 1 assert exc.offset == 5 exc = py.test.raises(SyntaxError, parse, "x = '''\n\n\n").value - assert exc.msg == "EOF while scanning triple-quoted string literal" + assert exc.msg == "end of file (EOF) while scanning triple-quoted string literal" assert exc.lineno == 1 assert exc.offset == 5 assert exc.lastlineno == 3 @@ -112,7 +112,7 @@ assert exc.msg == "expected an indented block" assert exc.lineno == 3 assert exc.text.startswith("pass") - assert exc.offset == 0 + assert exc.offset == 1 input = "hi\n indented" exc = py.test.raises(IndentationError, parse, input).value assert exc.msg == "unexpected indent" @@ -120,6 +120,7 @@ exc = py.test.raises(IndentationError, parse, input).value assert exc.msg == "unindent does not match any outer indentation level" assert exc.lineno == 3 + assert exc.offset == 3 def test_mac_newline(self): self.parse("this_is\ra_mac\rfile")