From noreply at buildbot.pypy.org Sun Mar 1 12:37:08 2015 From: noreply at buildbot.pypy.org (mjacob) Date: Sun, 1 Mar 2015 12:37:08 +0100 (CET) Subject: [pypy-commit] pypy spaceops-are-variables: Temporarily disable malloc removal in the translator -- it's not yet fixed and some (otherwise unrelated) tests fails because it was enabled by default. Message-ID: <20150301113708.2AE091C02BF@cobra.cs.uni-duesseldorf.de> Author: Manuel Jacob Branch: spaceops-are-variables Changeset: r76209:eb037c42c6be Date: 2015-03-01 12:35 +0100 http://bitbucket.org/pypy/pypy/changeset/eb037c42c6be/ Log: Temporarily disable malloc removal in the translator -- it's not yet fixed and some (otherwise unrelated) tests fails because it was enabled by default. diff --git a/rpython/translator/backendopt/all.py b/rpython/translator/backendopt/all.py --- a/rpython/translator/backendopt/all.py +++ b/rpython/translator/backendopt/all.py @@ -170,8 +170,8 @@ # vaporize mallocs if config.mallocs: - log.malloc("starting malloc removal") - remove_mallocs(translator, graphs, type_system) + #log.malloc("starting malloc removal") + #remove_mallocs(translator, graphs, type_system) if config.print_statistics: print "after malloc removal:" From noreply at buildbot.pypy.org Sun Mar 1 19:28:34 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 1 Mar 2015 19:28:34 +0100 (CET) Subject: [pypy-commit] stmgc c8-card-marking: Write down the stm_write_card() version I'm talking about. Message-ID: <20150301182834.2E3B31C0233@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-card-marking Changeset: r1686:0b71086399c8 Date: 2015-03-01 19:29 +0100 http://bitbucket.org/pypy/stmgc/changeset/0b71086399c8/ Log: Write down the stm_write_card() version I'm talking about. I think the PyPy JIT should be modified to do exactly that. diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -933,6 +933,7 @@ bool mark_card = obj_should_use_cards(STM_SEGMENT->segment_base, obj); write_slowpath_common(obj, mark_card); return mark_card; + /* XXX likely, this whole function can be removed now */ } @@ -975,22 +976,19 @@ /* Write into the card's lock. This is used by the next minor collection to know what parts of the big object may have changed. We already own the object here or it is an overflow obj. */ - struct stm_read_marker_s *cards = get_read_marker(STM_SEGMENT->segment_base, - (uintptr_t)obj); - uintptr_t card_index = get_index_to_card_index(index); - if (cards[card_index].rm == CARD_MARKED) - return; + stm_read_marker_t *card = (stm_read_marker_t *)(((uintptr_t)obj) >> 4); + card += get_index_to_card_index(index); if (!IS_OVERFLOW_OBJ(STM_PSEGMENT, obj) - && !(cards[card_index].rm == CARD_MARKED - || cards[card_index].rm == STM_SEGMENT->transaction_read_version)) { + && !(card->rm == CARD_MARKED + || card->rm == STM_SEGMENT->transaction_read_version)) { /* need to do the backup slice of the card */ make_bk_slices(obj, false, /* first_call */ index, /* index: only 1 card */ false); /* do_missing_cards */ } - cards[card_index].rm = CARD_MARKED; + card->rm = CARD_MARKED; dprintf(("mark %p index %lu, card:%lu with %d\n", obj, index, get_index_to_card_index(index), CARD_MARKED)); diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -202,8 +202,27 @@ __attribute__((always_inline)) static inline void stm_write_card(object_t *obj, uintptr_t index) { - if (UNLIKELY((obj->stm_flags & _STM_GCFLAG_WRITE_BARRIER) != 0)) - _stm_write_slowpath_card(obj, index); + /* if GCFLAG_WRITE_BARRIER is set, then don't do anything more. */ + if (UNLIKELY((obj->stm_flags & _STM_GCFLAG_WRITE_BARRIER) != 0)) { + + /* GCFLAG_WRITE_BARRIER is not set. This might be because + it's the first time we see a given small array; or it might + be because it's a big array with card marking. In the + latter case we will always reach this point, even if we + already marked the correct card. Based on the idea that it + is actually the most common case, check it here. If the + array doesn't actually use card marking, the following read + is a bit nonsensical, but in a way that should never return + CARD_MARKED by mistake. + */ + stm_read_marker_t *card = (stm_read_marker_t *)(((uintptr_t)obj) >> 4); + card += (index / _STM_CARD_SIZE) + 1; /* get_index_to_card_index() */ + if (card->rm != _STM_CARD_MARKED) { + + /* slow path. */ + _stm_write_slowpath_card(obj, index); + } + } } From noreply at buildbot.pypy.org Mon Mar 2 00:14:05 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Mon, 2 Mar 2015 00:14:05 +0100 (CET) Subject: [pypy-commit] pypy default: Hide c_last_exception as an implementation detail of Block Message-ID: <20150301231405.586AA1C0378@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r76210:b6ada241779b Date: 2015-03-01 20:45 +0000 http://bitbucket.org/pypy/pypy/changeset/b6ada241779b/ Log: Hide c_last_exception as an implementation detail of Block diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py --- a/rpython/annotator/annrpython.py +++ b/rpython/annotator/annrpython.py @@ -6,8 +6,8 @@ from rpython.tool.pairtype import pair from rpython.tool.error import (format_blocked_annotation_error, gather_error, source_lines) -from rpython.flowspace.model import (Variable, Constant, FunctionGraph, - c_last_exception, checkgraph) +from rpython.flowspace.model import ( + Variable, Constant, FunctionGraph, checkgraph) from rpython.translator import simplify, transform from rpython.annotator import model as annmodel, signature from rpython.annotator.argument import simple_args @@ -407,8 +407,7 @@ self.bookkeeper.leave() except BlockedInference as e: - if (e.op is block.operations[-1] and - block.exitswitch == c_last_exception): + if e.op is block.raising_op: # this is the case where the last operation of the block will # always raise an exception which is immediately caught by # an exception handler. We then only follow the exceptional @@ -450,8 +449,8 @@ # filter out those exceptions which cannot # occour for this specific, typed operation. - if block.exitswitch == c_last_exception: - op = block.operations[-1] + if block.canraise: + op = block.raising_op can_only_throw = op.get_can_only_throw(self) if can_only_throw is not None: candidates = can_only_throw diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py --- a/rpython/flowspace/model.py +++ b/rpython/flowspace/model.py @@ -190,6 +190,15 @@ txt = "%s(%s)" % (txt, self.exitswitch) return txt + @property + def canraise(self): + return self.exitswitch is c_last_exception + + @property + def raising_op(self): + if self.canraise: + return self.operations[-1] + def getvariables(self): "Return all variables mentioned in this Block." result = self.inputargs[:] @@ -591,11 +600,11 @@ assert len(block.exits) <= 1 if block.exits: assert block.exits[0].exitcase is None - elif block.exitswitch == Constant(last_exception): + elif block.canraise: assert len(block.operations) >= 1 # check if an exception catch is done on a reasonable # operation - assert block.operations[-1].opname not in ("keepalive", + assert block.raising_op.opname not in ("keepalive", "cast_pointer", "same_as") assert len(block.exits) >= 2 diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py --- a/rpython/flowspace/test/test_objspace.py +++ b/rpython/flowspace/test/test_objspace.py @@ -4,7 +4,7 @@ from contextlib import contextmanager from rpython.flowspace.model import ( - Constant, mkentrymap, c_last_exception, const) + Constant, mkentrymap, const) from rpython.translator.simplify import simplify_graph from rpython.flowspace.objspace import build_flow from rpython.flowspace.flowcontext import FlowingError, FlowContext @@ -826,7 +826,7 @@ return None graph = self.codetest(myfunc) simplify_graph(graph) - assert graph.startblock.exitswitch == c_last_exception + assert graph.startblock.canraise assert graph.startblock.exits[0].target is graph.returnblock assert graph.startblock.exits[1].target is graph.returnblock diff --git a/rpython/jit/codewriter/flatten.py b/rpython/jit/codewriter/flatten.py --- a/rpython/jit/codewriter/flatten.py +++ b/rpython/jit/codewriter/flatten.py @@ -1,4 +1,4 @@ -from rpython.flowspace.model import Variable, Constant, c_last_exception +from rpython.flowspace.model import Variable, Constant from rpython.jit.metainterp.history import AbstractDescr, getkind from rpython.rtyper.lltypesystem import lltype @@ -167,7 +167,7 @@ # up in the manually hacked graphs for generators... self.make_link(link) # - elif block.exitswitch is c_last_exception: + elif block.canraise: # An exception block. See test_exc_exitswitch in test_flatten.py # for an example of what kind of code this makes. index = -1 @@ -184,7 +184,7 @@ # actually a '-live-' self.make_link(block.exits[0]) return - # + # self.emitline('catch_exception', TLabel(block.exits[0])) self.make_link(block.exits[0]) self.emitline(Label(block.exits[0])) diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -8,7 +8,7 @@ from rpython.jit.metainterp.history import getkind from rpython.jit.metainterp.typesystem import deref, arrayItem from rpython.jit.metainterp.blackhole import BlackholeInterpreter -from rpython.flowspace.model import SpaceOperation, Variable, Constant, c_last_exception +from rpython.flowspace.model import SpaceOperation, Variable, Constant from rpython.rlib import objectmodel from rpython.rlib.jit import _we_are_jitted from rpython.rlib.rgc import lltype_is_gc @@ -110,7 +110,7 @@ else: raise TypeError(repr(op1)) # - if block.exitswitch == c_last_exception: + if block.canraise: if len(newoperations) == count_before_last_operation: self._killed_exception_raising_operation(block) block.operations = newoperations @@ -175,7 +175,7 @@ def follow_constant_exit(self, block): v = block.exitswitch - if isinstance(v, Constant) and v != c_last_exception: + if isinstance(v, Constant) and not block.canraise: llvalue = v.value for link in block.exits: if link.llexitcase == llvalue: @@ -192,7 +192,7 @@ if len(block.exits) != 2: return False v = block.exitswitch - if (v == c_last_exception or isinstance(v, tuple) + if (block.canraise or isinstance(v, tuple) or v.concretetype != lltype.Bool): return False for op in block.operations[::-1]: diff --git a/rpython/memory/gctransform/test/test_transform.py b/rpython/memory/gctransform/test/test_transform.py --- a/rpython/memory/gctransform/test/test_transform.py +++ b/rpython/memory/gctransform/test/test_transform.py @@ -1,5 +1,5 @@ from rpython.memory.gctransform.transform import BaseGCTransformer -from rpython.flowspace.model import c_last_exception, Variable +from rpython.flowspace.model import Variable from rpython.translator.backendopt.support import var_needsgc from rpython.translator.translator import TranslationContext, graphof from rpython.translator.exceptiontransform import ExceptionTransformer @@ -119,8 +119,8 @@ if pop_alives == len(block.operations): # it's a block we inserted return + assert not block.canraise for link in block.exits: - assert block.exitswitch is not c_last_exception refs_out = 0 for v2 in link.target.inputargs: if var_needsgc(v2) and not is_borrowed(v2): diff --git a/rpython/memory/gctransform/transform.py b/rpython/memory/gctransform/transform.py --- a/rpython/memory/gctransform/transform.py +++ b/rpython/memory/gctransform/transform.py @@ -1,6 +1,6 @@ from rpython.rtyper.lltypesystem import lltype, llmemory -from rpython.flowspace.model import SpaceOperation, Variable, Constant, \ - c_last_exception, checkgraph +from rpython.flowspace.model import ( + SpaceOperation, Variable, Constant, checkgraph) from rpython.translator.unsimplify import insert_empty_block from rpython.translator.unsimplify import insert_empty_startblock from rpython.translator.unsimplify import starts_with_empty_block @@ -180,7 +180,7 @@ hop.dispatch() if len(block.exits) != 0: # i.e not the return block - assert block.exitswitch is not c_last_exception + assert not block.canraise deadinallexits = set(self.livevars) for link in block.exits: diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py --- a/rpython/rtyper/llinterp.py +++ b/rpython/rtyper/llinterp.py @@ -5,8 +5,7 @@ import py -from rpython.flowspace.model import (FunctionGraph, Constant, Variable, - c_last_exception) +from rpython.flowspace.model import (FunctionGraph, Constant, Variable) from rpython.rlib import rstackovf from rpython.rlib.objectmodel import (ComputedIntSymbolic, CDefinedIntSymbolic, Symbolic) @@ -292,7 +291,6 @@ is None, values is the concrete return value. """ self.curr_block = block - catch_exception = block.exitswitch == c_last_exception e = None try: @@ -300,7 +298,7 @@ self.curr_operation_index = i self.eval_operation(op) except LLException, e: - if not (catch_exception and op is block.operations[-1]): + if op is not block.raising_op: raise except RuntimeError, e: rstackovf.check_stack_overflow() @@ -312,7 +310,7 @@ evalue = exdata.get_standard_ll_exc_instance(rtyper, classdef) etype = exdata.fn_type_of_exc_inst(evalue) e = LLException(etype, evalue) - if not (catch_exception and op is block.operations[-1]): + if op is not block.raising_op: raise e # determine nextblock and/or return value @@ -354,7 +352,7 @@ # single-exit block assert len(block.exits) == 1 link = block.exits[0] - elif catch_exception: + elif block.canraise: link = block.exits[0] if e: exdata = self.llinterpreter.typer.exceptiondata diff --git a/rpython/rtyper/rtyper.py b/rpython/rtyper/rtyper.py --- a/rpython/rtyper/rtyper.py +++ b/rpython/rtyper/rtyper.py @@ -17,7 +17,7 @@ from rpython.annotator import model as annmodel, unaryop, binaryop from rpython.rtyper.llannotation import SomePtr, lltype_to_annotation -from rpython.flowspace.model import Variable, Constant, SpaceOperation, c_last_exception +from rpython.flowspace.model import Variable, Constant, SpaceOperation from rpython.rtyper.annlowlevel import annotate_lowlevel_helper, LowLevelAnnotatorPolicy from rpython.rtyper.error import TyperError from rpython.rtyper.exceptiondata import ExceptionData @@ -353,7 +353,7 @@ if (pos is not None and pos != len(newops) - 1): # this is for the case where the llop that raises the exceptions # is not the last one in the list. - assert block.exitswitch == c_last_exception + assert block.canraise noexclink = block.exits[0] assert noexclink.exitcase is None if pos == "removed": @@ -388,7 +388,7 @@ if isinstance(block.exitswitch, Variable): r_case = self.bindingrepr(block.exitswitch) else: - assert block.exitswitch == c_last_exception + assert block.canraise r_case = rclass.get_type_repr(self) link.llexitcase = r_case.convert_const(link.exitcase) else: @@ -458,7 +458,7 @@ for op in block.operations[:-1]: yield HighLevelOp(self, op, [], llops) # look for exception links for the last operation - if block.exitswitch == c_last_exception: + if block.canraise: exclinks = block.exits[1:] else: exclinks = [] diff --git a/rpython/translator/backendopt/constfold.py b/rpython/translator/backendopt/constfold.py --- a/rpython/translator/backendopt/constfold.py +++ b/rpython/translator/backendopt/constfold.py @@ -1,5 +1,5 @@ from rpython.flowspace.model import (Constant, Variable, SpaceOperation, - c_last_exception, mkentrymap) + mkentrymap) from rpython.rtyper.lltypesystem import lltype from rpython.rtyper.lltypesystem.lloperation import llop from rpython.translator.unsimplify import insert_empty_block, split_block @@ -66,7 +66,7 @@ def constant_fold_block(block): constants = {} block.operations = fold_op_list(block.operations, constants, - exc_catch=block.exitswitch == c_last_exception) + exc_catch=block.canraise) if constants: if block.exitswitch in constants: switch = constants[block.exitswitch].value @@ -155,7 +155,7 @@ folded_count = fold_op_list(block.operations, constants, exit_early=True) n = len(block.operations) - if block.exitswitch == c_last_exception: + if block.canraise: n -= 1 # is the next, non-folded operation an indirect_call? if folded_count < n: diff --git a/rpython/translator/backendopt/inline.py b/rpython/translator/backendopt/inline.py --- a/rpython/translator/backendopt/inline.py +++ b/rpython/translator/backendopt/inline.py @@ -1,7 +1,7 @@ import sys from rpython.flowspace.model import (Variable, Constant, Block, Link, - SpaceOperation, c_last_exception, FunctionGraph, mkentrymap) + SpaceOperation, FunctionGraph, mkentrymap) from rpython.rtyper.lltypesystem.lltype import Bool, Signed, typeOf, Void, Ptr, normalizeptr from rpython.tool.algo import sparsemat from rpython.translator.backendopt import removenoops @@ -112,7 +112,7 @@ for block in graph.iterblocks(): if block is graph.exceptblock: return True # the except block is reachable - if block.exitswitch == c_last_exception: + if block.canraise: consider_ops_to = -1 else: consider_ops_to = len(block.operations) @@ -132,7 +132,7 @@ else: return True # conservatively for block in from_graph.iterblocks(): - if block.exitswitch == c_last_exception: + if block.canraise: consider_ops_to = -1 else: consider_ops_to = len(block.operations) @@ -196,8 +196,7 @@ self.op = block.operations[index_operation] self.graph_to_inline = self.get_graph_from_op(self.op) self.exception_guarded = False - if (block.exitswitch == c_last_exception and - index_operation == len(block.operations) - 1): + if self.op is block.raising_op: self.exception_guarded = True if self.inline_guarded_calls: if (not self.inline_guarded_calls_no_matter_what and diff --git a/rpython/translator/backendopt/mallocv.py b/rpython/translator/backendopt/mallocv.py --- a/rpython/translator/backendopt/mallocv.py +++ b/rpython/translator/backendopt/mallocv.py @@ -1,7 +1,6 @@ from rpython.flowspace.model import Variable, Constant, Block, Link from rpython.flowspace.model import SpaceOperation, copygraph from rpython.flowspace.model import checkgraph -from rpython.flowspace.model import c_last_exception from rpython.translator.backendopt.support import log from rpython.translator.simplify import join_blocks from rpython.translator.unsimplify import varoftype @@ -534,7 +533,7 @@ return None def has_exception_catching(self, catchingframe): - if catchingframe.sourceblock.exitswitch != c_last_exception: + if not catchingframe.sourceblock.canraise: return False else: operations = catchingframe.sourceblock.operations @@ -711,7 +710,7 @@ self.specblock.exitswitch = self.rename_nonvirtual(block.exitswitch, 'exitswitch') links = block.exits - catch_exc = self.specblock.exitswitch == c_last_exception + catch_exc = self.specblock.canraise if not catch_exc and isinstance(self.specblock.exitswitch, Constant): # constant-fold the switch diff --git a/rpython/translator/backendopt/removeassert.py b/rpython/translator/backendopt/removeassert.py --- a/rpython/translator/backendopt/removeassert.py +++ b/rpython/translator/backendopt/removeassert.py @@ -1,4 +1,4 @@ -from rpython.flowspace.model import Constant, checkgraph, c_last_exception +from rpython.flowspace.model import Constant, checkgraph from rpython.rtyper.lltypesystem import lltype from rpython.rtyper.rtyper import LowLevelOpList, inputconst from rpython.translator.backendopt.support import log @@ -59,7 +59,7 @@ if len(exits) <= 1: return False remove_condition = len(exits) == 2 - if block.exitswitch == c_last_exception: + if block.canraise: if link is exits[0]: return False # cannot remove the non-exceptional path else: diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py --- a/rpython/translator/c/funcgen.py +++ b/rpython/translator/c/funcgen.py @@ -3,8 +3,7 @@ from rpython.translator.c.support import cdecl from rpython.translator.c.support import llvalue_from_constant, gen_assignments from rpython.translator.c.support import c_string_constant, barebonearray -from rpython.flowspace.model import Variable, Constant -from rpython.flowspace.model import c_last_exception, copygraph +from rpython.flowspace.model import Variable, Constant, copygraph from rpython.rtyper.lltypesystem.lltype import (Ptr, Void, Bool, Signed, Unsigned, SignedLongLong, Float, UnsignedLongLong, Char, UniChar, ContainerType, Array, FixedSizeArray, ForwardReference, FuncType) @@ -233,7 +232,7 @@ for op in self.gen_link(block.exits[0]): yield op else: - assert block.exitswitch != c_last_exception + assert not block.canraise # block ending in a switch on a value TYPE = self.lltypemap(block.exitswitch) if TYPE == Bool: diff --git a/rpython/translator/exceptiontransform.py b/rpython/translator/exceptiontransform.py --- a/rpython/translator/exceptiontransform.py +++ b/rpython/translator/exceptiontransform.py @@ -3,7 +3,7 @@ from rpython.translator.unsimplify import insert_empty_block, split_block from rpython.translator.backendopt import canraise, inline from rpython.flowspace.model import Block, Constant, Variable, Link, \ - c_last_exception, SpaceOperation, FunctionGraph, mkentrymap + SpaceOperation, FunctionGraph, mkentrymap from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.rtyper.lltypesystem import lloperation from rpython.rtyper.rclass import ll_inst_type @@ -243,7 +243,7 @@ elif block is graph.returnblock: return need_exc_matching, n_gen_exc_checks last_operation = len(block.operations) - 1 - if block.exitswitch == c_last_exception: + if block.canraise: need_exc_matching = True last_operation -= 1 elif (len(block.exits) == 1 and @@ -267,7 +267,7 @@ self.gen_exc_check(block, graph.returnblock, afterblock) n_gen_exc_checks += 1 if need_exc_matching: - assert lastblock.exitswitch == c_last_exception + assert lastblock.canraise if not self.raise_analyzer.can_raise(lastblock.operations[-1]): lastblock.exitswitch = None lastblock.recloseblock(lastblock.exits[0]) diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py --- a/rpython/translator/simplify.py +++ b/rpython/translator/simplify.py @@ -8,8 +8,8 @@ from collections import defaultdict from rpython.tool.algo.unionfind import UnionFind -from rpython.flowspace.model import (Variable, Constant, - c_last_exception, checkgraph, mkentrymap) +from rpython.flowspace.model import ( + Variable, Constant, checkgraph, mkentrymap) from rpython.flowspace.operation import OverflowingOperation, op from rpython.rlib import rarithmetic from rpython.translator import unsimplify @@ -43,8 +43,6 @@ def replace_exitswitch_by_constant(block, const): - assert isinstance(const, Constant) - assert const != c_last_exception newexits = [link for link in block.exits if link.exitcase == const.value] if len(newexits) == 0: @@ -133,13 +131,12 @@ chain of is_/issubtype tests. We collapse them all into the block's single list of exits. """ - clastexc = c_last_exception renaming = {} def rename(v): return renaming.get(v, v) for block in graph.iterblocks(): - if not (block.exitswitch == clastexc + if not (block.canraise and block.exits[-1].exitcase is Exception): continue covered = [link.exitcase for link in block.exits[1:-1]] @@ -199,8 +196,8 @@ def transform_xxxitem(graph): # xxx setitem too for block in graph.iterblocks(): - if block.operations and block.exitswitch == c_last_exception: - last_op = block.operations[-1] + if block.canraise: + last_op = block.raising_op if last_op.opname == 'getitem': postfx = [] for exit in block.exits: @@ -217,9 +214,6 @@ def remove_dead_exceptions(graph): """Exceptions can be removed if they are unreachable""" - - clastexc = c_last_exception - def issubclassofmember(cls, seq): for member in seq: if member and issubclass(cls, member): @@ -227,7 +221,7 @@ return False for block in list(graph.iterblocks()): - if block.exitswitch != clastexc: + if not block.canraise: continue exits = [] seen = [] @@ -263,7 +257,7 @@ continue source = link.prevblock switch = source.exitswitch - if (isinstance(switch, Constant) and switch != c_last_exception): + if (isinstance(switch, Constant) and not source.canraise): exits = replace_exitswitch_by_constant(source, switch) stack.extend(exits) else: @@ -339,7 +333,8 @@ newexitswitch = rename(link.target.exitswitch) link.prevblock.exitswitch = newexitswitch link.prevblock.recloseblock(*exits) - if isinstance(newexitswitch, Constant) and newexitswitch != c_last_exception: + if (isinstance(newexitswitch, Constant) and + not link.prevblock.canraise): exits = replace_exitswitch_by_constant(link.prevblock, newexitswitch) stack.extend(exits) @@ -363,7 +358,7 @@ # can we remove this exit without breaking the graph? if len(block.exits) < 2: break - if block.exitswitch == c_last_exception: + if block.canraise: if exit.exitcase is None: break if len(block.exits) == 2: @@ -465,12 +460,7 @@ start_blocks = find_start_blocks(graphs) def canremove(op, block): - if op.opname not in CanRemove: - return False - if block.exitswitch != c_last_exception: - return True - # cannot remove the exc-raising operation - return op is not block.operations[-1] + return op.opname in CanRemove and op is not block.raising_op # compute dependencies and an initial read_vars for block in blocks: @@ -538,9 +528,8 @@ if translator is not None: graph = get_graph(op.args[0], translator) if (graph is not None and - has_no_side_effects(translator, graph) and - (block.exitswitch != c_last_exception or - i != len(block.operations)- 1)): + has_no_side_effects(translator, graph) and + op is not block.raising_op): del block.operations[i] # look for output variables never used # warning: this must be completely done *before* we attempt to @@ -764,9 +753,8 @@ # collect relevant operations based on the family of their result for block in graph.iterblocks(): if (len(block.operations) == 1 and - block.operations[0].opname == 'next' and - block.exitswitch == c_last_exception and - len(block.exits) >= 2): + block.operations[0].opname == 'next' and + block.canraise and len(block.exits) >= 2): cases = [link.exitcase for link in block.exits] if None in cases and StopIteration in cases: # it's a straightforward loop start block diff --git a/rpython/translator/transform.py b/rpython/translator/transform.py --- a/rpython/translator/transform.py +++ b/rpython/translator/transform.py @@ -5,9 +5,8 @@ completed. """ -from rpython.flowspace.model import SpaceOperation -from rpython.flowspace.model import Variable, Constant, Link -from rpython.flowspace.model import c_last_exception, checkgraph +from rpython.flowspace.model import ( + SpaceOperation, Variable, Constant, Link, checkgraph) from rpython.annotator import model as annmodel from rpython.rtyper.lltypesystem import lltype @@ -155,7 +154,7 @@ if not block.exits: # oups! cannot reach the end of this block cutoff_alwaysraising_block(self, block) - elif block.exitswitch == c_last_exception: + elif block.canraise: # exceptional exit if block.exits[0].exitcase is not None: # killed the non-exceptional path! @@ -271,4 +270,3 @@ transform_dead_op_vars(ann, block_subset) if ann.translator: checkgraphs(ann, block_subset) - From noreply at buildbot.pypy.org Mon Mar 2 17:05:13 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 2 Mar 2015 17:05:13 +0100 (CET) Subject: [pypy-commit] stmgc c8-card-marking: Support card marking in duhton-c8. Change the "stop" argument passed Message-ID: <20150302160513.E12441C1C96@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-card-marking Changeset: r1687:cba4ee0e9be6 Date: 2015-03-02 16:32 +0100 http://bitbucket.org/pypy/stmgc/changeset/cba4ee0e9be6/ Log: Support card marking in duhton-c8. Change the "stop" argument passed to stmcb_trace_cards() to be always within range. diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -926,17 +926,6 @@ } -char _stm_write_slowpath_card_extra(object_t *obj) -{ - /* the PyPy JIT calls this function directly if it finds that an - array doesn't have the GCFLAG_CARDS_SET */ - bool mark_card = obj_should_use_cards(STM_SEGMENT->segment_base, obj); - write_slowpath_common(obj, mark_card); - return mark_card; - /* XXX likely, this whole function can be removed now */ -} - - void _stm_write_slowpath_card(object_t *obj, uintptr_t index) { dprintf_test(("write_slowpath_card(%p, %lu)\n", @@ -946,7 +935,8 @@ If the object is large enough, ask it to set up the object for card marking instead. */ if (!(obj->stm_flags & GCFLAG_CARDS_SET)) { - char mark_card = _stm_write_slowpath_card_extra(obj); + bool mark_card = obj_should_use_cards(STM_SEGMENT->segment_base, obj); + write_slowpath_common(obj, mark_card); if (!mark_card) return; } diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c --- a/c8/stm/nursery.c +++ b/c8/stm/nursery.c @@ -274,6 +274,13 @@ uintptr_t start = get_card_index_to_index(card_index); uintptr_t stop = get_card_index_to_index(card_index + 1); + if (card_index == last_card_index) { + assert(stop >= size); + stop = size; + } + else { + assert(stop < size); + } dprintf(("trace_cards on %p with start:%lu stop:%lu\n", obj, start, stop)); diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -85,7 +85,6 @@ void _stm_write_slowpath(object_t *); void _stm_write_slowpath_card(object_t *, uintptr_t); -char _stm_write_slowpath_card_extra(object_t *); object_t *_stm_allocate_slowpath(ssize_t); object_t *_stm_allocate_external(ssize_t); void _stm_become_inevitable(const char*); diff --git a/duhton-c8/demo/long_list.duh b/duhton-c8/demo/long_list.duh new file mode 100644 --- /dev/null +++ b/duhton-c8/demo/long_list.duh @@ -0,0 +1,7 @@ + +(setq a (list)) + +(setq i 0) +(while (< i 10000000) + (append a i) + (setq i (+ i 1))) diff --git a/duhton-c8/duhton.h b/duhton-c8/duhton.h --- a/duhton-c8/duhton.h +++ b/duhton-c8/duhton.h @@ -31,6 +31,8 @@ typedef void(*trace_fn)(struct DuObject_s *, void visit(object_t **)); +typedef void(*trace_cards_fn)(struct DuObject_s *, void visit(object_t **), + uintptr_t start, uintptr_t stop); typedef size_t(*bytesize_fn)(struct DuObject_s *); typedef void(*print_fn)(DuObject *); typedef DuObject *(*eval_fn)(DuObject *, DuObject *); @@ -46,6 +48,9 @@ len_fn dt_is_true; len_fn dt_length; bytesize_fn dt_bytesize; + uintptr_t dt_cards_offset; + uintptr_t dt_cards_itemsize; + trace_cards_fn dt_trace_cards; } DuType; /* keep this list in sync with object.c's Du_Types[] */ @@ -181,6 +186,7 @@ #define _du_read1(p1) stm_read((object_t *)(p1)) #define _du_write1(p1) stm_write((object_t *)(p1)) +#define _du_write1_card(p, i) stm_write_card((object_t *)(p), (i)) #define INIT_PREBUILT(p) ((typeof(p))stm_setup_prebuilt((object_t *)(p))) diff --git a/duhton-c8/listobject.c b/duhton-c8/listobject.c --- a/duhton-c8/listobject.c +++ b/duhton-c8/listobject.c @@ -26,6 +26,15 @@ } } +void tuple_trace_cards(struct DuTupleObject_s *ob, void visit(object_t **), + uintptr_t start, uintptr_t stop) +{ + int i; + for (i = start; i < stop; i++) { + visit((object_t **)&ob->ob_items[i]); + } +} + size_t tuple_bytesize(struct DuTupleObject_s *ob) { return sizeof(DuTupleObject) + (ob->ob_capacity - 1) * sizeof(DuObject *); @@ -91,7 +100,7 @@ int i, newcount = olditems->ob_count + 1; if (newcount <= olditems->ob_capacity) { - _du_write1(olditems); + _du_write1_card(olditems, newcount-1); olditems->ob_items[newcount-1] = x; olditems->ob_count = newcount; } else { /* allocate new one */ @@ -144,9 +153,9 @@ _du_read1(ob); DuTupleObject *p = ob->ob_tuple; - _du_write1(p); if (index < 0 || index >= p->ob_count) Du_FatalError("list_set: index out of range"); + _du_write1_card(p, index); p->ob_items[index] = newitem; } @@ -188,6 +197,9 @@ (len_fn)NULL, (len_fn)NULL, (bytesize_fn)tuple_bytesize, + offsetof(struct DuTupleObject_s, ob_items), + sizeof(DuObject *), + (trace_cards_fn)tuple_trace_cards, }; DuType DuList_Type = { diff --git a/duhton-c8/object.c b/duhton-c8/object.c --- a/duhton-c8/object.c +++ b/duhton-c8/object.c @@ -37,16 +37,23 @@ void stmcb_get_card_base_itemsize(struct object_s *obj, uintptr_t offset_itemsize[2]) { - abort(); + DuType *tp = Du_Types[((struct DuObject_s *)obj)->type_id]; + offset_itemsize[0] = tp->dt_cards_offset; + offset_itemsize[1] = tp->dt_cards_itemsize; } void stmcb_trace_cards(struct object_s *obj, void visit(object_t **), uintptr_t start, uintptr_t stop) { - abort(); + DuType *tp = Du_Types[((struct DuObject_s *)obj)->type_id]; + tp->dt_trace_cards((struct DuObject_s *)obj, visit, start, stop); } +long stmcb_obj_supports_cards(struct object_s *obj) +{ + DuType *tp = Du_Types[((struct DuObject_s *)obj)->type_id]; + return tp->dt_trace_cards != NULL; +} + void stmcb_commit_soon(void) { } -long stmcb_obj_supports_cards(struct object_s *obj) {return 0;} - DuObject *DuObject_New(DuType *tp) From noreply at buildbot.pypy.org Mon Mar 2 17:06:04 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 2 Mar 2015 17:06:04 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c8: import stmgc/cba4ee0e9be6 and streamline the code in the JIT. Message-ID: <20150302160604.268271C1C96@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c8 Changeset: r76211:a569a367e940 Date: 2015-03-02 17:05 +0100 http://bitbucket.org/pypy/pypy/changeset/a569a367e940/ Log: import stmgc/cba4ee0e9be6 and streamline the code in the JIT. diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py --- a/rpython/jit/backend/llsupport/assembler.py +++ b/rpython/jit/backend/llsupport/assembler.py @@ -88,8 +88,6 @@ self._build_wb_slowpath(False) self._build_wb_slowpath(True) self._build_wb_slowpath(False, for_frame=True) - if gc_ll_descr.stm: - self._build_stm_wb_card_slowpath(False) # only one of those self.build_frame_realloc_slowpath() if self.cpu.supports_floats: @@ -97,8 +95,6 @@ self._build_failure_recovery(True, withfloats=True) self._build_wb_slowpath(False, withfloats=True) self._build_wb_slowpath(True, withfloats=True) - if gc_ll_descr.stm: - self._build_stm_wb_card_slowpath(True) self._build_propagate_exception_path() if gc_ll_descr.get_malloc_slowpath_addr() is not None: diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -54,7 +54,6 @@ self.malloc_slowpath = 0 self.malloc_slowpath_varsize = 0 self.wb_slowpath = [0, 0, 0, 0, 0] - self.wb_card_slowpath = [0, 0] self.setup_failure_recovery() self.datablockwrapper = None self.stack_check_slowpath = 0 @@ -364,26 +363,6 @@ rawstart = mc.materialize(self.cpu.asmmemmgr, []) self.stack_check_slowpath = rawstart - def _build_stm_wb_card_slowpath(self, withfloats): - mc = codebuf.MachineCodeBlockWrapper() - - self._push_all_regs_to_frame(mc, [], withfloats, callee_only=True) - - mc.MOV_rs(esi.value, WORD) #index - mc.MOV_rs(edi.value, 2*WORD) #obj - - mc.PUSH(r11) # for alignment - func = rstm.adr_write_slowpath_card - mc.CALL(imm(func)) - mc.POP(r11) - - self._pop_all_regs_from_frame(mc, [], withfloats, callee_only=True) - mc.RET16_i(2 * WORD) - - rawstart = mc.materialize(self.cpu.asmmemmgr, []) - self.wb_card_slowpath[withfloats] = rawstart - - def _build_wb_slowpath(self, withcards, withfloats=False, for_frame=False): descr = self.cpu.gc_ll_descr.write_barrier_descr exc0, exc1 = None, None @@ -415,12 +394,14 @@ # is to simply push it on the shadowstack, from its source # location as two extra arguments on the machine stack # (at this point containing: [usual STM_FRAME_FIXED_SIZE] + # [index, only if 'withcards'] # [obj] # [num] # [ref] # [retaddr]) # XXX this should also be done if 'for_frame' is true... - mc.MOV_rs(esi.value, STM_SHADOWSTACK_BASE_OFS + 4 * WORD) + extra = 4 + withcards + mc.MOV_rs(esi.value, STM_SHADOWSTACK_BASE_OFS + extra * WORD) # esi = base address in the shadowstack + 1 # write the marker to [esi - 1] and [esi + 7] mc.MOV_rs(edi.value, 2 * WORD) # [num] @@ -432,6 +413,9 @@ mc.MOV_rs(edi.value, 1 * WORD) # [ref] mc.MOV_mr((self.SEGMENT_NO, esi.value, +7), edi.value) mc.MOV_rs(edi.value, 3 * WORD) # [obj] + if withcards: + mc.MOV_rs(esi.value, 4 * WORD) # [index] + mc.SUB_ri(esp.value, WORD) # re-align the stack elif IS_X86_32: # we have 2 extra words on stack for retval and we pass 1 extra # arg, so we need to substract 2 words @@ -463,13 +447,14 @@ mc.CALL(imm(func)) # if withcards: - # A final TEST8 before the RET, for the caller. Careful to - # not follow this instruction with another one that changes - # the status of the CPU flags! if self.cpu.gc_ll_descr.stm: - mc.TEST8_rr(eax.value | rx86.BYTE_REG_FLAG, - eax.value | rx86.BYTE_REG_FLAG) + # Nothing more to do in this case, except removing the + # stack alignment word pushed above + mc.ADD_ri(esp.value, WORD) else: + # A final TEST8 before the RET, for the caller. Careful to + # not follow this instruction with another one that changes + # the status of the CPU flags! if IS_X86_32: mc.MOV_rs(eax.value, 3*WORD) else: @@ -485,7 +470,7 @@ mc.LEA_rs(esp.value, 2 * WORD) self._pop_all_regs_from_frame(mc, [], withfloats, callee_only=True) if self.cpu.gc_ll_descr.stm: - mc.RET16_i(3 * WORD) + mc.RET16_i((3 + withcards) * WORD) else: mc.RET16_i(WORD) else: @@ -2368,78 +2353,22 @@ # for cond_call_gc_wb_array, also add another fast path: # if GCFLAG_CARDS_SET, then we can just set one bit and be done + loc_index = None if card_marking: if stm: - loc2 = addr_add_const(self.SEGMENT_GC, loc_base, - descr.jit_wb_cards_set_byteofs) - mask2 = descr.jit_wb_cards_set_singlebyte - mc.TEST8(loc2, imm(mask2)) - mc.J_il8(rx86.Conditions['NZ'], 0) # patched later - js_location = mc.get_relative_pos() - else: - # GCFLAG_CARDS_SET is in this byte at 0x80, so this fact can - # been checked by the status flags of the previous TEST8 - mc.J_il8(rx86.Conditions['S'], 0) # patched later - js_location = mc.get_relative_pos() - else: - js_location = 0 - - # Write only a CALL to the helper prepared in advance, passing it as - # argument the address of the structure we are writing into - # (the first argument to COND_CALL_GC_WB). - withfloats = self._regalloc is not None and bool(self._regalloc.xrm.reg_bindings) - helper_num = card_marking - if is_frame: - helper_num = 4 - elif withfloats: - helper_num += 2 - if self.wb_slowpath[helper_num] == 0: # tests only - assert not we_are_translated() - self.cpu.gc_ll_descr.write_barrier_descr = descr - self._build_wb_slowpath(card_marking, - bool(self._regalloc.xrm.reg_bindings)) - assert self.wb_slowpath[helper_num] != 0 - # - if not is_frame: - mc.PUSH(loc_base) - if stm: - # get the num and ref components of the stm_location, and - # push them to the stack. It's 16 bytes, so alignment is - # still ok. The one or three words pushed here are removed - # by the callee. - assert IS_X86_64 - num, ref = self._regalloc.extract_raw_stm_location() - mc.PUSH(imm(num)) - mc.PUSH(imm(ref)) - if is_frame and align_stack: - mc.SUB_ri(esp.value, 16 - WORD) # erase the return address - mc.CALL(imm(self.wb_slowpath[helper_num])) - if is_frame and align_stack: - mc.ADD_ri(esp.value, 16 - WORD) # erase the return address - - if card_marking: - # The helper ends again with a check of the flag in the object. - # So here, we can simply write again a 'JNS', which will be - # taken if GCFLAG_CARDS_SET is still not set. - if stm: - # here it's actually the result of _stm_write_slowpath_card_extra - mc.J_il8(rx86.Conditions['Z'], 0) # patched later - else: - mc.J_il8(rx86.Conditions['NS'], 0) # patched later - jns_location = mc.get_relative_pos() - # - # patch the JS above - offset = mc.get_relative_pos() - js_location - assert 0 < offset <= 127 - mc.overwrite(js_location-1, chr(offset)) - # - # case GCFLAG_CARDS_SET: emit a few instructions to do - # directly the card flag setting - loc_index = arglocs[1] - - if stm: - # if CARD_MARKED, we are done - # (object >> 4) + (index / CARD_SIZE) + 1 + # see stm_write_card() in stmgc.h + # + # implementation idea: + # mov r11, loc_base # the object + # and r11, ~15 # align + # lea r11, [loc_index + r11<<(card_bits-4)] + # shr r11, card_bits + # cmp [r11+1], card_marked + # + # This assumes that the value computed by the "lea" fits + # in 64 bits. It clearly does, because (card_bits-4) is + # at most 3 and both loc_base and loc_index cannot come + # anywhere close to 2 ** 60. # if rstm.CARD_SIZE == 32: card_bits = 5 @@ -2449,23 +2378,8 @@ card_bits = 7 else: raise AssertionError("CARD_SIZE should be 32/64/128") - # - # idea: - # mov r11, loc_base # the object - # and r11, ~15 # align - # lea r11, [loc_index + r11<<(card_bits-4)] - # shr r11, card_bits - # cmp [r11+1], card_marked - # - # this assumes that the value computed up to the - # "shr r11, card_bits" instruction does not overflow - # the (64-card_bits) bound. For now we silently assumes - # that it is the case. As far as I can tell, this is - # clearly true on any reasonable setting (which assume - # that the non-kernel addresses are numbers between 0 - # and 2**X, for X <= 56). - # r11 = X86_64_SCRATCH_REG + loc_index = arglocs[1] if isinstance(loc_index, RegLoc): if isinstance(loc_base, RegLoc): mc.MOV_ri(r11.value, loc_base.value) @@ -2496,17 +2410,76 @@ mc.CMP8_mi((self.SEGMENT_GC, r11.value, 1), rstm.CARD_MARKED) mc.J_il8(rx86.Conditions['E'], 0) # patched later - before_loc = mc.get_relative_pos() - # slowpath: call _stm_write_slowpath_card + js_location = mc.get_relative_pos() + else: + # GCFLAG_CARDS_SET is in this byte at 0x80, so this fact can + # been checked by the status flags of the previous TEST8 + mc.J_il8(rx86.Conditions['S'], 0) # patched later + js_location = mc.get_relative_pos() + else: + js_location = 0 + + # Write only a CALL to the helper prepared in advance, passing it as + # argument the address of the structure we are writing into + # (the first argument to COND_CALL_GC_WB). + withfloats = self._regalloc is not None and bool(self._regalloc.xrm.reg_bindings) + helper_num = card_marking + if is_frame: + helper_num = 4 + elif withfloats: + helper_num += 2 + if self.wb_slowpath[helper_num] == 0: # tests only + assert not we_are_translated() + self.cpu.gc_ll_descr.write_barrier_descr = descr + self._build_wb_slowpath(card_marking, + bool(self._regalloc.xrm.reg_bindings)) + assert self.wb_slowpath[helper_num] != 0 + # + if not is_frame: + if stm: + # get the num and ref components of the stm_location, and + # push them to the stack. The three or four words pushed + # here are removed by the callee. + assert IS_X86_64 + if card_marking: + mc.PUSH(loc_index) mc.PUSH(loc_base) - mc.PUSH(loc_index) - mc.CALL(imm(self.wb_card_slowpath[withfloats])) + num, ref = self._regalloc.extract_raw_stm_location() + mc.PUSH(imm(num)) + mc.PUSH(imm(ref)) + else: + # The word pushed here is removed by the callee. + mc.PUSH(loc_base) + # + if is_frame and align_stack: + mc.SUB_ri(esp.value, 16 - WORD) # erase the return address + mc.CALL(imm(self.wb_slowpath[helper_num])) + if is_frame and align_stack: + mc.ADD_ri(esp.value, 16 - WORD) # erase the return address - offset = mc.get_relative_pos() - before_loc - assert 0 < offset <= 127 - mc.overwrite(before_loc-1, chr(offset)) + if stm and card_marking: + # patch the JE above (at 'js_location') + offset = mc.get_relative_pos() - js_location + assert 0 < offset <= 127 + mc.overwrite(js_location-1, chr(offset)) + # + elif card_marking: + # The helper ends again with a check of the flag in the object. + # So here, we can simply write again a 'JNS', which will be + # taken if GCFLAG_CARDS_SET is still not set. + mc.J_il8(rx86.Conditions['NS'], 0) # patched later + jns_location = mc.get_relative_pos() + # + # patch the JS above + offset = mc.get_relative_pos() - js_location + assert 0 < offset <= 127 + mc.overwrite(js_location-1, chr(offset)) + # + # case GCFLAG_CARDS_SET: emit a few instructions to do + # directly the card flag setting + loc_index = arglocs[1] - elif isinstance(loc_index, RegLoc): + if isinstance(loc_index, RegLoc): if IS_X86_64 and isinstance(loc_base, RegLoc): # copy loc_index into r11 tmp1 = X86_64_SCRATCH_REG diff --git a/rpython/memory/gctransform/stmframework.py b/rpython/memory/gctransform/stmframework.py --- a/rpython/memory/gctransform/stmframework.py +++ b/rpython/memory/gctransform/stmframework.py @@ -6,6 +6,7 @@ BaseFrameworkGCTransformer, BaseRootWalker, sizeofaddr) from rpython.memory.gctypelayout import WEAKREF, WEAKREFPTR from rpython.memory.gc.stmgc import StmGC +from rpython.rlib.debug import ll_assert from rpython.rtyper import rmodel, llannotation from rpython.rtyper.annlowlevel import llhelper from rpython.translator.backendopt.support import var_needsgc @@ -75,7 +76,7 @@ if not gc.has_gcptr_in_varsize(typeid): return # there are cards, but they don't need tracing length = (obj + gc.varsize_offset_to_length(typeid)).signed[0] - stop = min(stop, length) + ll_assert(stop <= length, "trace_cards: stop > length") gc.trace_partial(obj, start, stop, invokecallback, visit_fn) pypy_stmcb_trace_cards.c_name = "pypy_stmcb_trace_cards" self.autoregister_ptrs.append( @@ -190,7 +191,7 @@ def gct_get_write_barrier_from_array_failing_case(self, hop): op = hop.spaceop c_write_slowpath = rmodel.inputconst( - lltype.Signed, rstm.adr_write_slowpath_card_extra) + lltype.Signed, rstm.adr_write_slowpath_card) hop.genop("cast_int_to_ptr", [c_write_slowpath], resultvar=op.result) def gct_gc_can_move(self, hop): diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py --- a/rpython/rlib/rstm.py +++ b/rpython/rlib/rstm.py @@ -28,8 +28,6 @@ adr_segment_base = ( CFlexSymbolic('((long)&STM_SEGMENT->segment_base)')) adr_write_slowpath = CFlexSymbolic('((long)&_stm_write_slowpath)') -adr_write_slowpath_card_extra = ( - CFlexSymbolic('((long)&_stm_write_slowpath_card_extra)')) adr_write_slowpath_card = ( CFlexSymbolic('((long)&_stm_write_slowpath_card)')) diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision --- a/rpython/translator/stm/src_stm/revision +++ b/rpython/translator/stm/src_stm/revision @@ -1,1 +1,1 @@ -8ca689687bfa +cba4ee0e9be6 diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c --- a/rpython/translator/stm/src_stm/stm/core.c +++ b/rpython/translator/stm/src_stm/stm/core.c @@ -926,16 +926,6 @@ } -char _stm_write_slowpath_card_extra(object_t *obj) -{ - /* the PyPy JIT calls this function directly if it finds that an - array doesn't have the GCFLAG_CARDS_SET */ - bool mark_card = obj_should_use_cards(STM_SEGMENT->segment_base, obj); - write_slowpath_common(obj, mark_card); - return mark_card; -} - - void _stm_write_slowpath_card(object_t *obj, uintptr_t index) { dprintf_test(("write_slowpath_card(%p, %lu)\n", @@ -945,7 +935,8 @@ If the object is large enough, ask it to set up the object for card marking instead. */ if (!(obj->stm_flags & GCFLAG_CARDS_SET)) { - char mark_card = _stm_write_slowpath_card_extra(obj); + bool mark_card = obj_should_use_cards(STM_SEGMENT->segment_base, obj); + write_slowpath_common(obj, mark_card); if (!mark_card) return; } @@ -975,22 +966,19 @@ /* Write into the card's lock. This is used by the next minor collection to know what parts of the big object may have changed. We already own the object here or it is an overflow obj. */ - struct stm_read_marker_s *cards = get_read_marker(STM_SEGMENT->segment_base, - (uintptr_t)obj); - uintptr_t card_index = get_index_to_card_index(index); - if (cards[card_index].rm == CARD_MARKED) - return; + stm_read_marker_t *card = (stm_read_marker_t *)(((uintptr_t)obj) >> 4); + card += get_index_to_card_index(index); if (!IS_OVERFLOW_OBJ(STM_PSEGMENT, obj) - && !(cards[card_index].rm == CARD_MARKED - || cards[card_index].rm == STM_SEGMENT->transaction_read_version)) { + && !(card->rm == CARD_MARKED + || card->rm == STM_SEGMENT->transaction_read_version)) { /* need to do the backup slice of the card */ make_bk_slices(obj, false, /* first_call */ index, /* index: only 1 card */ false); /* do_missing_cards */ } - cards[card_index].rm = CARD_MARKED; + card->rm = CARD_MARKED; dprintf(("mark %p index %lu, card:%lu with %d\n", obj, index, get_index_to_card_index(index), CARD_MARKED)); diff --git a/rpython/translator/stm/src_stm/stm/nursery.c b/rpython/translator/stm/src_stm/stm/nursery.c --- a/rpython/translator/stm/src_stm/stm/nursery.c +++ b/rpython/translator/stm/src_stm/stm/nursery.c @@ -274,6 +274,13 @@ uintptr_t start = get_card_index_to_index(card_index); uintptr_t stop = get_card_index_to_index(card_index + 1); + if (card_index == last_card_index) { + assert(stop >= size); + stop = size; + } + else { + assert(stop < size); + } dprintf(("trace_cards on %p with start:%lu stop:%lu\n", obj, start, stop)); diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h --- a/rpython/translator/stm/src_stm/stmgc.h +++ b/rpython/translator/stm/src_stm/stmgc.h @@ -85,7 +85,6 @@ void _stm_write_slowpath(object_t *); void _stm_write_slowpath_card(object_t *, uintptr_t); -char _stm_write_slowpath_card_extra(object_t *); object_t *_stm_allocate_slowpath(ssize_t); object_t *_stm_allocate_external(ssize_t); void _stm_become_inevitable(const char*); @@ -202,8 +201,27 @@ __attribute__((always_inline)) static inline void stm_write_card(object_t *obj, uintptr_t index) { - if (UNLIKELY((obj->stm_flags & _STM_GCFLAG_WRITE_BARRIER) != 0)) - _stm_write_slowpath_card(obj, index); + /* if GCFLAG_WRITE_BARRIER is set, then don't do anything more. */ + if (UNLIKELY((obj->stm_flags & _STM_GCFLAG_WRITE_BARRIER) != 0)) { + + /* GCFLAG_WRITE_BARRIER is not set. This might be because + it's the first time we see a given small array; or it might + be because it's a big array with card marking. In the + latter case we will always reach this point, even if we + already marked the correct card. Based on the idea that it + is actually the most common case, check it here. If the + array doesn't actually use card marking, the following read + is a bit nonsensical, but in a way that should never return + CARD_MARKED by mistake. + */ + stm_read_marker_t *card = (stm_read_marker_t *)(((uintptr_t)obj) >> 4); + card += (index / _STM_CARD_SIZE) + 1; /* get_index_to_card_index() */ + if (card->rm != _STM_CARD_MARKED) { + + /* slow path. */ + _stm_write_slowpath_card(obj, index); + } + } } From noreply at buildbot.pypy.org Mon Mar 2 17:31:09 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 2 Mar 2015 17:31:09 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c8: fix test Message-ID: <20150302163109.0D54F1C04A8@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c8 Changeset: r76212:be7d50706ee0 Date: 2015-03-02 17:30 +0100 http://bitbucket.org/pypy/pypy/changeset/be7d50706ee0/ Log: fix test diff --git a/rpython/jit/backend/llsupport/test/test_gc_integration.py b/rpython/jit/backend/llsupport/test/test_gc_integration.py --- a/rpython/jit/backend/llsupport/test/test_gc_integration.py +++ b/rpython/jit/backend/llsupport/test/test_gc_integration.py @@ -315,13 +315,11 @@ def test_malloc_slowpath(self): def check(frame): - # xxx for now we always have GCMAP_STM_LOCATION, but it should - # be added only if we really have stm in the first place - expected_size = 1 + gcmap.GCMAP_STM_LOCATION + expected_size = 1 idx = 0 if self.cpu.backend_name.startswith('arm'): # jitframe fixed part is larger here - expected_size = 2 + gcmap.GCMAP_STM_LOCATION + expected_size = 2 idx = 1 assert len(frame.jf_gcmap) == expected_size if self.cpu.IS_64_BIT: @@ -357,11 +355,11 @@ def check(frame): x = frame.jf_gcmap if self.cpu.IS_64_BIT: - assert len(x) == 1 + gcmap.GCMAP_STM_LOCATION + assert len(x) == 1 assert (bin(x[0]).count('1') == '0b1111100000000000000001111111011110'.count('1')) else: - assert len(x) == 2 + gcmap.GCMAP_STM_LOCATION + assert len(x) == 2 s = bin(x[0]).count('1') + bin(x[1]).count('1') assert s == 16 # all but two registers + some stuff on stack From noreply at buildbot.pypy.org Mon Mar 2 17:31:10 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 2 Mar 2015 17:31:10 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c8: Kill old test for stmgc-c4 Message-ID: <20150302163110.65B801C04A8@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c8 Changeset: r76213:680637a4b092 Date: 2015-03-02 17:30 +0100 http://bitbucket.org/pypy/pypy/changeset/680637a4b092/ Log: Kill old test for stmgc-c4 diff --git a/rpython/jit/backend/x86/test/test_stm_integration.py b/rpython/jit/backend/x86/test/test_stm_integration.py deleted file mode 100644 --- a/rpython/jit/backend/x86/test/test_stm_integration.py +++ /dev/null @@ -1,1128 +0,0 @@ -import py -from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rstr -from rpython.rtyper import rclass -from rpython.jit.metainterp.history import ResOperation, TargetToken,\ - JitCellToken -from rpython.jit.metainterp.history import (BoxInt, BoxPtr, ConstInt, - ConstPtr, Box, Const, - BasicFailDescr, BasicFinalDescr) -from rpython.jit.backend.detect_cpu import getcpuclass -from rpython.jit.backend.x86.arch import WORD -from rpython.jit.backend.x86.rx86 import fits_in_32bits -from rpython.jit.backend.llsupport import symbolic -from rpython.jit.metainterp.resoperation import rop -from rpython.jit.metainterp.executor import execute -from rpython.jit.backend.test.runner_test import LLtypeBackendTest -from rpython.jit.tool.oparser import parse -from rpython.rtyper.annlowlevel import llhelper -from rpython.jit.backend.llsupport.gc import BarrierDescr -from rpython.jit.backend.llsupport.test.test_gc_integration import ( - GCDescrShadowstackDirect, BaseTestRegalloc, JitFrameDescrs) -from rpython.jit.backend.llsupport import jitframe -from rpython.memory.gc.stmgc import StmGC -from rpython.jit.metainterp import history -from rpython.jit.codewriter.effectinfo import EffectInfo -from rpython.rlib import rgc -from rpython.rtyper.llinterp import LLException -import itertools, sys -import ctypes - -def cast_to_int(obj): - if isinstance(obj, rgc._GcRef): - return rgc.cast_gcref_to_int(obj) - else: - return rffi.cast(lltype.Signed, obj) - -CPU = getcpuclass() - -class MockSTMRootMap(object): - is_shadow_stack = True - is_stm = True - def __init__(self): - TP = rffi.CArray(lltype.Signed) - self.stack = lltype.malloc(TP, 10, flavor='raw') - self.stack_addr = lltype.malloc(TP, 1, - flavor='raw') - self.stack_addr[0] = rffi.cast(lltype.Signed, self.stack) - def register_asm_addr(self, start, mark): - pass - def get_root_stack_top_addr(self): - return rffi.cast(lltype.Signed, self.stack_addr) - -class FakeSTMBarrier(BarrierDescr): - def __init__(self, gc_ll_descr, stmcat, func): - BarrierDescr.__init__(self, gc_ll_descr) - self.stmcat = stmcat - self.returns_modified_object = True - self.B_FUNCPTR_MOD = lltype.Ptr(lltype.FuncType( - [llmemory.Address], llmemory.Address)) - self.write_barrier_fn = llhelper(self.B_FUNCPTR_MOD, func) - def get_barrier_funcptr(self, returns_modified_object): - assert returns_modified_object - return self.write_barrier_fn - def get_barrier_fn(self, cpu, returns_modified_object): - assert returns_modified_object - return self.write_barrier_fn - -# ____________________________________________________________ - -def allocate_protected(TP, n=1, zero=True, tid=124): - obj = lltype.malloc(TP, n=n, zero=zero) - obj.h_tid = rffi.cast(lltype.Unsigned, - StmGC.GCFLAG_OLD|StmGC.GCFLAG_WRITE_BARRIER | tid) - obj.h_revision = rffi.cast(lltype.Signed, -sys.maxint) - return obj - -def allocate_prebuilt(TP, n=1, zero=True, tid=123): - obj = lltype.malloc(TP, n=n, zero=zero) - obj.h_tid = rffi.cast(lltype.Unsigned, StmGC.PREBUILT_FLAGS | tid) - obj.h_revision = rffi.cast(lltype.Signed, StmGC.PREBUILT_REVISION) - return obj - -def jitframe_allocate(frame_info): - frame = allocate_protected(JITFRAME, n=frame_info.jfi_frame_depth, - zero=True) - frame.jf_frame_info = frame_info - return frame - -JITFRAME = lltype.GcStruct( - 'JITFRAME', - ('h_tid', lltype.Unsigned), - ('h_revision', lltype.Signed), - ('h_original', lltype.Unsigned), - ('jf_frame_info', lltype.Ptr(jitframe.JITFRAMEINFO)), - ('jf_descr', llmemory.GCREF), - ('jf_force_descr', llmemory.GCREF), - ('jf_extra_stack_depth', lltype.Signed), - ('jf_guard_exc', llmemory.GCREF), - ('jf_gcmap', lltype.Ptr(jitframe.GCMAP)), - ('jf_gc_trace_state', lltype.Signed), - ('jf_frame', lltype.Array(lltype.Signed)), - adtmeths = { - 'allocate': jitframe_allocate, - }, -) - -JITFRAMEPTR = lltype.Ptr(JITFRAME) - -class FakeGCHeaderBuilder: - size_gc_header = WORD - -class fakellop: - PRIV_REV = 66 - def __init__(self): - self.TP = rffi.CArray(lltype.Signed) - self.privrevp = lltype.malloc(self.TP, n=1, flavor='raw', - track_allocation=False, zero=True) - self.privrevp[0] = fakellop.PRIV_REV - - entries = (StmGC.FX_MASK + 1) / WORD - self.read_cache = lltype.malloc(self.TP, n=entries, flavor='raw', - track_allocation=False, zero=True) - self.read_cache_adr = lltype.malloc(self.TP, 1, flavor='raw', - track_allocation=False) - self.read_cache_adr[0] = rffi.cast(lltype.Signed, self.read_cache) - - def set_cache_item(self, obj, value): - obj_int = rffi.cast(lltype.Signed, obj) - idx = (obj_int & StmGC.FX_MASK) / WORD - self.read_cache[idx] = rffi.cast(lltype.Signed, value) - - def stm_get_adr_of_private_rev_num(self, _): - return self.privrevp - - def stm_get_adr_of_read_barrier_cache(self, _): - return self.read_cache_adr - -class GCDescrStm(GCDescrShadowstackDirect): - def __init__(self): - GCDescrShadowstackDirect.__init__(self) - self.gcrootmap = MockSTMRootMap() - self.gcheaderbuilder = FakeGCHeaderBuilder() - self.write_barrier_descr = None - self.llop1 = None - self.rb_called_on = [] - self.wb_called_on = [] - self.ptr_eq_called_on = [] - self.stm = True - - def read_barrier(obj): - self.rb_called_on.append(obj) - return obj - def write_barrier(obj): - self.wb_called_on.append(obj) - return obj - - self.A2Rdescr = FakeSTMBarrier(self, 'A2R', read_barrier) - self.A2Idescr = FakeSTMBarrier(self, 'A2I', read_barrier) - self.Q2Rdescr = FakeSTMBarrier(self, 'Q2R', read_barrier) - self.A2Wdescr = FakeSTMBarrier(self, 'A2W', write_barrier) - self.A2Vdescr = FakeSTMBarrier(self, 'A2V', write_barrier) - self.V2Wdescr = FakeSTMBarrier(self, 'V2W', write_barrier) - - self.do_write_barrier = None - self.get_nursery_top_addr = None - self.get_nursery_free_addr = None - - def malloc_str(length): - assert False - self.generate_function('malloc_str', malloc_str, - [lltype.Signed]) - def malloc_unicode(length): - assert False - self.generate_function('malloc_unicode', malloc_unicode, - [lltype.Signed]) - def inevitable(): - pass - self.generate_function('stm_try_inevitable', - inevitable, [], - RESULT=lltype.Void) - def ptr_eq(x, y): - print "=== ptr_eq", hex(cast_to_int(x)), hex(cast_to_int(y)) - self.ptr_eq_called_on.append((cast_to_int(x), cast_to_int(y))) - return x == y - self.generate_function('stm_ptr_eq', ptr_eq, [llmemory.GCREF] * 2, - RESULT=lltype.Bool) - - def stm_allocate_nonmovable_int_adr(obj): - assert False # should not be reached - return rgc.cast_gcref_to_int(obj) - self.generate_function('stm_allocate_nonmovable_int_adr', - stm_allocate_nonmovable_int_adr, - [llmemory.GCREF], - RESULT=lltype.Signed) - - def malloc_big_fixedsize(size, tid): - print "malloc:", size, tid - if size > sys.maxint / 2: - # for testing exception - raise LLException(0, 0) - - entries = size + StmGC.GCHDRSIZE - TP = rffi.CArray(lltype.Char) - obj = lltype.malloc(TP, n=entries, flavor='raw', - track_allocation=False, zero=True) - objptr = rffi.cast(StmGC.GCHDRP, obj) - objptr.h_tid = rffi.cast(lltype.Unsigned, - StmGC.GCFLAG_OLD - | StmGC.GCFLAG_WRITE_BARRIER | tid) - objptr.h_revision = rffi.cast(lltype.Signed, -sys.maxint) - print "return:", obj, objptr - return rffi.cast(llmemory.GCREF, objptr) - self.generate_function('malloc_big_fixedsize', malloc_big_fixedsize, - [lltype.Signed] * 2) - - - def malloc_jitframe(self, frame_info): - """ Allocate a new frame, overwritten by tests - """ - frame = JITFRAME.allocate(frame_info) - self.frames.append(frame) - return frame - - def getframedescrs(self, cpu): - descrs = JitFrameDescrs() - descrs.arraydescr = cpu.arraydescrof(JITFRAME) - for name in ['jf_descr', 'jf_guard_exc', 'jf_force_descr', - 'jf_frame_info', 'jf_gcmap', 'jf_extra_stack_depth']: - setattr(descrs, name, cpu.fielddescrof(JITFRAME, name)) - descrs.jfi_frame_depth = cpu.fielddescrof(jitframe.JITFRAMEINFO, - 'jfi_frame_depth') - descrs.jfi_frame_size = cpu.fielddescrof(jitframe.JITFRAMEINFO, - 'jfi_frame_size') - return descrs - - def get_malloc_slowpath_addr(self): - return None - - def clear_lists(self): - self.rb_called_on[:] = [] - self.wb_called_on[:] = [] - self.ptr_eq_called_on[:] = [] - - -class TestGcStm(BaseTestRegalloc): - - def setup_method(self, meth): - cpu = CPU(None, None) - cpu.gc_ll_descr = GCDescrStm() - - def latest_descr(self, deadframe): - deadframe = lltype.cast_opaque_ptr(JITFRAMEPTR, deadframe) - descr = deadframe.jf_descr - res = history.AbstractDescr.show(self, descr) - assert isinstance(res, history.AbstractFailDescr) - return res - import types - cpu.get_latest_descr = types.MethodType(latest_descr, cpu, - cpu.__class__) - - - self.a2wd = cpu.gc_ll_descr.A2Wdescr - self.a2vd = cpu.gc_ll_descr.A2Vdescr - self.v2wd = cpu.gc_ll_descr.V2Wdescr - self.a2rd = cpu.gc_ll_descr.A2Rdescr - self.a2id = cpu.gc_ll_descr.A2Idescr - self.q2rd = cpu.gc_ll_descr.Q2Rdescr - - TP = rffi.CArray(lltype.Signed) - self.priv_rev_num = lltype.malloc(TP, 1, flavor='raw') - self.clear_read_cache() - - cpu.assembler._get_stm_private_rev_num_addr = self.get_priv_rev_num - cpu.assembler._get_stm_read_barrier_cache_addr = self.get_read_cache - - S = lltype.GcForwardReference() - S.become(lltype.GcStruct( - 'S', ('h_tid', lltype.Unsigned), - ('h_revision', lltype.Signed), - ('h_original', lltype.Unsigned))) - cpu.gc_ll_descr.fielddescr_tid = None # not needed - # = cpu.fielddescrof(S, 'h_tid') - self.S = S - self.cpu = cpu - - def teardown_method(self, meth): - rffi.aroundstate._cleanup_() - - def assert_in(self, called_on, args): - for i, ref in enumerate(args): - assert rffi.cast_ptr_to_adr(ref) in called_on - - def assert_not_in(self, called_on, args): - for ref in args: - assert rffi.cast_ptr_to_adr(ref) not in called_on - - def get_priv_rev_num(self): - return rffi.cast(lltype.Signed, self.priv_rev_num) - - def get_read_cache(self): - return rffi.cast(lltype.Signed, self.read_cache_adr) - - def clear_read_cache(self): - TP = rffi.CArray(lltype.Signed) - entries = (StmGC.FX_MASK + 1) / WORD - self.read_cache = lltype.malloc(TP, n=entries, flavor='raw', - track_allocation=False, zero=True) - self.read_cache_adr = lltype.malloc(TP, 1, flavor='raw', - track_allocation=False) - self.read_cache_adr[0] = rffi.cast(lltype.Signed, self.read_cache) - - def set_cache_item(self, obj): - obj_int = rffi.cast(lltype.Signed, obj) - idx = (obj_int & StmGC.FX_MASK) / WORD - self.read_cache[idx] = obj_int - - def allocate_prebuilt_s(self, tid=66): - s = lltype.malloc(self.S, zero=True) - s.h_tid = rffi.cast(lltype.Unsigned, StmGC.PREBUILT_FLAGS | tid) - s.h_revision = rffi.cast(lltype.Signed, StmGC.PREBUILT_REVISION) - return s - - - - def test_gc_read_barrier_fastpath(self): - from rpython.jit.backend.llsupport.gc import STMReadBarrierDescr - descr = STMReadBarrierDescr(self.cpu.gc_ll_descr, 'A2R') - - called = [] - def read(obj): - called.append(obj) - return obj - - functype = lltype.Ptr(lltype.FuncType( - [llmemory.Address], llmemory.Address)) - funcptr = llhelper(functype, read) - descr.b_failing_case_ptr = funcptr - descr.llop1 = fakellop() - - # -------- TEST -------- - for rev in [fakellop.PRIV_REV+4, fakellop.PRIV_REV]: - called[:] = [] - - s = self.allocate_prebuilt_s() - sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s) - s.h_revision = rev - - descr._do_barrier(sgcref, - returns_modified_object=True) - - # check if rev-fastpath worked - if rev == fakellop.PRIV_REV: - # fastpath - self.assert_not_in(called, [sgcref]) - else: - self.assert_in(called, [sgcref]) - - # now check if sgcref in readcache: - called[:] = [] - descr.llop1.set_cache_item(sgcref, sgcref) - descr._do_barrier(sgcref, - returns_modified_object=True) - self.assert_not_in(called, [sgcref]) - descr.llop1.set_cache_item(sgcref, 0) - - - def test_gc_repeat_read_barrier_fastpath(self): - from rpython.jit.backend.llsupport.gc import STMReadBarrierDescr - descr = STMReadBarrierDescr(self.cpu.gc_ll_descr, 'Q2R') - - called = [] - def read(obj): - called.append(obj) - return obj - - functype = lltype.Ptr(lltype.FuncType( - [llmemory.Address], llmemory.Address)) - funcptr = llhelper(functype, read) - descr.b_failing_case_ptr = funcptr - descr.llop1 = fakellop() - - # -------- TEST -------- - for flags in [StmGC.GCFLAG_PUBLIC_TO_PRIVATE|StmGC.GCFLAG_MOVED, 0]: - called[:] = [] - - s = self.allocate_prebuilt_s() - sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s) - s.h_tid |= flags - - descr._do_barrier(sgcref, - returns_modified_object=True) - - # check if rev-fastpath worked - if not flags: - # fastpath - self.assert_not_in(called, [sgcref]) - else: - self.assert_in(called, [sgcref]) - - def test_gc_immutable_read_barrier_fastpath(self): - from rpython.jit.backend.llsupport.gc import STMReadBarrierDescr - descr = STMReadBarrierDescr(self.cpu.gc_ll_descr, 'A2I') - - called = [] - def read(obj): - called.append(obj) - return obj - - functype = lltype.Ptr(lltype.FuncType( - [llmemory.Address], llmemory.Address)) - funcptr = llhelper(functype, read) - descr.b_failing_case_ptr = funcptr - descr.llop1 = fakellop() - - # -------- TEST -------- - for flags in [StmGC.GCFLAG_STUB, 0]: - called[:] = [] - - s = self.allocate_prebuilt_s() - sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s) - s.h_tid |= flags - - descr._do_barrier(sgcref, returns_modified_object=True) - - # check if rev-fastpath worked - if not flags: - # fastpath - self.assert_not_in(called, [sgcref]) - else: - self.assert_in(called, [sgcref]) - - - - def test_gc_write_barrier_fastpath(self): - from rpython.jit.backend.llsupport.gc import STMWriteBarrierDescr - descr = STMWriteBarrierDescr(self.cpu.gc_ll_descr, 'A2W') - - called = [] - def write(obj): - called.append(obj) - return obj - - functype = lltype.Ptr(lltype.FuncType( - [llmemory.Address], llmemory.Address)) - funcptr = llhelper(functype, write) - descr.b_failing_case_ptr = funcptr - descr.llop1 = fakellop() - - # -------- TEST -------- - for rev in [fakellop.PRIV_REV+4, fakellop.PRIV_REV]: - called[:] = [] - - s = self.allocate_prebuilt_s() - sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s) - s.h_revision = rev - - descr._do_barrier(sgcref, - returns_modified_object=True) - - # check if fastpath worked - if rev == fakellop.PRIV_REV: - # fastpath - self.assert_not_in(called, [sgcref]) - else: - self.assert_in(called, [sgcref]) - - # now set WRITE_BARRIER -> always call slowpath - called[:] = [] - s.h_tid |= StmGC.GCFLAG_WRITE_BARRIER - descr._do_barrier(sgcref, - returns_modified_object=True) - self.assert_in(called, [sgcref]) - - def test_gc_repeat_write_barrier_fastpath(self): - from rpython.jit.backend.llsupport.gc import STMWriteBarrierDescr - descr = STMWriteBarrierDescr(self.cpu.gc_ll_descr, 'V2W') - - called = [] - def write(obj): - called.append(obj) - return obj - - functype = lltype.Ptr(lltype.FuncType( - [llmemory.Address], llmemory.Address)) - funcptr = llhelper(functype, write) - descr.b_failing_case_ptr = funcptr - descr.llop1 = fakellop() - - # -------- TEST -------- - s = self.allocate_prebuilt_s() - sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s) - - descr._do_barrier(sgcref, - returns_modified_object=True) - - # fastpath (WRITE_BARRIER not set) - self.assert_not_in(called, [sgcref]) - - # now set WRITE_BARRIER -> always call slowpath - s.h_tid |= StmGC.GCFLAG_WRITE_BARRIER - descr._do_barrier(sgcref, - returns_modified_object=True) - self.assert_in(called, [sgcref]) - - def test_gc_noptr_write_barrier_fastpath(self): - from rpython.jit.backend.llsupport.gc import STMWriteBarrierDescr - descr = STMWriteBarrierDescr(self.cpu.gc_ll_descr, 'A2V') - - called = [] - def write(obj): - called.append(obj) - return obj - - functype = lltype.Ptr(lltype.FuncType( - [llmemory.Address], llmemory.Address)) - funcptr = llhelper(functype, write) - descr.b_failing_case_ptr = funcptr - descr.llop1 = fakellop() - - # -------- TEST -------- - for rev in [fakellop.PRIV_REV+4, fakellop.PRIV_REV]: - called[:] = [] - - s = self.allocate_prebuilt_s() - sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s) - s.h_revision = rev - - descr._do_barrier(sgcref, returns_modified_object=True) - - # check if fastpath worked - if rev == fakellop.PRIV_REV: - # fastpath - self.assert_not_in(called, [sgcref]) - else: - self.assert_in(called, [sgcref]) - - # now set WRITE_BARRIER -> no effect - called[:] = [] - s.h_tid |= StmGC.GCFLAG_WRITE_BARRIER - descr._do_barrier(sgcref, returns_modified_object=True) - if rev == fakellop.PRIV_REV: - # fastpath - self.assert_not_in(called, [sgcref]) - else: - self.assert_in(called, [sgcref]) - - - - def test_read_barrier_fastpath(self): - cpu = self.cpu - cpu.gc_ll_descr.init_nursery(100) - cpu.setup_once() - PRIV_REV = rffi.cast(lltype.Signed, StmGC.PREBUILT_REVISION) - self.priv_rev_num[0] = PRIV_REV - called_on = cpu.gc_ll_descr.rb_called_on - for rev in [PRIV_REV+4, PRIV_REV]: - cpu.gc_ll_descr.clear_lists() - self.clear_read_cache() - - s = self.allocate_prebuilt_s() - sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s) - s.h_revision = rev - - p0 = BoxPtr() - operations = [ - ResOperation(rop.COND_CALL_STM_B, [p0], None, - descr=self.a2rd), - ResOperation(rop.FINISH, [p0], None, - descr=BasicFinalDescr(0)), - ] - inputargs = [p0] - looptoken = JitCellToken() - cpu.compile_loop(None, inputargs, operations, looptoken) - self.cpu.execute_token(looptoken, sgcref) - - # check if rev-fastpath worked - if rev == PRIV_REV: - # fastpath - self.assert_not_in(called_on, [sgcref]) - else: - self.assert_in(called_on, [sgcref]) - - # now add it to the read-cache and check - # that it will never call the read_barrier - cpu.gc_ll_descr.clear_lists() - self.set_cache_item(sgcref) - - self.cpu.execute_token(looptoken, sgcref) - # not called: - assert not called_on - - def test_repeat_read_barrier_fastpath(self): - cpu = self.cpu - cpu.gc_ll_descr.init_nursery(100) - cpu.setup_once() - - called_on = cpu.gc_ll_descr.rb_called_on - for flags in [StmGC.GCFLAG_PUBLIC_TO_PRIVATE|StmGC.GCFLAG_MOVED, 0]: - cpu.gc_ll_descr.clear_lists() - self.clear_read_cache() - - s = self.allocate_prebuilt_s() - sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s) - s.h_tid |= flags - - p0 = BoxPtr() - operations = [ - ResOperation(rop.COND_CALL_STM_B, [p0], None, - descr=self.q2rd), - ResOperation(rop.FINISH, [p0], None, - descr=BasicFinalDescr(0)), - ] - inputargs = [p0] - looptoken = JitCellToken() - cpu.compile_loop(None, inputargs, operations, looptoken) - self.cpu.execute_token(looptoken, sgcref) - - # check if rev-fastpath worked - if not flags: - # fastpath - self.assert_not_in(called_on, [sgcref]) - else: - self.assert_in(called_on, [sgcref]) - - def test_immutable_read_barrier_fastpath(self): - cpu = self.cpu - cpu.gc_ll_descr.init_nursery(100) - cpu.setup_once() - - called_on = cpu.gc_ll_descr.rb_called_on - for flags in [StmGC.GCFLAG_STUB, 0]: - cpu.gc_ll_descr.clear_lists() - self.clear_read_cache() - - s = self.allocate_prebuilt_s() - sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s) - s.h_tid |= flags - - p0 = BoxPtr() - operations = [ - ResOperation(rop.COND_CALL_STM_B, [p0], None, - descr=self.a2id), - ResOperation(rop.FINISH, [p0], None, - descr=BasicFinalDescr(0)), - ] - inputargs = [p0] - looptoken = JitCellToken() - cpu.compile_loop(None, inputargs, operations, looptoken) - self.cpu.execute_token(looptoken, sgcref) - - # check if rev-fastpath worked - if not flags: - # fastpath - self.assert_not_in(called_on, [sgcref]) - else: - self.assert_in(called_on, [sgcref]) - - - - def test_write_barrier_fastpath(self): - cpu = self.cpu - cpu.gc_ll_descr.init_nursery(100) - cpu.setup_once() - PRIV_REV = rffi.cast(lltype.Signed, StmGC.PREBUILT_REVISION) - self.priv_rev_num[0] = PRIV_REV - called_on = cpu.gc_ll_descr.wb_called_on - - for rev in [PRIV_REV+4, PRIV_REV]: - cpu.gc_ll_descr.clear_lists() - - s = self.allocate_prebuilt_s() - sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s) - s.h_revision = rev - - p0 = BoxPtr() - operations = [ - ResOperation(rop.COND_CALL_STM_B, [p0], None, - descr=self.a2wd), - ResOperation(rop.FINISH, [p0], None, - descr=BasicFinalDescr(0)), - ] - inputargs = [p0] - looptoken = JitCellToken() - cpu.compile_loop(None, inputargs, operations, looptoken) - self.cpu.execute_token(looptoken, sgcref) - - # check if rev-fastpath worked - if rev == PRIV_REV: - # fastpath and WRITE_BARRIER not set - self.assert_not_in(called_on, [sgcref]) - else: - self.assert_in(called_on, [sgcref]) - - # now set WRITE_BARRIER -> always call slowpath - cpu.gc_ll_descr.clear_lists() - s.h_tid |= StmGC.GCFLAG_WRITE_BARRIER - self.cpu.execute_token(looptoken, sgcref) - self.assert_in(called_on, [sgcref]) - - def test_repeat_write_barrier_fastpath(self): - cpu = self.cpu - cpu.gc_ll_descr.init_nursery(100) - cpu.setup_once() - - called_on = cpu.gc_ll_descr.wb_called_on - cpu.gc_ll_descr.clear_lists() - - s = self.allocate_prebuilt_s() - sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s) - - p0 = BoxPtr() - operations = [ - ResOperation(rop.COND_CALL_STM_B, [p0], None, - descr=self.v2wd), - ResOperation(rop.FINISH, [p0], None, - descr=BasicFinalDescr(0)), - ] - - inputargs = [p0] - looptoken = JitCellToken() - cpu.compile_loop(None, inputargs, operations, looptoken) - self.cpu.execute_token(looptoken, sgcref) - - # fastpath and WRITE_BARRIER not set - self.assert_not_in(called_on, [sgcref]) - - # now set WRITE_BARRIER -> always call slowpath - cpu.gc_ll_descr.clear_lists() - s.h_tid |= StmGC.GCFLAG_WRITE_BARRIER - self.cpu.execute_token(looptoken, sgcref) - self.assert_in(called_on, [sgcref]) - - def test_noptr_write_barrier_fastpath(self): - cpu = self.cpu - cpu.gc_ll_descr.init_nursery(100) - cpu.setup_once() - PRIV_REV = rffi.cast(lltype.Signed, StmGC.PREBUILT_REVISION) - self.priv_rev_num[0] = PRIV_REV - called_on = cpu.gc_ll_descr.wb_called_on - - for rev in [PRIV_REV+4, PRIV_REV]: - cpu.gc_ll_descr.clear_lists() - - s = self.allocate_prebuilt_s() - sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s) - s.h_revision = rev - - p0 = BoxPtr() - operations = [ - ResOperation(rop.COND_CALL_STM_B, [p0], None, - descr=self.a2vd), - ResOperation(rop.FINISH, [p0], None, - descr=BasicFinalDescr(0)), - ] - inputargs = [p0] - looptoken = JitCellToken() - cpu.compile_loop(None, inputargs, operations, looptoken) - self.cpu.execute_token(looptoken, sgcref) - - # check if rev-fastpath worked - if rev == PRIV_REV: - # fastpath and WRITE_BARRIER not set - self.assert_not_in(called_on, [sgcref]) - else: - self.assert_in(called_on, [sgcref]) - - # now set WRITE_BARRIER -> no effect - cpu.gc_ll_descr.clear_lists() - s.h_tid |= StmGC.GCFLAG_WRITE_BARRIER - self.cpu.execute_token(looptoken, sgcref) - if rev == PRIV_REV: - # fastpath and WRITE_BARRIER not set - self.assert_not_in(called_on, [sgcref]) - else: - self.assert_in(called_on, [sgcref]) - - - def test_ptr_eq_fastpath(self): - cpu = self.cpu - cpu.gc_ll_descr.init_nursery(100) - cpu.setup_once() - called_on = cpu.gc_ll_descr.ptr_eq_called_on - - i0 = BoxInt() - i1 = BoxInt() - sa, sb = (rffi.cast(llmemory.GCREF, self.allocate_prebuilt_s()), - rffi.cast(llmemory.GCREF, self.allocate_prebuilt_s())) - ss = [sa, sa, sb, sb, - lltype.nullptr(llmemory.GCREF.TO), - lltype.nullptr(llmemory.GCREF.TO), - ] - for s1, s2 in itertools.combinations(ss, 2): - ps = [BoxPtr(), BoxPtr(), - ConstPtr(s1), - ConstPtr(s2)] - for p1, p2 in itertools.combinations(ps, 2): - for guard in [None, rop.GUARD_TRUE, rop.GUARD_FALSE, - rop.GUARD_VALUE]: - cpu.gc_ll_descr.clear_lists() - - # BUILD OPERATIONS: - i = i0 - guarddescr = BasicFailDescr() - finaldescr = BasicFinalDescr() - if guard == rop.GUARD_VALUE: - gop = ResOperation(rop.GUARD_VALUE, [p1, p2], None, - descr=guarddescr) - gop.setfailargs([]) - operations = [gop] - i = i1 - else: - operations = [ResOperation(rop.PTR_EQ, [p1, p2], i0)] - if guard is not None: - gop = ResOperation(guard, [i0], None, - descr=guarddescr) - gop.setfailargs([]) - operations.append(gop) - i = i1 - # finish must depend on result of ptr_eq if no guard - # is inbetween (otherwise ptr_eq gets deleted) - # if there is a guard, the result of ptr_eq must not - # be used after it again... -> i - operations.append( - ResOperation(rop.FINISH, [i], None, - descr=finaldescr) - ) - print operations - - - # COMPILE & EXECUTE LOOP: - inputargs = [p for p in (p1, p2) - if not isinstance(p, Const)] - looptoken = JitCellToken() - c_loop = cpu.compile_loop(None, inputargs + [i1], - operations, looptoken) - - args = [s for i, s in enumerate((s1, s2)) - if not isinstance((p1, p2)[i], Const)] + [7] - - deadframe = self.cpu.execute_token(looptoken, *args) - frame = rffi.cast(JITFRAMEPTR, deadframe) - frame_adr = rffi.cast(lltype.Signed, frame.jf_descr) - guard_failed = frame_adr != id(finaldescr) - - # CHECK: - a, b = cast_to_int(s1), cast_to_int(s2) - if isinstance(p1, Const): - a = cast_to_int(p1.value) - if isinstance(p2, Const): - b = cast_to_int(p2.value) - - # XXX: there is now no function being called in the - # slowpath, so we can't check if fast- vs. slowpath - # works :/ - - # if a == b or a == 0 or b == 0: - # assert (a, b) not in called_on - # assert (b, a) not in called_on - # else: - # assert ([(a, b)] == called_on - # or [(b, a)] == called_on) - - if guard is not None: - if a == b: - if guard in (rop.GUARD_TRUE, rop.GUARD_VALUE): - assert not guard_failed - else: - assert guard_failed - elif guard == rop.GUARD_FALSE: - assert not guard_failed - else: - assert guard_failed - - - - - def test_assembler_call(self): - cpu = self.cpu - cpu.gc_ll_descr.init_nursery(100) - cpu.setup_once() - - called = [] - def assembler_helper(deadframe, virtualizable): - frame = rffi.cast(JITFRAMEPTR, deadframe) - frame_adr = rffi.cast(lltype.Signed, frame.jf_descr) - called.append(frame_adr) - return 4 + 9 - - FUNCPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF, - llmemory.GCREF], - lltype.Signed)) - class FakeJitDriverSD: - index_of_virtualizable = -1 - _assembler_helper_ptr = llhelper(FUNCPTR, assembler_helper) - assembler_helper_adr = llmemory.cast_ptr_to_adr( - _assembler_helper_ptr) - - ops = ''' - [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9] - i10 = int_add(i0, i1) - i11 = int_add(i10, i2) - i12 = int_add(i11, i3) - i13 = int_add(i12, i4) - i14 = int_add(i13, i5) - i15 = int_add(i14, i6) - i16 = int_add(i15, i7) - i17 = int_add(i16, i8) - i18 = int_add(i17, i9) - finish(i18)''' - loop = parse(ops) - looptoken = JitCellToken() - looptoken.outermost_jitdriver_sd = FakeJitDriverSD() - finish_descr = loop.operations[-1].getdescr() - self.cpu.done_with_this_frame_descr_int = BasicFinalDescr() - self.cpu.compile_loop(None, loop.inputargs, loop.operations, looptoken) - ARGS = [lltype.Signed] * 10 - RES = lltype.Signed - FakeJitDriverSD.portal_calldescr = self.cpu.calldescrof( - lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES, - EffectInfo.MOST_GENERAL) - args = [i+1 for i in range(10)] - deadframe = self.cpu.execute_token(looptoken, *args) - - ops = ''' - [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9] - i10 = int_add(i0, 42) - i11 = call_assembler(i10, i1, i2, i3, i4, i5, i6, i7, i8, i9, descr=looptoken) - guard_not_forced()[] - finish(i11) - ''' - loop = parse(ops, namespace=locals()) - othertoken = JitCellToken() - self.cpu.compile_loop(None, loop.inputargs, loop.operations, - othertoken) - args = [i+1 for i in range(10)] - deadframe = self.cpu.execute_token(othertoken, *args) - assert called == [id(finish_descr)] - del called[:] - - # compile a replacement - ops = ''' - [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9] - i10 = int_sub(i0, i1) - i11 = int_sub(i10, i2) - i12 = int_sub(i11, i3) - i13 = int_sub(i12, i4) - i14 = int_sub(i13, i5) - i15 = int_sub(i14, i6) - i16 = int_sub(i15, i7) - i17 = int_sub(i16, i8) - i18 = int_sub(i17, i9) - finish(i18)''' - loop2 = parse(ops) - looptoken2 = JitCellToken() - looptoken2.outermost_jitdriver_sd = FakeJitDriverSD() - self.cpu.compile_loop(None, loop2.inputargs, loop2.operations, - looptoken2) - finish_descr2 = loop2.operations[-1].getdescr() - - # install it - self.cpu.redirect_call_assembler(looptoken, looptoken2) - - # now call_assembler should go to looptoken2 - args = [i+1 for i in range(10)] - deadframe = self.cpu.execute_token(othertoken, *args) - assert called == [id(finish_descr2)] - - - def test_call_malloc_gc(self): - cpu = self.cpu - cpu.gc_ll_descr.init_nursery(100) - cpu.setup_once() - - size = WORD*3 - addr = cpu.gc_ll_descr.get_malloc_fn_addr('malloc_big_fixedsize') - typeid = 11 - descr = cpu.gc_ll_descr.malloc_big_fixedsize_descr - - p0 = BoxPtr() - ops1 = [ResOperation(rop.CALL_MALLOC_GC, - [ConstInt(addr), ConstInt(size), ConstInt(typeid)], - p0, descr), - ResOperation(rop.FINISH, [p0], None, - descr=BasicFinalDescr(0)), - ] - - inputargs = [] - looptoken = JitCellToken() - c_loop = cpu.compile_loop(None, inputargs, ops1, - looptoken) - - args = [] - - frame = self.cpu.execute_token(looptoken, *args) - - - def test_assembler_call_propagate_exc(self): - cpu = self.cpu - cpu._setup_descrs() - cpu.gc_ll_descr.init_nursery(100) - - excdescr = BasicFailDescr(666) - cpu.propagate_exception_descr = excdescr - cpu.setup_once() # xxx redo it, because we added - # propagate_exception - - def assembler_helper(deadframe, virtualizable): - #assert cpu.get_latest_descr(deadframe) is excdescr - # let's assume we handled that - return 3 - - FUNCPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF, - llmemory.GCREF], - lltype.Signed)) - class FakeJitDriverSD: - index_of_virtualizable = -1 - _assembler_helper_ptr = llhelper(FUNCPTR, assembler_helper) - assembler_helper_adr = llmemory.cast_ptr_to_adr( - _assembler_helper_ptr) - - - - addr = cpu.gc_ll_descr.get_malloc_fn_addr('malloc_big_fixedsize') - typeid = 11 - descr = cpu.gc_ll_descr.malloc_big_fixedsize_descr - - p0 = BoxPtr() - i0 = BoxInt() - ops = [ResOperation(rop.CALL_MALLOC_GC, - [ConstInt(addr), i0, ConstInt(typeid)], - p0, descr), - ResOperation(rop.FINISH, [p0], None, - descr=BasicFinalDescr(0)), - ] - - inputargs = [i0] - looptoken = JitCellToken() - looptoken.outermost_jitdriver_sd = FakeJitDriverSD() - c_loop = cpu.compile_loop(None, inputargs, ops, looptoken) - - ARGS = [lltype.Signed] * 10 - RES = lltype.Signed - FakeJitDriverSD.portal_calldescr = cpu.calldescrof( - lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES, - EffectInfo.MOST_GENERAL) - i1 = ConstInt(sys.maxint - 1) - i2 = BoxInt() - finaldescr = BasicFinalDescr(1) - not_forced = ResOperation(rop.GUARD_NOT_FORCED, [], None, - descr=BasicFailDescr(1)) - not_forced.setfailargs([]) - no_exception = ResOperation(rop.GUARD_NO_EXCEPTION, [], None, - descr=BasicFailDescr(2)) - no_exception.setfailargs([]) - ops = [ResOperation(rop.CALL_ASSEMBLER, [i1], i2, descr=looptoken), - not_forced, - no_exception, - ResOperation(rop.FINISH, [i1], None, descr=finaldescr), - ] - othertoken = JitCellToken() - cpu.done_with_this_frame_descr_int = BasicFinalDescr() - c_loop = cpu.compile_loop(None, [], ops, othertoken) - - deadframe = cpu.execute_token(othertoken) - frame = rffi.cast(JITFRAMEPTR, deadframe) - descr = rffi.cast(lltype.Signed, frame.jf_descr) - assert descr != id(finaldescr) - - - def test_write_barrier_on_spilled(self): - cpu = self.cpu - - PRIV_REV = rffi.cast(lltype.Signed, StmGC.PREBUILT_REVISION) - self.priv_rev_num[0] = PRIV_REV - - s = self.allocate_prebuilt_s() - other_s = self.allocate_prebuilt_s() - sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s) - other_sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, other_s) - s.h_revision = PRIV_REV+4 - other_s.h_revision = PRIV_REV+4 - - called_on = [] - def write_barrier(obj): - called_on.append(obj) - if llmemory.cast_ptr_to_adr(sgcref) == obj: - return rffi.cast(llmemory.Address, other_sgcref) - return obj - A2W = FakeSTMBarrier(cpu.gc_ll_descr, 'A2W', write_barrier) - old_a2w = cpu.gc_ll_descr.A2Wdescr - cpu.gc_ll_descr.A2Wdescr = A2W - - cpu.gc_ll_descr.init_nursery(100) - cpu.setup_once() - - - from rpython.jit.tool.oparser import FORCE_SPILL - p0 = BoxPtr() - spill = FORCE_SPILL(None) - spill.initarglist([p0]) - operations = [ - ResOperation(rop.COND_CALL_STM_B, [p0], None, - descr=A2W), - spill, - ResOperation(rop.COND_CALL_STM_B, [p0], None, - descr=A2W), - ResOperation(rop.FINISH, [p0], None, - descr=BasicFinalDescr(0)), - ] - inputargs = [p0] - looptoken = JitCellToken() - print cpu.compile_loop(None, inputargs, operations, looptoken) - cpu.execute_token(looptoken, sgcref) - - # the second write-barrier must see the result of the - # first one - self.assert_in(called_on, [sgcref, other_sgcref]) - - # for other tests: - cpu.gc_ll_descr.A2Wdescr = old_a2w - - - - - - - - - From noreply at buildbot.pypy.org Mon Mar 2 17:33:45 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 2 Mar 2015 17:33:45 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c8: fix test Message-ID: <20150302163345.4B3E31C0282@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c8 Changeset: r76214:7a3eb00131ea Date: 2015-03-02 17:32 +0100 http://bitbucket.org/pypy/pypy/changeset/7a3eb00131ea/ Log: fix test diff --git a/rpython/jit/backend/llsupport/test/test_gc.py b/rpython/jit/backend/llsupport/test/test_gc.py --- a/rpython/jit/backend/llsupport/test/test_gc.py +++ b/rpython/jit/backend/llsupport/test/test_gc.py @@ -188,7 +188,7 @@ rewriter = GcRewriterAssembler(gc_ll_descr, None) newops = rewriter.newops v_base = BoxPtr() - rewriter.gen_write_barrier(v_base, stm_location=None) + rewriter.gen_write_barrier(v_base) assert llop1.record == [] assert len(newops) == 1 assert newops[0].getopnum() == rop.COND_CALL_GC_WB @@ -246,8 +246,7 @@ frame_info = lltype.malloc(jitframe.JITFRAMEINFO, zero=True, flavor='raw') frame = lltype.malloc(jitframe.JITFRAME, 200, zero=True) frame.jf_frame_info = frame_info - frame.jf_gcmap = lltype.malloc(jitframe.GCMAP, 4 + gcmap.GCMAP_STM_LOCATION, - flavor='raw') + frame.jf_gcmap = lltype.malloc(jitframe.GCMAP, 4, flavor='raw') if sys.maxint == 2**31 - 1: max = r_uint(2 ** 31) else: From noreply at buildbot.pypy.org Mon Mar 2 17:33:46 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 2 Mar 2015 17:33:46 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c8: fix test Message-ID: <20150302163346.A0B921C0282@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c8 Changeset: r76215:ea5e45a5384f Date: 2015-03-02 17:33 +0100 http://bitbucket.org/pypy/pypy/changeset/ea5e45a5384f/ Log: fix test diff --git a/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py b/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py --- a/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py +++ b/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py @@ -109,6 +109,7 @@ gcrootfinder = 'asmgcc' gctransformer = 'framework' gcremovetypeptr = False + stm = False gcdescr = get_description(config_) self.gc_ll_descr = GcLLDescr_framework(gcdescr, None, None, None, really_not_translated=True) From noreply at buildbot.pypy.org Mon Mar 2 18:05:01 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 2 Mar 2015 18:05:01 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c8: skip this test for now Message-ID: <20150302170501.2680A1C0755@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c8 Changeset: r76217:550b001baf4d Date: 2015-03-02 17:45 +0100 http://bitbucket.org/pypy/pypy/changeset/550b001baf4d/ Log: skip this test for now diff --git a/rpython/translator/stm/test/test_ztranslated.py b/rpython/translator/stm/test/test_ztranslated.py --- a/rpython/translator/stm/test/test_ztranslated.py +++ b/rpython/translator/stm/test/test_ztranslated.py @@ -611,6 +611,7 @@ assert 'ok!\n' in data def test_allocate_preexisting(self): + py.test.skip("kill me or re-add me") S = lltype.GcStruct('S', ('n', lltype.Signed)) def main(argv): From noreply at buildbot.pypy.org Mon Mar 2 18:04:59 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 2 Mar 2015 18:04:59 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c8: Move this test into its own file (where it can run in parallel) Message-ID: <20150302170459.B5AF11C0755@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c8 Changeset: r76216:d7d6579da261 Date: 2015-03-02 17:34 +0100 http://bitbucket.org/pypy/pypy/changeset/d7d6579da261/ Log: Move this test into its own file (where it can run in parallel) diff --git a/rpython/jit/backend/x86/test/test_zrpy_gc.py b/rpython/jit/backend/x86/test/test_zrpy_gc.py --- a/rpython/jit/backend/x86/test/test_zrpy_gc.py +++ b/rpython/jit/backend/x86/test/test_zrpy_gc.py @@ -1,10 +1,6 @@ from rpython.jit.backend.llsupport.test.zrpy_gc_test import CompileFrameworkTests -class TestSTMShadowStack(CompileFrameworkTests): - gcrootfinder = "stm" - - class TestShadowStack(CompileFrameworkTests): gcrootfinder = "shadowstack" gc = "incminimark" diff --git a/rpython/jit/backend/x86/test/test_zrpy_stm.py b/rpython/jit/backend/x86/test/test_zrpy_stm.py new file mode 100644 --- /dev/null +++ b/rpython/jit/backend/x86/test/test_zrpy_stm.py @@ -0,0 +1,5 @@ +from rpython.jit.backend.llsupport.test.zrpy_gc_test import CompileFrameworkTests + + +class TestSTMShadowStack(CompileFrameworkTests): + gcrootfinder = "stm" From noreply at buildbot.pypy.org Mon Mar 2 18:05:02 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 2 Mar 2015 18:05:02 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c8: fix Message-ID: <20150302170502.601DC1C0755@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c8 Changeset: r76218:2157c7589dd6 Date: 2015-03-02 17:50 +0100 http://bitbucket.org/pypy/pypy/changeset/2157c7589dd6/ Log: fix diff --git a/rpython/translator/c/src/debug_print.h b/rpython/translator/c/src/debug_print.h --- a/rpython/translator/c/src/debug_print.h +++ b/rpython/translator/c/src/debug_print.h @@ -34,7 +34,7 @@ #define __thread_if_stm /* nothing */ #endif -RPY_EXTERN __thread struct pypy_ExcData0 pypy_g_ExcData; +RPY_EXTERN __thread_if_stm struct pypy_ExcData0 pypy_g_ExcData; #define pypy_have_debug_prints pypy_g_ExcData.ed_have_debug_prints /* macros used by the generated code */ From noreply at buildbot.pypy.org Mon Mar 2 18:05:03 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 2 Mar 2015 18:05:03 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c8: wha Message-ID: <20150302170503.9C5C31C0755@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c8 Changeset: r76219:8b799238f219 Date: 2015-03-02 17:58 +0100 http://bitbucket.org/pypy/pypy/changeset/8b799238f219/ Log: wha diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -2382,7 +2382,7 @@ loc_index = arglocs[1] if isinstance(loc_index, RegLoc): if isinstance(loc_base, RegLoc): - mc.MOV_ri(r11.value, loc_base.value) + mc.MOV_rr(r11.value, loc_base.value) mc.AND_ri(r11.value, ~15) else: assert isinstance(loc_base, ImmedLoc) From noreply at buildbot.pypy.org Mon Mar 2 18:09:39 2015 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 2 Mar 2015 18:09:39 +0100 (CET) Subject: [pypy-commit] pypy recent-pure-ops: a bit random progress Message-ID: <20150302170939.C92FE1C0096@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: recent-pure-ops Changeset: r76220:0b3667fb138c Date: 2015-02-28 16:08 +0200 http://bitbucket.org/pypy/pypy/changeset/0b3667fb138c/ Log: a bit random progress diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -452,6 +452,9 @@ def getvalue(self, box): return self.optimizer.getvalue(box) + def get_box_replacement(self, box): + return self.optimizer.get_box_replacement(box) + def make_constant(self, box, constbox): return self.optimizer.make_constant(box, constbox) @@ -845,16 +848,6 @@ descr.make_a_counter_per_value(op) return op - def make_args_key(self, op): - n = op.numargs() - args = [None] * (n + 2) - for i in range(n): - arg = self.get_box_replacement(op.getarg(i)) - args[i] = arg - args[n] = ConstInt(op.getopnum()) - args[n + 1] = op.getdescr() - return args - def optimize_default(self, op): self.emit_operation(op) diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py --- a/rpython/jit/metainterp/optimizeopt/pure.py +++ b/rpython/jit/metainterp/optimizeopt/pure.py @@ -15,31 +15,34 @@ self.next_index = (next_index + 1) % self.REMEMBER_LIMIT self.lst[next_index] = op - def lookup1(self, box0): + def lookup1(self, box0, descr): for i in range(self.REMEMBER_LIMIT): op = self.lst[i] if op is None: break - if op.getarg(0).same_box(box0): + if op.getarg(0).same_box(box0) and op.getdescr() is descr: return op return None - def lookup2(self, box0, box1): + def lookup2(self, box0, box1, descr): for i in range(self.REMEMBER_LIMIT): op = self.lst[i] if op is None: break - if op.getarg(0).same_box(box0) and op.getarg(1).same_box(box1): + if (op.getarg(0).same_box(box0) and op.getarg(1).same_box(box1) + and op.getdescr() is descr): return op return None def lookup(self, optimizer, op): numargs = op.numargs() if numargs == 1: - return self.lookup1(optimizer.get_box_replacement(op.getarg(0))) + return self.lookup1(optimizer.get_box_replacement(op.getarg(0)), + op.getdescr()) elif numargs == 2: return self.lookup2(optimizer.get_box_replacement(op.getarg(0)), - optimizer.get_box_replacement(op.getarg(1))) + optimizer.get_box_replacement(op.getarg(1)), + op.getdescr()) else: assert False @@ -67,7 +70,6 @@ else: nextop = None - args = None if canfold: for i in range(op.numargs()): if self.get_constant_box(op.getarg(i)) is None: @@ -150,17 +152,18 @@ def pure(self, opnum, args, result): op = ResOperation(opnum, args, result) - key = self.optimizer.make_args_key(op) - if key not in self.pure_operations: - self.pure_operations[key] = self.getvalue(result) + recentops = self.getrecentops(opnum) + recentops.add(op) def has_pure_result(self, opnum, args, descr): + return False op = ResOperation(opnum, args, None, descr) key = self.optimizer.make_args_key(op) return self.pure_operations.get(key, None) is not None - def get_pure_result(self, key): - return self.pure_operations.get(key, None) + def get_pure_result(self, op): + recentops = self.getrecentops(op.getopnum()) + return recentops.lookup(self.optimizer, op) def produce_potential_short_preamble_ops(self, sb): ops = sb.optimizer._newoperations diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -26,15 +26,15 @@ def propagate_forward(self, op): if op.boolinverse != -1 or op.boolreflex != -1: - args = self.optimizer.make_args_key(op) - if self.find_rewritable_bool(op, args): + if self.find_rewritable_bool(op): return dispatch_opt(self, op) def try_boolinvers(self, op, targs): - value = self.get_pure_result(targs) - if value is not None: + op = self.get_pure_result(targs) + if op is not None: + value = self.getvalue(op.result) if value.is_constant(): if value.box.same_constant(CONST_1): self.make_constant(op.result, CONST_0) @@ -46,30 +46,30 @@ return False - def find_rewritable_bool(self, op, args): + def find_rewritable_bool(self, op): oldopnum = op.boolinverse + arg0 = op.getarg(0) + arg1 = op.getarg(1) if oldopnum != -1: - targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[0], args[1]], - None)) - if self.try_boolinvers(op, targs): + top = ResOperation(oldopnum, [arg0, arg1], None) + if self.try_boolinvers(op, top): return True oldopnum = op.boolreflex # FIXME: add INT_ADD, INT_MUL if oldopnum != -1: - targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[1], args[0]], - None)) - value = self.get_pure_result(targs) - if value is not None: - self.optimizer.make_equal_to(op.result, value, True) + top = ResOperation(oldopnum, [arg0, arg1], None) + oldop = self.get_pure_result(top) + if oldop is not None: + self.optimizer.make_equal_to(op.result, self.getvalue(oldop), + True) return True if op.boolreflex == -1: return False oldopnum = opclasses[op.boolreflex].boolinverse if oldopnum != -1: - targs = self.optimizer.make_args_key( - ResOperation(oldopnum, [args[1], args[0]], None)) - if self.try_boolinvers(op, targs): + top = ResOperation(oldopnum, [arg1, arg0], None) + if self.try_boolinvers(op, top): return True return False From noreply at buildbot.pypy.org Mon Mar 2 18:25:44 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 2 Mar 2015 18:25:44 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c8: Fix debugging code: it was passing a "TLPREFIX char *" to fprintf Message-ID: <20150302172544.A15191C009F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c8 Changeset: r76221:8b4db8387697 Date: 2015-03-02 18:25 +0100 http://bitbucket.org/pypy/pypy/changeset/8b4db8387697/ Log: Fix debugging code: it was passing a "TLPREFIX char *" to fprintf diff --git a/rpython/translator/c/src/debug_traceback.c b/rpython/translator/c/src/debug_traceback.c --- a/rpython/translator/c/src/debug_traceback.c +++ b/rpython/translator/c/src/debug_traceback.c @@ -65,9 +65,24 @@ void pypy_debug_catch_fatal_exception(void) { - pypy_debug_traceback_print(); - fprintf(stderr, "Fatal RPython error: %.*s\n", - (int)(RPyFetchExceptionType()->ov_name->rs_chars.length), - RPyFetchExceptionType()->ov_name->rs_chars.items); - abort(); + const char *clsname; + pypy_debug_traceback_print(); + fprintf(stderr, "Fatal RPython error: "); +#ifdef RPY_STM + char bufname[128]; + int i; + bufname[127] = 0; + for (i = 0; i < 127; i++) { + bufname[i] = RPyFetchExceptionType()->ov_name->rs_chars.items[i]; + if (bufname[i] == 0) + break; + } + clsname = bufname; +#else + clsname = RPyFetchExceptionType()->ov_name->rs_chars.items; +#endif + fprintf(stderr, "%.*s\n", + (int)(RPyFetchExceptionType()->ov_name->rs_chars.length), + clsname); + abort(); } From noreply at buildbot.pypy.org Mon Mar 2 18:30:21 2015 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 2 Mar 2015 18:30:21 +0100 (CET) Subject: [pypy-commit] pypy recent-pure-ops: give up and call get_op_replacement will be improved on optresult Message-ID: <20150302173021.50D2A1C0271@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: recent-pure-ops Changeset: r76222:0960b8c14703 Date: 2015-03-02 19:29 +0200 http://bitbucket.org/pypy/pypy/changeset/0960b8c14703/ Log: give up and call get_op_replacement will be improved on optresult diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py --- a/rpython/jit/metainterp/optimizeopt/pure.py +++ b/rpython/jit/metainterp/optimizeopt/pure.py @@ -70,6 +70,7 @@ else: nextop = None + save = False if canfold: for i in range(op.numargs()): if self.get_constant_box(op.getarg(i)) is None: @@ -84,20 +85,21 @@ # did we do the exact same operation already? recentops = self.getrecentops(op.getopnum()) + save = True oldop = recentops.lookup(self.optimizer, op) if oldop is not None: - self.optimizer.make_equal_to(op.result, oldop.result, True) + self.optimizer.make_equal_to(op.result, + self.getvalue(oldop.result), True) return # otherwise, the operation remains self.emit_operation(op) if op.returns_bool_result(): self.optimizer.bool_boxes[self.getvalue(op.result)] = None - if canfold: - realop = self.optimizer.getlastop() - if realop is not None: - recentops = self.getrecentops(realop.getopnum()) - recentops.add(realop) + if save: + realop = self.optimizer.get_op_replacement(op) + recentops = self.getrecentops(realop.getopnum()) + recentops.add(realop) if nextop: self.emit_operation(nextop) From noreply at buildbot.pypy.org Mon Mar 2 18:35:27 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 2 Mar 2015 18:35:27 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c8: Don't run the pinning tests with STM, Message-ID: <20150302173527.D91781C009F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c8 Changeset: r76223:6b6e9194be71 Date: 2015-03-02 18:32 +0100 http://bitbucket.org/pypy/pypy/changeset/6b6e9194be71/ Log: Don't run the pinning tests with STM, where the nursery objects cannot be pinned at all. diff --git a/rpython/jit/backend/llsupport/test/zrpy_gc_test.py b/rpython/jit/backend/llsupport/test/zrpy_gc_test.py --- a/rpython/jit/backend/llsupport/test/zrpy_gc_test.py +++ b/rpython/jit/backend/llsupport/test/zrpy_gc_test.py @@ -220,6 +220,12 @@ def run_orig(self, name, n, x): self.main_allfuncs(name, n, x) + def can_pin(self): + """Don't run the pinning tests with STM, + where the nursery objects cannot be pinned at all. + """ + return self.gcrootfinder != 'stm' + class CompileFrameworkTests(BaseFrameworkTests): # Test suite using (so far) the minimark GC. @@ -921,7 +927,8 @@ return None, fn, None def test_pinned_simple(self): - self.run('pinned_simple') + if self.can_pin(): + self.run('pinned_simple') def define_pinned_unpin(cls): class H: @@ -965,7 +972,8 @@ return None, fn, after def test_pinned_unpin(self): - self.run('pinned_unpin') + if self.can_pin(): + self.run('pinned_unpin') def define_multiple_pinned(cls): class H: @@ -1012,4 +1020,5 @@ return None, fn, None def test_multiple_pinned(self): - self.run('multiple_pinned') + if self.can_pin(): + self.run('multiple_pinned') From noreply at buildbot.pypy.org Mon Mar 2 21:02:10 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 2 Mar 2015 21:02:10 +0100 (CET) Subject: [pypy-commit] stmgc c8-card-marking: Simplify further stm_write_card() by assuming that large objects Message-ID: <20150302200210.E9D6D1C0755@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-card-marking Changeset: r1688:f1272b890ba0 Date: 2015-03-02 21:02 +0100 http://bitbucket.org/pypy/stmgc/changeset/f1272b890ba0/ Log: Simplify further stm_write_card() by assuming that large objects with cards are allocated to a multiple of 16. diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -961,6 +961,9 @@ a direct way to know the length. We know that it is smaller than the size in bytes. */ assert(index < size); + /* this object was allocated with allocate_outside_nursery_large(), + which returns addresses aligned to 16 bytes */ + assert((((uintptr_t)obj) & 15) == 0); #endif /* Write into the card's lock. This is used by the next minor diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -52,10 +52,14 @@ static stm_char *allocate_outside_nursery_large(uint64_t size) { - /* Allocate the object with largemalloc.c from the lower addresses. */ - char *addr = _stm_large_malloc(size); + /* Allocate the object with largemalloc.c from the lower + addresses. Round up the size to a multiple of 16, rather than + 8, as a quick way to simplify the code in stm_write_card(). + */ + char *addr = _stm_large_malloc((size + 15) & ~15); if (addr == NULL) stm_fatalerror("not enough memory!"); + assert((((uintptr_t)addr) & 15) == 0); /* alignment check */ if (LIKELY(addr + size <= uninitialized_page_start)) { dprintf(("allocate_outside_nursery_large(%lu): %p, page=%lu\n", diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -79,7 +79,8 @@ #define _STM_CARD_MARKED 1 /* should always be 1... */ #define _STM_GCFLAG_CARDS_SET 0x8 -#define _STM_CARD_SIZE 32 /* must be >= 32 */ +#define _STM_CARD_BITS 5 /* must be 5/6/7 for the pypy jit */ +#define _STM_CARD_SIZE (1 << _STM_CARD_BITS) #define _STM_MIN_CARD_COUNT 17 #define _STM_MIN_CARD_OBJ_SIZE (_STM_CARD_SIZE * _STM_MIN_CARD_COUNT) @@ -213,10 +214,22 @@ array doesn't actually use card marking, the following read is a bit nonsensical, but in a way that should never return CARD_MARKED by mistake. + + The computation of the card marker is further optimized by + assuming that large objects are allocated to multiples of + 16 (rather than just 8, as all objects are). Under this + assumption the following code is equivalent to: + + (obj >> 4) + (index / _STM_CARD_SIZE) + 1 + + The code below however takes only a couple of assembler + instructions. It also assumes that the intermediate value + fits in a 64-bit value, which it clearly does (all values + are much smaller than 2 ** 60). */ - stm_read_marker_t *card = (stm_read_marker_t *)(((uintptr_t)obj) >> 4); - card += (index / _STM_CARD_SIZE) + 1; /* get_index_to_card_index() */ - if (card->rm != _STM_CARD_MARKED) { + uintptr_t v = (((uintptr_t)obj) << (_STM_CARD_BITS - 4)) + index; + stm_read_marker_t *card1 = (stm_read_marker_t *)(v >> _STM_CARD_BITS); + if (card1[1].rm != _STM_CARD_MARKED) { /* slow path. */ _stm_write_slowpath_card(obj, index); diff --git a/c8/test/test_gcpage.py b/c8/test/test_gcpage.py --- a/c8/test/test_gcpage.py +++ b/c8/test/test_gcpage.py @@ -124,15 +124,15 @@ def test_major_collection(self): self.start_transaction() - new = stm_allocate(5000) + new = stm_allocate(5008) self.push_root(new) stm_minor_collect() - assert lib._stm_total_allocated() == 5000 + LMO + assert lib._stm_total_allocated() == 5008 + LMO new = self.pop_root() assert not is_in_nursery(new) stm_minor_collect() - assert lib._stm_total_allocated() == 5000 + LMO + assert lib._stm_total_allocated() == 5008 + LMO stm_major_collect() assert lib._stm_total_allocated() == 0 @@ -143,12 +143,12 @@ assert lib._stm_total_allocated() == CLEO self.start_transaction() - o = stm_allocate(5000) + o = stm_allocate(5008) self.push_root(o) self.commit_transaction() assert last_commit_log_entry_objs() == [] # 2 CLEs, 1 old object - assert lib._stm_total_allocated() == 2*CLEO + (5000 + LMO) + assert lib._stm_total_allocated() == 2*CLEO + (5008 + LMO) self.start_transaction() o = self.pop_root() @@ -158,13 +158,13 @@ assert last_commit_log_entry_objs() == [o]*2 # 3 CLEs, 1 old object # also, 2 slices of bk_copy and thus 2 CLE entries - assert lib._stm_total_allocated() == 3*CLEO + (5000+LMO) + (5000 + CLEEO*2) + assert lib._stm_total_allocated() == 3*CLEO + (5008+LMO) + (5008 + CLEEO*2) self.start_transaction() - assert lib._stm_total_allocated() == 3*CLEO + (5000+LMO) + (5000 + CLEEO*2) + assert lib._stm_total_allocated() == 3*CLEO + (5008+LMO) + (5008 + CLEEO*2) stm_major_collect() # all CLE and CLE entries freed: - assert lib._stm_total_allocated() == (5000+LMO) + assert lib._stm_total_allocated() == (5008+LMO) self.commit_transaction() @@ -180,39 +180,39 @@ return prev self.start_transaction() - self.push_root(make_chain(5000)) - self.push_root(make_chain(4312)) + self.push_root(make_chain(5008)) + self.push_root(make_chain(4304)) stm_minor_collect() - assert lib._stm_total_allocated() == (10 * (5000 + LMO) + - 10 * (4312 + LMO)) + assert lib._stm_total_allocated() == (10 * (5008 + LMO) + + 10 * (4304 + LMO)) stm_major_collect() - assert lib._stm_total_allocated() == (10 * (5000 + LMO) + - 10 * (4312 + LMO)) + assert lib._stm_total_allocated() == (10 * (5008 + LMO) + + 10 * (4304 + LMO)) stm_major_collect() - assert lib._stm_total_allocated() == (10 * (5000 + LMO) + - 10 * (4312 + LMO)) + assert lib._stm_total_allocated() == (10 * (5008 + LMO) + + 10 * (4304 + LMO)) self.pop_root() stm_major_collect() - assert lib._stm_total_allocated() == 10 * (5000 + LMO) + assert lib._stm_total_allocated() == 10 * (5008 + LMO) def test_trace_all_versions(self): self.start_transaction() - x = stm_allocate(5000) + x = stm_allocate(5008) stm_set_char(x, 'A') stm_set_char(x, 'a', 4999) self.push_root(x) self.commit_transaction() - assert lib._stm_total_allocated() == 5000 + LMO + CLEO + assert lib._stm_total_allocated() == 5008 + LMO + CLEO self.start_transaction() x = self.pop_root() self.push_root(x) - assert lib._stm_total_allocated() == 5000 + LMO + CLEO + assert lib._stm_total_allocated() == 5008 + LMO + CLEO stm_set_char(x, 'B') stm_set_char(x, 'b', 4999) py.test.skip("we don't account for private pages right now") - assert lib._stm_total_allocated() == 5000 + LMO + 2 * 4096 # 2 pages + assert lib._stm_total_allocated() == 5008 + LMO + 2 * 4096 # 2 pages stm_major_collect() assert stm_get_char(x) == 'B' @@ -226,7 +226,7 @@ self.switch(0) assert stm_get_char(x) == 'B' assert stm_get_char(x, 4999) == 'b' - assert lib._stm_total_allocated() == 5000 + LMO + 2 * 4096 # 2 pages + assert lib._stm_total_allocated() == 5008 + LMO + 2 * 4096 # 2 pages def test_trace_correct_version_of_overflow_objects_1(self, size=32): self.start_transaction() @@ -245,13 +245,13 @@ assert stm_get_char(x, size - 1) == 'E' def test_trace_correct_version_of_overflow_objects_2(self): - self.test_trace_correct_version_of_overflow_objects_1(size=5000) + self.test_trace_correct_version_of_overflow_objects_1(size=5008) def test_reshare_if_no_longer_modified_0(self, invert=0): if invert: self.switch(1) self.start_transaction() - x = stm_allocate(5000) + x = stm_allocate(5008) self.push_root(x) self.commit_transaction() x = self.pop_root() @@ -264,12 +264,12 @@ stm_major_collect() py.test.skip("we don't account for private pages right now") - assert lib._stm_total_allocated() == 5000 + LMO + 2 * 4096 # 2 pages + assert lib._stm_total_allocated() == 5008 + LMO + 2 * 4096 # 2 pages self.commit_transaction() # self.start_transaction() stm_major_collect() - assert lib._stm_total_allocated() == 5000 + LMO # shared again + assert lib._stm_total_allocated() == 5008 + LMO # shared again def test_reshare_if_no_longer_modified_1(self): self.test_reshare_if_no_longer_modified_0(invert=1) @@ -307,7 +307,7 @@ stm_set_char(p2, 't') self.push_root(p2) stm_major_collect() - assert lib._stm_total_allocated() == 2 * 616 + assert lib._stm_total_allocated() == 2 * 624 # p2 = self.pop_root() m = self.pop_root() diff --git a/c8/test/test_nursery.py b/c8/test/test_nursery.py --- a/c8/test/test_nursery.py +++ b/c8/test/test_nursery.py @@ -210,7 +210,7 @@ stm_set_char(p2, 't') self.push_root(p2) stm_minor_collect() - assert lib._stm_total_allocated() == 2 * 616 + assert lib._stm_total_allocated() == 2 * 624 # p2 = self.pop_root() m = self.pop_root() @@ -230,7 +230,7 @@ stm_set_char(p2, 't') self.push_root(p2) stm_minor_collect() - assert lib._stm_total_allocated() == 1 * 616 + assert lib._stm_total_allocated() == 1 * 624 # p2 = self.pop_root() m = self.pop_root() From noreply at buildbot.pypy.org Mon Mar 2 21:22:31 2015 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 2 Mar 2015 21:22:31 +0100 (CET) Subject: [pypy-commit] pypy default: fix for arm Message-ID: <20150302202231.CFDF81C151A@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r76224:0d8099ffa579 Date: 2015-03-02 22:21 +0200 http://bitbucket.org/pypy/pypy/changeset/0d8099ffa579/ Log: fix for arm diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py --- a/rpython/jit/backend/arm/assembler.py +++ b/rpython/jit/backend/arm/assembler.py @@ -553,7 +553,7 @@ debug_stop('jit-backend-ops') def _call_header(self): - assert self.currpos() == 0 + assert self.mc.currpos() == 0 self.gen_func_prolog() def _call_header_with_stack_check(self): From noreply at buildbot.pypy.org Mon Mar 2 21:54:06 2015 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 2 Mar 2015 21:54:06 +0100 (CET) Subject: [pypy-commit] pypy default: test, fix for mmap subtype not writeable; asarray not respecting implementation.start Message-ID: <20150302205406.E0FCA1C009F@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r76225:c5c4df0df240 Date: 2015-03-02 21:05 +0200 http://bitbucket.org/pypy/pypy/changeset/c5c4df0df240/ Log: test, fix for mmap subtype not writeable; asarray not respecting implementation.start diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py --- a/pypy/module/micronumpy/base.py +++ b/pypy/module/micronumpy/base.py @@ -46,7 +46,7 @@ @staticmethod def from_shape_and_storage(space, shape, storage, dtype, storage_bytes=-1, order='C', owning=False, w_subtype=None, - w_base=None, writable=True, strides=None): + w_base=None, writable=True, strides=None, start=0): from pypy.module.micronumpy import concrete from pypy.module.micronumpy.strides import (calc_strides, calc_backstrides) @@ -75,8 +75,9 @@ raise OperationError(space.w_ValueError, space.wrap("Cannot have owning=True when specifying a buffer")) if writable: - impl = concrete.ConcreteArrayWithBase(shape, dtype, order, strides, - backstrides, storage, w_base) + impl = concrete.ConcreteArrayWithBase(shape, dtype, order, + strides, backstrides, storage, w_base, + start=start) else: impl = concrete.ConcreteNonWritableArrayWithBase(shape, dtype, order, strides, backstrides, diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py --- a/pypy/module/micronumpy/concrete.py +++ b/pypy/module/micronumpy/concrete.py @@ -331,7 +331,7 @@ class ConcreteArrayNotOwning(BaseConcreteArray): - def __init__(self, shape, dtype, order, strides, backstrides, storage): + def __init__(self, shape, dtype, order, strides, backstrides, storage, start=0): make_sure_not_resized(shape) make_sure_not_resized(strides) make_sure_not_resized(backstrides) @@ -342,6 +342,7 @@ self.strides = strides self.backstrides = backstrides self.storage = storage + self.start = start def fill(self, space, box): self.dtype.itemtype.fill(self.storage, self.dtype.elsize, @@ -350,7 +351,7 @@ def set_shape(self, space, orig_array, new_shape): strides, backstrides = calc_strides(new_shape, self.dtype, self.order) - return SliceArray(0, strides, backstrides, new_shape, self, + return SliceArray(self.start, strides, backstrides, new_shape, self, orig_array) def set_dtype(self, space, dtype): @@ -384,9 +385,10 @@ class ConcreteArrayWithBase(ConcreteArrayNotOwning): - def __init__(self, shape, dtype, order, strides, backstrides, storage, orig_base): + def __init__(self, shape, dtype, order, strides, backstrides, storage, + orig_base, start=0): ConcreteArrayNotOwning.__init__(self, shape, dtype, order, - strides, backstrides, storage) + strides, backstrides, storage, start) self.orig_base = orig_base def base(self): diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py --- a/pypy/module/micronumpy/ctors.py +++ b/pypy/module/micronumpy/ctors.py @@ -99,10 +99,11 @@ for i in range(w_object.get_size()): elems_w[i] = w_object.implementation.getitem(i * elsize) else: + imp = w_object.implementation sz = support.product(w_object.get_shape()) * dtype.elsize return W_NDimArray.from_shape_and_storage(space, - w_object.get_shape(),w_object.implementation.storage, - dtype, storage_bytes=sz, w_base=w_object) + w_object.get_shape(),imp.storage, dtype, storage_bytes=sz, + w_base=w_object, start=imp.start) else: # not an array shape, elems_w = strides.find_shape_and_elems(space, w_object, dtype) diff --git a/pypy/module/micronumpy/test/test_subtype.py b/pypy/module/micronumpy/test/test_subtype.py --- a/pypy/module/micronumpy/test/test_subtype.py +++ b/pypy/module/micronumpy/test/test_subtype.py @@ -2,7 +2,7 @@ class AppTestSupport(BaseNumpyAppTest): - spaceconfig = dict(usemodules=["micronumpy", "struct", "binascii"]) + spaceconfig = dict(usemodules=["micronumpy", "struct", "binascii", "mmap"]) def setup_class(cls): BaseNumpyAppTest.setup_class.im_func(cls) @@ -476,3 +476,120 @@ a = self.SubType(array([[1, 2], [3, 4]])) b = array(a, subok=False) assert type(b) is ndarray + + def test_numpypy_mmap(self): + # issue #21 on pypy/numpy + from numpy import array, ndarray, arange, dtype as dtypedescr + import mmap + import os.path + from tempfile import mkdtemp + import os.path as path + valid_filemodes = ["r", "c", "r+", "w+"] + writeable_filemodes = ["r+", "w+"] + mode_equivalents = { + "readonly":"r", + "copyonwrite":"c", + "readwrite":"r+", + "write":"w+" + } + + class memmap(ndarray): + def __new__(subtype, filename, dtype='uint8', mode='r+', offset=0, shape=None, order='C'): + # Import here to minimize 'import numpy' overhead + try: + mode = mode_equivalents[mode] + except KeyError: + if mode not in valid_filemodes: + raise ValueError("mode must be one of %s" % + (valid_filemodes + list(mode_equivalents.keys()))) + + if hasattr(filename, 'read'): + fid = filename + own_file = False + else: + fid = open(filename, (mode == 'c' and 'r' or mode)+'b') + own_file = True + + if (mode == 'w+') and shape is None: + raise ValueError("shape must be given") + + fid.seek(0, 2) + flen = fid.tell() + descr = dtypedescr(dtype) + _dbytes = descr.itemsize + + if shape is None: + bytes = flen - offset + if (bytes % _dbytes): + fid.close() + raise ValueError("Size of available data is not a " + "multiple of the data-type size.") + size = bytes // _dbytes + shape = (size,) + else: + if not isinstance(shape, tuple): + shape = (shape,) + size = 1 + for k in shape: + size *= k + + bytes = long(offset + size*_dbytes) + + if mode == 'w+' or (mode == 'r+' and flen < bytes): + fid.seek(bytes - 1, 0) + fid.write('\0') + fid.flush() + + if mode == 'c': + acc = mmap.ACCESS_COPY + elif mode == 'r': + acc = mmap.ACCESS_READ + else: + acc = mmap.ACCESS_WRITE + + start = offset - offset % mmap.ALLOCATIONGRANULARITY + bytes -= start + offset -= start + mm = mmap.mmap(fid.fileno(), bytes, access=acc, offset=start) + + self = ndarray.__new__(subtype, shape, dtype=descr, buffer=mm, + offset=offset, order=order) + self._mmap = mm + self.offset = offset + self.mode = mode + + if isinstance(filename, basestring): + self.filename = os.path.abspath(filename) + # py3 returns int for TemporaryFile().name + elif (hasattr(filename, "name") and + isinstance(filename.name, basestring)): + self.filename = os.path.abspath(filename.name) + # same as memmap copies (e.g. memmap + 1) + else: + self.filename = None + + if own_file: + fid.close() + + return self + + def flush(self): + if self.base is not None and hasattr(self.base, 'flush'): + self.base.flush() + + def asarray(obj, itemsize=None, order=None): + return array(obj, itemsize, copy=False, order=order) + + filename = path.join(mkdtemp(), 'newfile.dat') + data = arange(10*10*36).reshape(10, 10, 36) + fp = memmap(filename, dtype='float32', mode='w+', shape=data.shape) + vals = [ 242, 507, 255, 505, 315, 316, 308, 506, + 309, 255, 211, 505, 315, 316, 308, 506, + 309, 255, 255, 711, 194, 232, 711, 711, + 709, 710, 709, 710, 882, 897, 711, 245, + 711, 711, 168, 245] + fp[:] = data + fp[5:6][:,4] = vals + a = asarray(fp[5:6][:,4]) + assert (a == vals).all() + diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py --- a/pypy/module/mmap/interp_mmap.py +++ b/pypy/module/mmap/interp_mmap.py @@ -22,6 +22,10 @@ self.check_valid() return MMapBuffer(self.space, self.mmap, True) + def writebuf_w(self, space): + self.check_writeable() + return MMapBuffer(self.space, self.mmap, False) + def close(self): self.mmap.close() From noreply at buildbot.pypy.org Mon Mar 2 22:23:29 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 2 Mar 2015 22:23:29 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c8: import stmgc/f1272b890ba0 and further simplify the code written by the jit Message-ID: <20150302212329.0053B1C0096@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c8 Changeset: r76226:001a2796489f Date: 2015-03-02 21:45 +0100 http://bitbucket.org/pypy/pypy/changeset/001a2796489f/ Log: import stmgc/f1272b890ba0 and further simplify the code written by the jit for stm_write_card diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -2356,58 +2356,82 @@ loc_index = None if card_marking: if stm: - # see stm_write_card() in stmgc.h + # see stm_write_card() in stmgc.h. # - # implementation idea: - # mov r11, loc_base # the object - # and r11, ~15 # align - # lea r11, [loc_index + r11<<(card_bits-4)] + # If loc_base and loc_index are both registers: + # lea r11, [loc_index + loc_base<<(card_bits-4)] # shr r11, card_bits # cmp [r11+1], card_marked # - # This assumes that the value computed by the "lea" fits - # in 64 bits. It clearly does, because (card_bits-4) is - # at most 3 and both loc_base and loc_index cannot come - # anywhere close to 2 ** 60. + # If loc_base is a register but loc_index an immediate: + # mov r11, loc_base + # shr r11, 4 + # cmp [r11+(loc_index>>card_bits)+1], card_marked # - if rstm.CARD_SIZE == 32: - card_bits = 5 - elif rstm.CARD_SIZE == 64: - card_bits = 6 - elif rstm.CARD_SIZE == 128: - card_bits = 7 - else: - raise AssertionError("CARD_SIZE should be 32/64/128") + # If the value above does not fit 32 bits, we do instead + # mov r11, loc_index + # (then the rest like the register-register case) + # + # If loc_base is an immediate but loc_index a register: + # mov r11, loc_base<<(card_bits-4) + (1<>4)+(loc_index>>card_bits)+1 + # cmp [r11], card_marked + # + card_bits = rstm.CARD_BITS + assert 5 <= card_bits <= 7 r11 = X86_64_SCRATCH_REG loc_index = arglocs[1] - if isinstance(loc_index, RegLoc): - if isinstance(loc_base, RegLoc): - mc.MOV_rr(r11.value, loc_base.value) - mc.AND_ri(r11.value, ~15) + if isinstance(loc_base, RegLoc): + if isinstance(loc_index, RegLoc): + loc_index_reg = loc_index + add_constant = 1 else: - assert isinstance(loc_base, ImmedLoc) - initial_value = loc_base.value & ~15 - mc.MOV_ri(r11.value, initial_value) - mc.LEA_ra(r11.value, (self.SEGMENT_NO, - loc_index.value, - r11.value, - card_bits - 4, - 0)) - mc.SHR_ri(r11.value, card_bits) + assert isinstance(loc_index, ImmedLoc) + add_constant = (loc_index.value >> card_bits) + 1 + if rx86.fits_in_32bits(add_constant): + mc.MOV_rr(r11.value, loc_base.value) + mc.SHR_ri(r11.value, 4) + loc_index_reg = None + else: + mc.MOV_ri(r11.value, loc_index.value) + loc_index_reg = r11 + add_constant = 1 + if loc_index_reg is not None: + mc.LEA_ra(r11.value, (self.SEGMENT_NO, + loc_index_reg.value, + loc_base.value, + card_bits - 4, + 0)) + mc.SHR_ri(r11.value, card_bits) else: - assert isinstance(loc_index, ImmedLoc) - initial_value = (loc_index.value >> card_bits) << 4 - if isinstance(loc_base, RegLoc): - mc.MOV_ri(r11.value, initial_value) - mc.ADD_rr(r11.value, loc_base.value) - mc.SHR_ri(r11.value, 4) + # xxx we could try to know statically if loc_base + # points to a large object or not, and produce a + # non-card-marking version of the barrier if not + assert isinstance(loc_base, ImmedLoc) + load_value = loc_base.value << (card_bits - 4) + load_value += (1 << card_bits) + if isinstance(loc_index, RegLoc): + add_constant = load_value >> card_bits + if rx86.fits_in_32bits(add_constant): + mc.MOV_rr(r11.value, loc_index.value) + else: + add_constant = 0 + mc.MOV_ri(r11.value, load_value) + mc.ADD_rr(r11.value, loc_index.value) + mc.SHR_ri(r11.value, card_bits) else: - assert isinstance(loc_base, ImmedLoc) - initial_value += loc_base.value - initial_value >>= 4 - mc.MOV_ri(r11.value, initial_value) + assert isinstance(loc_index, ImmedLoc) + load_value += loc_index.value + load_value >>= card_bits + mc.MOV_ri(r11.value, load_value) + add_constant = 0 # - mc.CMP8_mi((self.SEGMENT_GC, r11.value, 1), + mc.CMP8_mi((self.SEGMENT_GC, r11.value, add_constant), rstm.CARD_MARKED) mc.J_il8(rx86.Conditions['E'], 0) # patched later js_location = mc.get_relative_pos() diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py --- a/rpython/rlib/rstm.py +++ b/rpython/rlib/rstm.py @@ -32,6 +32,7 @@ CFlexSymbolic('((long)&_stm_write_slowpath_card)')) CARD_MARKED = CFlexSymbolic('_STM_CARD_MARKED') +CARD_BITS = CFlexSymbolic('_STM_CARD_BITS') CARD_SIZE = CFlexSymbolic('_STM_CARD_SIZE') GCFLAG_CARDS_SET = CFlexSymbolic('_STM_GCFLAG_CARDS_SET') diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision --- a/rpython/translator/stm/src_stm/revision +++ b/rpython/translator/stm/src_stm/revision @@ -1,1 +1,1 @@ -cba4ee0e9be6 +f1272b890ba0 diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c --- a/rpython/translator/stm/src_stm/stm/core.c +++ b/rpython/translator/stm/src_stm/stm/core.c @@ -961,6 +961,9 @@ a direct way to know the length. We know that it is smaller than the size in bytes. */ assert(index < size); + /* this object was allocated with allocate_outside_nursery_large(), + which returns addresses aligned to 16 bytes */ + assert((((uintptr_t)obj) & 15) == 0); #endif /* Write into the card's lock. This is used by the next minor diff --git a/rpython/translator/stm/src_stm/stm/gcpage.c b/rpython/translator/stm/src_stm/stm/gcpage.c --- a/rpython/translator/stm/src_stm/stm/gcpage.c +++ b/rpython/translator/stm/src_stm/stm/gcpage.c @@ -52,10 +52,14 @@ static stm_char *allocate_outside_nursery_large(uint64_t size) { - /* Allocate the object with largemalloc.c from the lower addresses. */ - char *addr = _stm_large_malloc(size); + /* Allocate the object with largemalloc.c from the lower + addresses. Round up the size to a multiple of 16, rather than + 8, as a quick way to simplify the code in stm_write_card(). + */ + char *addr = _stm_large_malloc((size + 15) & ~15); if (addr == NULL) stm_fatalerror("not enough memory!"); + assert((((uintptr_t)addr) & 15) == 0); /* alignment check */ if (LIKELY(addr + size <= uninitialized_page_start)) { dprintf(("allocate_outside_nursery_large(%lu): %p, page=%lu\n", diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h --- a/rpython/translator/stm/src_stm/stmgc.h +++ b/rpython/translator/stm/src_stm/stmgc.h @@ -79,7 +79,8 @@ #define _STM_CARD_MARKED 1 /* should always be 1... */ #define _STM_GCFLAG_CARDS_SET 0x8 -#define _STM_CARD_SIZE 32 /* must be >= 32 */ +#define _STM_CARD_BITS 5 /* must be 5/6/7 for the pypy jit */ +#define _STM_CARD_SIZE (1 << _STM_CARD_BITS) #define _STM_MIN_CARD_COUNT 17 #define _STM_MIN_CARD_OBJ_SIZE (_STM_CARD_SIZE * _STM_MIN_CARD_COUNT) @@ -213,10 +214,22 @@ array doesn't actually use card marking, the following read is a bit nonsensical, but in a way that should never return CARD_MARKED by mistake. + + The computation of the card marker is further optimized by + assuming that large objects are allocated to multiples of + 16 (rather than just 8, as all objects are). Under this + assumption the following code is equivalent to: + + (obj >> 4) + (index / _STM_CARD_SIZE) + 1 + + The code below however takes only a couple of assembler + instructions. It also assumes that the intermediate value + fits in a 64-bit value, which it clearly does (all values + are much smaller than 2 ** 60). */ - stm_read_marker_t *card = (stm_read_marker_t *)(((uintptr_t)obj) >> 4); - card += (index / _STM_CARD_SIZE) + 1; /* get_index_to_card_index() */ - if (card->rm != _STM_CARD_MARKED) { + uintptr_t v = (((uintptr_t)obj) << (_STM_CARD_BITS - 4)) + index; + stm_read_marker_t *card1 = (stm_read_marker_t *)(v >> _STM_CARD_BITS); + if (card1[1].rm != _STM_CARD_MARKED) { /* slow path. */ _stm_write_slowpath_card(obj, index); From noreply at buildbot.pypy.org Mon Mar 2 23:32:56 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 2 Mar 2015 23:32:56 +0100 (CET) Subject: [pypy-commit] pypy py3.3: Populate __qualname__ in type.__new__, this fixes test_class.py Message-ID: <20150302223256.70A0B1C0292@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3.3 Changeset: r76227:fd0caf2bbcc8 Date: 2015-03-02 23:31 +0100 http://bitbucket.org/pypy/pypy/changeset/fd0caf2bbcc8/ Log: Populate __qualname__ in type.__new__, this fixes test_class.py diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py --- a/pypy/objspace/std/typeobject.py +++ b/pypy/objspace/std/typeobject.py @@ -1123,6 +1123,9 @@ hasoldstylebase = copy_flags_from_bases(w_self, w_bestbase) create_all_slots(w_self, hasoldstylebase) + if '__qualname__' in w_self.dict_w: + w_self.qualname = w_self.space.unicode_w(w_self.dict_w['__qualname__']) + ensure_common_attributes(w_self) def setup_builtin_type(w_self): From noreply at buildbot.pypy.org Tue Mar 3 10:49:10 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 3 Mar 2015 10:49:10 +0100 (CET) Subject: [pypy-commit] cffi release-0.9: Branch for the 0.9 release Message-ID: <20150303094910.AF43C1C087E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: release-0.9 Changeset: r1652:9f8613882976 Date: 2015-03-03 10:28 +0100 http://bitbucket.org/cffi/cffi/changeset/9f8613882976/ Log: Branch for the 0.9 release From noreply at buildbot.pypy.org Tue Mar 3 10:49:11 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 3 Mar 2015 10:49:11 +0100 (CET) Subject: [pypy-commit] cffi release-0.9: Prepare release 0.9.0 Message-ID: <20150303094911.DC5871C087E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: release-0.9 Changeset: r1653:ffffd7e45a50 Date: 2015-03-03 10:34 +0100 http://bitbucket.org/cffi/cffi/changeset/ffffd7e45a50/ Log: Prepare release 0.9.0 diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -5689,7 +5689,7 @@ if (v == NULL || PyModule_AddObject(m, "_C_API", v) < 0) INITERROR; - v = PyText_FromString("0.8.6+"); + v = PyText_FromString("0.9.0"); if (v == NULL || PyModule_AddObject(m, "__version__", v) < 0) INITERROR; diff --git a/c/test_c.py b/c/test_c.py --- a/c/test_c.py +++ b/c/test_c.py @@ -3260,4 +3260,4 @@ def test_version(): # this test is here mostly for PyPy - assert __version__ == "0.8.6+" + assert __version__ == "0.9.0" diff --git a/cffi/__init__.py b/cffi/__init__.py --- a/cffi/__init__.py +++ b/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "0.8.6+" -__version_info__ = (0, 8, 6, "plus") +__version__ = "0.9.0" +__version_info__ = (0, 9, 0) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/doc/source/conf.py b/doc/source/conf.py --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -45,9 +45,9 @@ # built documents. # # The short X.Y version. -version = '0.8' +version = '0.9' # The full version, including alpha/beta/rc tags. -release = '0.8.6+' +release = '0.9.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/source/index.rst b/doc/source/index.rst --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -88,13 +88,13 @@ Download and Installation: -* http://pypi.python.org/packages/source/c/cffi/cffi-0.8.6.tar.gz +* http://pypi.python.org/packages/source/c/cffi/cffi-0.9.0.tar.gz - Or grab the most current version by following the instructions below. - - MD5: 474b5a68299a6f05009171de1dc91be6 + - MD5: ... - - SHA: 4e82390201e6f30e9df8a91cd176df19b8f2d547 + - SHA: ... * Or get it from the `Bitbucket page`_: ``hg clone https://bitbucket.org/cffi/cffi`` diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -139,7 +139,7 @@ `Mailing list `_ """, - version='0.8.6', + version='0.9.0', packages=['cffi'], zip_safe=False, From noreply at buildbot.pypy.org Tue Mar 3 10:49:12 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 3 Mar 2015 10:49:12 +0100 (CET) Subject: [pypy-commit] cffi release-0.9: Add the MD5/SHA Message-ID: <20150303094912.E9DC51C087E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: release-0.9 Changeset: r1654:509a04a2cdca Date: 2015-03-03 10:48 +0100 http://bitbucket.org/cffi/cffi/changeset/509a04a2cdca/ Log: Add the MD5/SHA diff --git a/doc/source/index.rst b/doc/source/index.rst --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -92,9 +92,9 @@ - Or grab the most current version by following the instructions below. - - MD5: ... + - MD5: 3a2f6b9f16e8082271aed6dcac51a71a - - SHA: ... + - SHA: 02e44ecada40cb859e18e0b628cc52deba064a39 * Or get it from the `Bitbucket page`_: ``hg clone https://bitbucket.org/cffi/cffi`` From noreply at buildbot.pypy.org Tue Mar 3 10:49:14 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 3 Mar 2015 10:49:14 +0100 (CET) Subject: [pypy-commit] cffi default: hg merge release-0.9 Message-ID: <20150303094914.0F3F11C087E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1655:c98121733586 Date: 2015-03-03 10:48 +0100 http://bitbucket.org/cffi/cffi/changeset/c98121733586/ Log: hg merge release-0.9 diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -5689,7 +5689,7 @@ if (v == NULL || PyModule_AddObject(m, "_C_API", v) < 0) INITERROR; - v = PyText_FromString("0.8.6+"); + v = PyText_FromString("0.9.0"); if (v == NULL || PyModule_AddObject(m, "__version__", v) < 0) INITERROR; diff --git a/c/test_c.py b/c/test_c.py --- a/c/test_c.py +++ b/c/test_c.py @@ -3260,4 +3260,4 @@ def test_version(): # this test is here mostly for PyPy - assert __version__ == "0.8.6+" + assert __version__ == "0.9.0" diff --git a/cffi/__init__.py b/cffi/__init__.py --- a/cffi/__init__.py +++ b/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "0.8.6+" -__version_info__ = (0, 8, 6, "plus") +__version__ = "0.9.0" +__version_info__ = (0, 9, 0) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/doc/source/conf.py b/doc/source/conf.py --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -45,9 +45,9 @@ # built documents. # # The short X.Y version. -version = '0.8' +version = '0.9' # The full version, including alpha/beta/rc tags. -release = '0.8.6+' +release = '0.9.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/source/index.rst b/doc/source/index.rst --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -88,13 +88,13 @@ Download and Installation: -* http://pypi.python.org/packages/source/c/cffi/cffi-0.8.6.tar.gz +* http://pypi.python.org/packages/source/c/cffi/cffi-0.9.0.tar.gz - Or grab the most current version by following the instructions below. - - MD5: 474b5a68299a6f05009171de1dc91be6 + - MD5: 3a2f6b9f16e8082271aed6dcac51a71a - - SHA: 4e82390201e6f30e9df8a91cd176df19b8f2d547 + - SHA: 02e44ecada40cb859e18e0b628cc52deba064a39 * Or get it from the `Bitbucket page`_: ``hg clone https://bitbucket.org/cffi/cffi`` diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -139,7 +139,7 @@ `Mailing list `_ """, - version='0.8.6', + version='0.9.0', packages=['cffi'], zip_safe=False, From noreply at buildbot.pypy.org Tue Mar 3 11:00:36 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 3 Mar 2015 11:00:36 +0100 (CET) Subject: [pypy-commit] pypy default: Bump version number Message-ID: <20150303100036.840EC1C0D5A@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76229:892ae69f849c Date: 2015-03-03 11:00 +0100 http://bitbucket.org/pypy/pypy/changeset/892ae69f849c/ Log: Bump version number diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -8,7 +8,7 @@ appleveldefs = { } interpleveldefs = { - '__version__': 'space.wrap("0.8.6+")', + '__version__': 'space.wrap("0.9.0")', 'load_library': 'libraryobj.load_library', diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -3249,4 +3249,4 @@ def test_version(): # this test is here mostly for PyPy - assert __version__ == "0.8.6+" + assert __version__ == "0.9.0" From noreply at buildbot.pypy.org Tue Mar 3 13:06:45 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 3 Mar 2015 13:06:45 +0100 (CET) Subject: [pypy-commit] pypy default: Shortcut the two-step #define, which ends in a warning from the C compiler Message-ID: <20150303120645.CAB171C0378@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76230:2ff6288a1cf6 Date: 2015-03-03 13:06 +0100 http://bitbucket.org/pypy/pypy/changeset/2ff6288a1cf6/ Log: Shortcut the two-step #define, which ends in a warning from the C compiler if we also have cpyext/include/pymem.h diff --git a/rpython/translator/c/src/dtoa.c b/rpython/translator/c/src/dtoa.c --- a/rpython/translator/c/src/dtoa.c +++ b/rpython/translator/c/src/dtoa.c @@ -129,8 +129,6 @@ #include #include #include "src/asm.h" -#define PyMem_Malloc malloc -#define PyMem_Free free /* End PYPY hacks */ @@ -140,8 +138,8 @@ #include -#define MALLOC PyMem_Malloc -#define FREE PyMem_Free +#define MALLOC malloc +#define FREE free /* This code should also work for ARM mixed-endian format on little-endian machines, where doubles have byte order 45670123 (in increasing address From noreply at buildbot.pypy.org Tue Mar 3 13:30:26 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 3 Mar 2015 13:30:26 +0100 (CET) Subject: [pypy-commit] benchmarks default: Modernize Message-ID: <20150303123026.2E0531C0282@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r319:f6c32be6bf85 Date: 2015-03-03 13:30 +0100 http://bitbucket.org/pypy/benchmarks/changeset/f6c32be6bf85/ Log: Modernize diff --git a/multithread/multithread-richards.py b/multithread/multithread-richards.py --- a/multithread/multithread-richards.py +++ b/multithread/multithread-richards.py @@ -8,7 +8,7 @@ # Outer loop added by Alex Jacoby import thread, os -#from __pypy__.thread import atomic +from pypystm import atomic, hint_commit_soon # Task IDs @@ -371,23 +371,19 @@ self.finished_lock = thread.allocate_lock() self.finished_lock.acquire() - def run_and_unlock(self, to_do): + def run_and_unlock(self, count): print 'running...' iterations = 0 self.result = True - while 1: - try: - to_do.pop() - except IndexError: - break - iterations += 1 + for i in range(count): self.result = self.run() - print 'done, iterations=%d, result=%r' % (iterations, self.result) + print 'done, iterations=%d, result=%r' % (count, self.result) self.finished_lock.release() def run(self): - #with atomic: - if 1: + hint_commit_soon() + with atomic: + #if 1: taskWorkArea = TaskWorkArea() IdleTask(I_IDLE, 1, 10000, TaskState().running(), IdleTaskRec(), @@ -422,19 +418,20 @@ pass else: return False + hint_commit_soon() return True def entry_point(iterations, NUM_THREADS): rlist = [Richards() for i in range(NUM_THREADS)] - to_do = [None] * iterations startTime = time() - for r in rlist: - thread.start_new_thread(r.run_and_unlock, (to_do,)) + for i, r in enumerate(rlist): + count = (iterations * (i + 1)) // NUM_THREADS + count -= (iterations * i) // NUM_THREADS + thread.start_new_thread(r.run_and_unlock, (count,)) for r in rlist: r.finished_lock.acquire() endTime = time() - assert to_do == [] result = all(r.result for r in rlist) return result, startTime, endTime From noreply at buildbot.pypy.org Tue Mar 3 13:30:27 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 3 Mar 2015 13:30:27 +0100 (CET) Subject: [pypy-commit] benchmarks default: Modernize: the "atomic" module no longer exists Message-ID: <20150303123027.8793A1C0282@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r320:5c9bb48bb75d Date: 2015-03-03 13:30 +0100 http://bitbucket.org/pypy/benchmarks/changeset/5c9bb48bb75d/ Log: Modernize: the "atomic" module no longer exists diff --git a/multithread/common/abstract_threading.py b/multithread/common/abstract_threading.py --- a/multithread/common/abstract_threading.py +++ b/multithread/common/abstract_threading.py @@ -3,18 +3,17 @@ import thread, atexit, sys, time try: - from atomic import (atomic, getsegmentlimit, print_abort_info, - hint_commit_soon, is_atomic) -except: + from pypystm import atomic, getsegmentlimit, hint_commit_soon +except ImportError: + raise atomic = RLock() def getsegmentlimit(): return 1 - def print_abort_info(tm=0.0): - pass def hint_commit_soon(): pass - def is_atomic(): - return atomic._RLock__count > 0 + +def print_abort_info(tm=0.0): + "backward compatibility: no-op" class TLQueue_concurrent(object): @@ -114,7 +113,7 @@ def shutdown(self): for w in self.workers: - self.input_queue.put((print_abort_info, (), {})) + #self.input_queue.put((print_abort_info, (), {})) self.input_queue.put((sys.exit, (), {})) for w in self.workers: w.join() From noreply at buildbot.pypy.org Tue Mar 3 13:45:45 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 3 Mar 2015 13:45:45 +0100 (CET) Subject: [pypy-commit] stmgc c8-card-marking: Instead of SIGINT, raise SIGABRT here, and if that doesn't kill the process, Message-ID: <20150303124545.C6CA61C0488@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-card-marking Changeset: r1689:3c11d9ba56a7 Date: 2015-03-03 13:46 +0100 http://bitbucket.org/pypy/stmgc/changeset/3c11d9ba56a7/ Log: Instead of SIGINT, raise SIGABRT here, and if that doesn't kill the process, raise SIGKILL. diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -228,7 +228,8 @@ addr >= stm_object_pages+TOTAL_MEMORY) { /* actual segfault, unrelated to stmgc */ fprintf(stderr, "Segmentation fault: accessing %p\n", addr); - raise(SIGINT); + raise(SIGABRT); + raise(SIGKILL); } int segnum = get_segment_of_linear_address(addr); @@ -236,7 +237,8 @@ if (segnum != STM_SEGMENT->segment_num) { fprintf(stderr, "Segmentation fault: accessing %p (seg %d) from" " seg %d\n", addr, segnum, STM_SEGMENT->segment_num); - raise(SIGINT); + raise(SIGABRT); + raise(SIGKILL); } dprintf(("-> segment: %d\n", segnum)); @@ -245,7 +247,8 @@ if (pagenum < END_NURSERY_PAGE) { fprintf(stderr, "Segmentation fault: accessing %p (seg %d " "page %lu)\n", addr, segnum, pagenum); - raise(SIGINT); + raise(SIGABRT); + raise(SIGKILL); } DEBUG_EXPECT_SEGFAULT(false); From noreply at buildbot.pypy.org Tue Mar 3 17:47:38 2015 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 3 Mar 2015 17:47:38 +0100 (CET) Subject: [pypy-commit] buildbot issue-1759: close branch to be merged Message-ID: <20150303164738.ACCFA1C009F@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: issue-1759 Changeset: r935:2c0ec1e92001 Date: 2015-03-03 18:48 +0200 http://bitbucket.org/pypy/buildbot/changeset/2c0ec1e92001/ Log: close branch to be merged From noreply at buildbot.pypy.org Tue Mar 3 17:47:39 2015 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 3 Mar 2015 17:47:39 +0100 (CET) Subject: [pypy-commit] buildbot default: merge issue-1759 which moves testing runs to /pytest and transfers numpy-compatability test to triggered builds Message-ID: <20150303164739.D06C01C009F@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r936:9368e32b1675 Date: 2015-03-03 18:49 +0200 http://bitbucket.org/pypy/buildbot/changeset/9368e32b1675/ Log: merge issue-1759 which moves testing runs to /pytest and transfers numpy-compatability test to triggered builds diff --git a/bot2/pypybuildbot/arm_master.py b/bot2/pypybuildbot/arm_master.py --- a/bot2/pypybuildbot/arm_master.py +++ b/bot2/pypybuildbot/arm_master.py @@ -1,5 +1,5 @@ from buildbot.scheduler import Nightly, Triggerable -from pypybuildbot.util import we_are_debugging, load +from pypybuildbot.util import load pypybuilds = load('pypybuildbot.builds') ARMCrossLock = pypybuilds.ARMCrossLock diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -4,7 +4,7 @@ from buildbot.process import factory from buildbot.steps import shell, transfer from buildbot.steps.trigger import Trigger -from buildbot.process.properties import WithProperties +from buildbot.process.properties import WithProperties, Interpolate from buildbot import locks from pypybuildbot.util import symlink_force from buildbot.status.results import SKIPPED, SUCCESS @@ -326,14 +326,14 @@ workdir=workdir, logEnviron=False)) -def update_git(platform, factory, repourl, workdir, use_branch, - force_branch=None): +def update_git(platform, factory, repourl, workdir, branch='master'): factory.addStep( Git( repourl=repourl, mode='full', method='fresh', workdir=workdir, + branch=branch, logEnviron=False)) def setup_steps(platform, factory, workdir=None, @@ -378,6 +378,35 @@ return ".tar.bz2" def add_translated_tests(factory, prefix, platform, app_tests, lib_python, pypyjit): + factory.addStep(shell.SetPropertyFromCommand( + command=['python', '-c', "import tempfile, os ;print" + " tempfile.gettempdir() + os.path.sep"], + property="target_tmpdir")) + # If target_tmpdir is empty, crash. + tmp_or_crazy = '%(prop:target_tmpdir:-crazy/name/so/mkdir/fails/)s' + pytest = "pytest" + factory.addStep(PytestCmd( + description="mkdir for tests", + command=['python', '-c', Interpolate("import os; os.mkdir(r'" + \ + tmp_or_crazy + pytest + "') if not os.path.exists(r'" + \ + tmp_or_crazy + pytest + "') else True")], + haltOnFailure=True, + )) + + nDays = '3' #str, not int + if platform == 'win32': + command = ['FORFILES', '/P', Interpolate(tmp_or_crazy + pytest), + '/D', '-' + nDays, '/c', "cmd /c rmdir /q /s @path"] + else: + command = ['find', Interpolate(tmp_or_crazy + pytest), '-mtime', + '+' + nDays, '-exec', 'rm', '{}', ';'] + factory.addStep(PytestCmd( + description="cleanout old test files", + command = command, + flunkOnFailure=False, + haltOnFailure=False, + )) + if app_tests: if app_tests == True: app_tests = [] @@ -392,7 +421,9 @@ ] + ["--config=%s" % cfg for cfg in app_tests], logfiles={'pytestLog': 'pytest-A.log'}, timeout=4000, - env={"PYTHONPATH": ['.']})) + env={"PYTHONPATH": ['.'], + "TMPDIR": Interpolate('%(prop:target_tmpdir)s' + pytest), + })) if lib_python: factory.addStep(PytestCmd( @@ -402,7 +433,9 @@ "--timeout=3600", "--resultlog=cpython.log", "lib-python"], timeout=4000, - logfiles={'pytestLog': 'cpython.log'})) + logfiles={'pytestLog': 'cpython.log'}, + env={"TMPDIR": Interpolate('%(prop:target_tmpdir)s' + pytest), + })) if pypyjit: # kill this step when the transition to test_pypy_c_new has been @@ -414,7 +447,9 @@ "--pypy=pypy/goal/pypy-c", "--resultlog=pypyjit.log", "pypy/module/pypyjit/test"], - logfiles={'pytestLog': 'pypyjit.log'})) + logfiles={'pytestLog': 'pypyjit.log'}, + env={"TMPDIR": Interpolate('%(prop:target_tmpdir)s' + pytest), + })) # # "new" test_pypy_c if platform == 'win32': @@ -426,7 +461,10 @@ command=prefix + [cmd, "pypy/test_all.py", "--resultlog=pypyjit_new.log", "pypy/module/pypyjit/test_pypy_c"], - logfiles={'pytestLog': 'pypyjit_new.log'})) + logfiles={'pytestLog': 'pypyjit_new.log'}, + env={"TMPDIR": Interpolate('%(prop:target_tmpdir)s' + pytest), + })) + # ---- @@ -438,6 +476,36 @@ setup_steps(platform, self) timeout=kwargs.get('timeout', 4000) + + self.addStep(shell.SetPropertyFromCommand( + command=['python', '-c', "import tempfile, os ;print" + " tempfile.gettempdir() + os.path.sep"], + property="target_tmpdir")) + # If target_tmpdir is empty, crash. + tmp_or_crazy = '%(prop:target_tmpdir:-crazy/name/so/mkdir/fails/)s' + pytest = "pytest" + self.addStep(PytestCmd( + description="mkdir for tests", + command=['python', '-c', Interpolate("import os; os.mkdir(r'" + \ + tmp_or_crazy + pytest + "') if not os.path.exists(r'" + \ + tmp_or_crazy + pytest + "') else True")], + haltOnFailure=True, + )) + + nDays = '3' #str, not int + if platform == 'win32': + command = ['FORFILES', '/P', Interpolate(tmp_or_crazy + pytest), + '/D', '-' + nDays, '/c', "cmd /c rmdir /q /s @path"] + else: + command = ['find', Interpolate(tmp_or_crazy + pytest), '-mtime', + '+' + nDays, '-exec', 'rm', '{}', ';'] + self.addStep(PytestCmd( + description="cleanout old test files", + command = command, + flunkOnFailure=False, + haltOnFailure=False, + )) + self.addStep(PytestCmd( description="pytest pypy", command=["python", "testrunner/runner.py", @@ -449,7 +517,9 @@ logfiles={'pytestLog': 'testrun.log'}, timeout=timeout, env={"PYTHONPATH": ['.'], - "PYPYCHERRYPICK": cherrypick})) + "PYPYCHERRYPICK": cherrypick, + "TMPDIR": Interpolate('%(prop:target_tmpdir)s' + pytest), + })) self.addStep(PytestCmd( description="pytest rpython", @@ -462,7 +532,9 @@ logfiles={'pytestLog': 'testrun.log'}, timeout=timeout, env={"PYTHONPATH": ['.'], - "PYPYCHERRYPICK": cherrypick})) + "PYPYCHERRYPICK": cherrypick, + "TMPDIR": Interpolate('%(prop:target_tmpdir)s' + pytest), + })) class Translated(factory.BuildFactory): @@ -473,7 +545,8 @@ interpreter='pypy', lib_python=False, pypyjit=False, - prefix=None + prefix=None, + trigger=None, ): factory.BuildFactory.__init__(self) if prefix is not None: @@ -502,6 +575,9 @@ workdir='.', blocksize=100 * 1024)) + if trigger: # if provided trigger schedulers that depend on this one + self.addStep(Trigger(schedulerNames=[trigger])) + add_translated_tests(self, prefix, platform, app_tests, lib_python, pypyjit) @@ -839,7 +915,6 @@ ''' def __init__(self, platform='linux', app_tests=False, - host = 'tannit', lib_python=False, pypyjit=True, prefix=None, @@ -850,9 +925,18 @@ self.addStep(ParseRevision(hideStepIf=ParseRevision.hideStepIf, doStepIf=ParseRevision.doStepIf)) # download corresponding nightly build + if platform == 'win32': + target = r'pypy-c\pypy.exe' + untar = ['unzip'] + sep = '\\' + else: + target = r'pypy-c/bin/pypy' + untar = ['tar', '--strip-components=1', '--directory=.', '-xf'] + sep = '/' self.addStep(ShellCmd( - description="Clear pypy-c", - command=['rm', '-rf', 'pypy-c'], + description="Clear", + # assume, as part of git, that windows has rm + command=['rm', '-rf', 'pypy-c', 'install'], workdir='.')) extension = get_extension(platform) name = build_name(platform, pypyjit, translationArgs, placeholder='%(final_file_name)s') + extension @@ -863,12 +947,17 @@ workdir='pypy-c')) # extract downloaded file - if platform.startswith('win'): - raise NotImplementedError - else: + self.addStep(ShellCmd( + description="decompress pypy-c", + command=untar + ['pypy_build'+ extension], + workdir='pypy-c', + haltOnFailure=True, + )) + + if platform == 'win32': self.addStep(ShellCmd( - description="decompress pypy-c", - command=['tar', '--extract', '--file=pypy_build'+ extension, '--strip-components=1', '--directory=.'], + description='move decompressed dir', + command = ['mv', '*/*', '.'], workdir='pypy-c', haltOnFailure=True, )) @@ -876,39 +965,45 @@ # virtualenv the download self.addStep(ShellCmd( description="create virtualenv", - command=['virtualenv','-p', 'pypy-c/bin/pypy', 'install'], + command=['virtualenv','-p', target, 'install'], + workdir='./', + haltOnFailure=True, + )) + + self.addStep(ShellCmd( + description="report version", + command=[sep.join(['install','bin','pypy'])] + ['--version'], workdir='./', haltOnFailure=True, )) self.addStep(ShellCmd( description="install nose", - command=['install/bin/pip', 'install','nose'], + command=[sep.join(['install','bin','pip'])] + ['install','nose'], workdir='./', haltOnFailure=True, )) # obtain a pypy-compatible branch of numpy numpy_url = 'https://www.bitbucket.org/pypy/numpy' - numpy_pypy_branch = 'pypy-compat' - update_git(platform, self, numpy_url, 'numpy_src', use_branch=True, - force_branch=numpy_pypy_branch) + update_git(platform, self, numpy_url, 'numpy_src', branch='master') self.addStep(ShellCmd( description="install numpy", - command=['../install/bin/python', 'setup.py','install'], + command=[sep.join(['..', 'install', 'bin', 'pypy'])] + ['setup.py','install'], workdir='numpy_src')) self.addStep(ShellCmd( description="test numpy", - command=['bin/nosetests', 'site-packages/numpy', + command=[sep.join(['bin', 'nosetests'])] + ['site-packages/numpy', + # XXX enable '-with-doctest', ], #logfiles={'pytestLog': 'pytest-numpy.log'}, timeout=4000, workdir='install', #env={"PYTHONPATH": ['download']}, # shouldn't be needed, but what if it is set externally? )) - if host == 'tannit': + if platform != 'win32': self.addStep(ShellCmd( description="install jinja2", command=['install/bin/pip', 'install', 'jinja2'], diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -1,6 +1,6 @@ import os -from buildbot.scheduler import Nightly +from buildbot.scheduler import Nightly, Triggerable from buildbot.schedulers.forcesched import ForceScheduler from buildbot.schedulers.forcesched import ValidationError from buildbot.buildslave import BuildSlave @@ -96,6 +96,7 @@ pypyjit=True, app_tests=True, platform='linux64', + trigger='NUMPY64_scheduler', ) pypyJITTranslatedTestFactoryIndiana = pypybuilds.Translated( @@ -133,6 +134,7 @@ lib_python=True, pypyjit=True, app_tests=True, + trigger='NUMPYWIN_scheduler', ) pypyJITTranslatedTestFactoryFreeBSD = pypybuilds.Translated( @@ -144,14 +146,17 @@ app_tests=True, ) -pypyJITBenchmarkFactory_tannit = pypybuilds.JITBenchmark() +pypyJITBenchmarkFactory_tannit = pypybuilds.JITBenchmark(host='tannit') pypyJITBenchmarkFactory64_tannit = pypybuilds.JITBenchmark(platform='linux64', + host='tannit', postfix='-64') pypyJITBenchmarkFactory64_speed = pypybuilds.JITBenchmarkSingleRun( platform='linux64', + host='speed_python', postfix='-64') pypyNumpyCompatability = pypybuilds.NativeNumpyTests(platform='linux64') +pypyNumpyCompatabilityWin = pypybuilds.NativeNumpyTests(platform='win32') # @@ -186,6 +191,7 @@ JITBENCH64_NEW = 'jit-benchmark-linux-x86-64-single-run' CPYTHON_64 = "cpython-2-benchmark-x86-64" NUMPY_64 = "numpy-compatability-linux-x86-64" +NUMPY_WIN = "numpy-compatability-win-x86-32" # buildbot builder PYPYBUILDBOT = 'pypy-buildbot' @@ -216,8 +222,8 @@ APPLVLLINUX64, # on allegro64, uses 1 core # other platforms #MACOSX32, # on minime - JITWIN32, # on aurora, SalsaSalsa - WIN32, # on aurora, SalsaSalsa + JITWIN32, # on allegro_win32, SalsaSalsa + WIN32, # on allegro_win32, SalsaSalsa #JITFREEBSD764, # on headless #JITFREEBSD864, # on ananke JITFREEBSD964, # on tavendo @@ -227,14 +233,20 @@ ], branch='default', hour=0, minute=0), Nightly("nightly-1-00", [ - NUMPY_64, # on tannit64, uses 1 core, takes about 15min. - # XXX maybe use a trigger instead? JITBENCH, # on tannit32, uses 1 core (in part exclusively) JITBENCH64, # on tannit64, uses 1 core (in part exclusively) JITBENCH64_NEW, # on speed64, uses 1 core (in part exclusively) ], branch=None, hour=1, minute=0), + Triggerable("NUMPY64_scheduler", [ + NUMPY_64, # on tannit64, uses 1 core, takes about 15min. + ]), + + Triggerable("NUMPYWIN_scheduler", [ + NUMPY_WIN, # on allegro_win32, SalsaSalsa + ]), + Nightly("nightly-2-00-py3k", [ LINUX64, # on allegro64, uses all cores APPLVLLINUX64, # on allegro64, uses 1 core @@ -279,6 +291,7 @@ JITBENCH64, JITBENCH64_NEW, NUMPY_64, + NUMPY_WIN, ] + ARM.builderNames, properties=[]), ] + ARM.schedulers, @@ -463,7 +476,14 @@ 'factory': pypyNumpyCompatability, 'category': 'numpy', 'locks': [TannitCPU.access('counting')], - }, + }, + {'name': NUMPY_WIN, + 'slavenames': ["allegro_win32", "SalsaSalsa"], + 'builddir': NUMPY_WIN, + 'factory': pypyNumpyCompatabilityWin, + "locks": [WinSlaveLock.access('counting')], + 'category': 'numpy', + }, {'name': PYPYBUILDBOT, 'slavenames': ['cobra'], 'builddir': PYPYBUILDBOT, diff --git a/master/master.cfg b/master/master.cfg --- a/master/master.cfg +++ b/master/master.cfg @@ -23,5 +23,9 @@ if we_are_debugging(): for builderdict in BuildmasterConfig['builders']: - builderdict["slavenames"] = ['localhost'] + valid_slaves = ['localhost'] + for s in builderdict['slavenames']: + if s in slaveinfo.passwords: + valid_slaves.append(s) + builderdict["slavenames"] = valid_slaves BuildmasterConfig['buildbotURL'] = "http://localhost:%d/" % (httpPortNumber) From noreply at buildbot.pypy.org Tue Mar 3 18:14:35 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 3 Mar 2015 18:14:35 +0100 (CET) Subject: [pypy-commit] cffi default: gcc complains if given the obscure flag "-Werror=declaration-after-statement" Message-ID: <20150303171435.7733E1C014D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1656:9ee016c3276b Date: 2015-03-03 18:14 +0100 http://bitbucket.org/cffi/cffi/changeset/9ee016c3276b/ Log: gcc complains if given the obscure flag "-Werror=declaration-after- statement" diff --git a/c/malloc_closure.h b/c/malloc_closure.h --- a/c/malloc_closure.h +++ b/c/malloc_closure.h @@ -125,6 +125,7 @@ if (item == NULL) return; #else + { int prot = PROT_READ | PROT_WRITE | PROT_EXEC; if (is_emutramp_enabled ()) prot &= ~PROT_EXEC; @@ -136,6 +137,7 @@ 0); if (item == (void *)MAP_FAILED) return; + } #endif #ifdef MALLOC_CLOSURE_DEBUG From noreply at buildbot.pypy.org Tue Mar 3 19:54:59 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 3 Mar 2015 19:54:59 +0100 (CET) Subject: [pypy-commit] pypy default: Import cffi release 0.9 Message-ID: <20150303185459.D4EC21C030A@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76231:8fc3eeed2c5f Date: 2015-03-03 19:54 +0100 http://bitbucket.org/pypy/pypy/changeset/8fc3eeed2c5f/ Log: Import cffi release 0.9 diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "0.8.6+" -__version_info__ = (0, 8, 6, "plus") +__version__ = "0.9.0" +__version_info__ = (0, 9, 0) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ From noreply at buildbot.pypy.org Tue Mar 3 20:10:38 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 3 Mar 2015 20:10:38 +0100 (CET) Subject: [pypy-commit] pypy default: hg backout e36875ce12ab Message-ID: <20150303191038.036741C1401@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76232:a063ef6478b1 Date: 2015-03-03 20:10 +0100 http://bitbucket.org/pypy/pypy/changeset/a063ef6478b1/ Log: hg backout e36875ce12ab This was a bogus solution. We can see in x86/runner.py that multiple invalidate_loop() with the same looptoken never patches the same location more than once. We need to debug and understand the original problem again. Additionally, e36875ce12ab has a bad performance effect on some benchmarks. I think it is because it is possible to have one loop that gets invalidated, but not the other loops that JMP or CALL_ASSEMBLER to it. That's how a guard_not_invalidated can fail a lot, and a reason why attaching a bridge to it can be a very good idea. diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -682,9 +682,6 @@ class ResumeGuardNotInvalidated(ResumeGuardDescr): guard_opnum = rop.GUARD_NOT_INVALIDATED - def must_compile(self, deadframe, metainterp_sd, jitdriver_sd): - return False - class ResumeAtPositionDescr(ResumeGuardDescr): guard_opnum = rop.GUARD_FUTURE_CONDITION From noreply at buildbot.pypy.org Tue Mar 3 20:32:53 2015 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 3 Mar 2015 20:32:53 +0100 (CET) Subject: [pypy-commit] buildbot default: ignore revision on git update of numpy-compatible builds since they are triggered with a pypy revision id Message-ID: <20150303193253.2727A1C069F@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r937:6ccf9981ec27 Date: 2015-03-03 21:34 +0200 http://bitbucket.org/pypy/buildbot/changeset/6ccf9981ec27/ Log: ignore revision on git update of numpy-compatible builds since they are triggered with a pypy revision id diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -326,7 +326,8 @@ workdir=workdir, logEnviron=False)) -def update_git(platform, factory, repourl, workdir, branch='master'): +def update_git(platform, factory, repourl, workdir, branch='master', + alwaysUseLatest=False): factory.addStep( Git( repourl=repourl, @@ -334,6 +335,7 @@ method='fresh', workdir=workdir, branch=branch, + alwaysUseLatest=alwaysUseLatest, logEnviron=False)) def setup_steps(platform, factory, workdir=None, @@ -986,7 +988,10 @@ # obtain a pypy-compatible branch of numpy numpy_url = 'https://www.bitbucket.org/pypy/numpy' - update_git(platform, self, numpy_url, 'numpy_src', branch='master') + update_git(platform, self, numpy_url, 'numpy_src', branch='master', + alwaysUseLatest=True, # ignore pypy rev number when + # triggered by a pypy build + ) self.addStep(ShellCmd( description="install numpy", From noreply at buildbot.pypy.org Tue Mar 3 22:13:28 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Tue, 3 Mar 2015 22:13:28 +0100 (CET) Subject: [pypy-commit] pypy default: Add a few flowspace tests with convoluted control flow Message-ID: <20150303211328.996F01C009F@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r76233:96a8a84a5510 Date: 2015-03-03 21:12 +0000 http://bitbucket.org/pypy/pypy/changeset/96a8a84a5510/ Log: Add a few flowspace tests with convoluted control flow diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py --- a/rpython/flowspace/test/test_objspace.py +++ b/rpython/flowspace/test/test_objspace.py @@ -204,6 +204,22 @@ def test_break_continue(self): x = self.codetest(self.break_continue) + def test_break_from_handler(self): + def f(x): + while True: + try: + x() + except TypeError: + if x: + raise + break + assert f(0) is None + graph = self.codetest(f) + simplify_graph(graph) + entrymap = mkentrymap(graph) + links = entrymap[graph.returnblock] + assert len(links) == 1 + #__________________________________________________________ def unpack_tuple(lst): a, b, c = lst @@ -245,6 +261,19 @@ def test_finallys(self): x = self.codetest(self.finallys) + def test_branching_in_finally(self): + def f(x, y): + try: + return x + finally: + if x: + x = 0 + if y > 0: + y -= 1 + return y + self.codetest(f) + + #__________________________________________________________ def const_pow(): return 2 ** 5 @@ -942,6 +971,22 @@ 'simple_call': 4, # __enter__, g and 2 possible calls to __exit__ } + def test_return_in_with(self): + def f(x): + with x: + return 1 + graph = self.codetest(f) + simplify_graph(graph) + assert self.all_operations(graph) == {'getattr': 2, 'simple_call': 2} + + def test_break_in_with(self): + def f(n, x): + for i in range(n): + with x: + break + return 1 + self.codetest(f) + def monkey_patch_code(self, code, stacksize, flags, codestring, names, varnames): c = code return types.CodeType(c.co_argcount, c.co_nlocals, stacksize, flags, From noreply at buildbot.pypy.org Wed Mar 4 00:31:33 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Wed, 4 Mar 2015 00:31:33 +0100 (CET) Subject: [pypy-commit] pypy py3.3: Fix compilation of _testcapimodule Message-ID: <20150303233133.38AA01C087E@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3.3 Changeset: r76234:f60fdd5b5b05 Date: 2015-03-04 00:29 +0100 http://bitbucket.org/pypy/pypy/changeset/f60fdd5b5b05/ Log: Fix compilation of _testcapimodule diff --git a/lib_pypy/_testcapimodule.c b/lib_pypy/_testcapimodule.c --- a/lib_pypy/_testcapimodule.c +++ b/lib_pypy/_testcapimodule.c @@ -2444,8 +2444,6 @@ return PyLong_FromLong(r); } -#endif /* PYPY_VERSION */ - static PyObject * test_pytime_object_to_time_t(PyObject *self, PyObject *args) { @@ -2484,6 +2482,8 @@ return Py_BuildValue("Nl", _PyLong_FromTime_t(sec), nsec); } +#endif /* PYPY_VERSION */ + #ifdef WITH_THREAD typedef struct { PyThread_type_lock start_event; @@ -2668,10 +2668,10 @@ {"crash_no_current_thread", (PyCFunction)crash_no_current_thread, METH_NOARGS}, #ifndef PYPY_VERSION {"run_in_subinterp", run_in_subinterp, METH_VARARGS}, -#endif {"pytime_object_to_time_t", test_pytime_object_to_time_t, METH_VARARGS}, {"pytime_object_to_timeval", test_pytime_object_to_timeval, METH_VARARGS}, {"pytime_object_to_timespec", test_pytime_object_to_timespec, METH_VARARGS}, +#endif #ifdef WITH_THREAD {"call_in_temporary_c_thread", call_in_temporary_c_thread, METH_O, PyDoc_STR("set_error_class(error_class) -> None")}, diff --git a/pypy/module/cpyext/include/pymacro.h b/pypy/module/cpyext/include/pymacro.h --- a/pypy/module/cpyext/include/pymacro.h +++ b/pypy/module/cpyext/include/pymacro.h @@ -1,6 +1,21 @@ #ifndef Py_PYMACRO_H #define Py_PYMACRO_H +/* Assert a build-time dependency, as an expression. + + Your compile will fail if the condition isn't true, or can't be evaluated + by the compiler. This can be used in an expression: its value is 0. + + Example: + + #define foo_to_char(foo) \ + ((char *)(foo) \ + + Py_BUILD_ASSERT_EXPR(offsetof(struct foo, string) == 0)) + + Written by Rusty Russell, public domain, http://ccodearchive.net/ */ +#define Py_BUILD_ASSERT_EXPR(cond) \ + (sizeof(char [1 - 2*!(cond)]) - 1) + /* Get the number of elements in a visible array This does not work on pointers, or arrays declared as [], or function From noreply at buildbot.pypy.org Wed Mar 4 02:03:11 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 4 Mar 2015 02:03:11 +0100 (CET) Subject: [pypy-commit] pypy default: issue #1990: fix trouble caused by dde0fac9f1a4. See the issue for Message-ID: <20150304010311.9FAEF1C0488@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76235:dd560e38c568 Date: 2015-03-04 02:00 +0100 http://bitbucket.org/pypy/pypy/changeset/dd560e38c568/ Log: issue #1990: fix trouble caused by dde0fac9f1a4. See the issue for details. Should be fixed by more explicitly asking for follow_jump(), often but not always. diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -26,6 +26,7 @@ imm0, imm1, FloatImmedLoc, RawEbpLoc, RawEspLoc) from rpython.rlib.objectmodel import we_are_translated from rpython.jit.backend.x86 import rx86, codebuf, callbuilder +from rpython.jit.backend.x86.callbuilder import follow_jump from rpython.jit.metainterp.resoperation import rop from rpython.jit.backend.x86 import support from rpython.rlib.debug import debug_print, debug_start, debug_stop @@ -245,7 +246,7 @@ mc.MOV_rr(esi.value, eax.value) # tid mc.MOV_rs(edi.value, WORD * 3) # load the itemsize self.set_extra_stack_depth(mc, 16) - mc.CALL(imm(addr)) + mc.CALL(imm(follow_jump(addr))) mc.ADD_ri(esp.value, 16 - WORD) mc.TEST_rr(eax.value, eax.value) mc.J_il(rx86.Conditions['Z'], 0xfffff) # patched later @@ -312,7 +313,7 @@ mc.PUSH_r(esp.value) # # esp is now aligned to a multiple of 16 again - mc.CALL(imm(slowpathaddr)) + mc.CALL(imm(follow_jump(slowpathaddr))) # if IS_X86_32: mc.ADD_ri(esp.value, 3*WORD) # alignment @@ -819,7 +820,7 @@ newlooptoken.compiled_loop_token.update_frame_info( oldlooptoken.compiled_loop_token, baseofs) mc = codebuf.MachineCodeBlockWrapper() - mc.JMP(imm(target)) + mc.JMP(imm(follow_jump(target))) if WORD == 4: # keep in sync with prepare_loop() assert mc.get_relative_pos() == 5 else: @@ -2228,7 +2229,7 @@ if self._regalloc.xrm.reg_bindings: floats = True cond_call_adr = self.cond_call_slowpath[floats * 2 + callee_only] - self.mc.CALL(imm(cond_call_adr)) + self.mc.CALL(imm(follow_jump(cond_call_adr))) # restoring the registers saved above, and doing pop_gcmap(), is left # to the cond_call_slowpath helper. We never have any result value. offset = self.mc.get_relative_pos() - jmp_adr @@ -2246,7 +2247,7 @@ jmp_adr = self.mc.get_relative_pos() # save the gcmap self.push_gcmap(self.mc, gcmap, mov=True) - self.mc.CALL(imm(self.malloc_slowpath)) + self.mc.CALL(imm(follow_jump(self.malloc_slowpath))) offset = self.mc.get_relative_pos() - jmp_adr assert 0 < offset <= 127 self.mc.overwrite(jmp_adr-1, chr(offset)) @@ -2267,7 +2268,7 @@ jmp_adr = self.mc.get_relative_pos() # save the gcmap self.push_gcmap(self.mc, gcmap, mov=True) - self.mc.CALL(imm(self.malloc_slowpath)) + self.mc.CALL(imm(follow_jump(self.malloc_slowpath))) offset = self.mc.get_relative_pos() - jmp_adr assert 0 < offset <= 127 self.mc.overwrite(jmp_adr-1, chr(offset)) @@ -2332,7 +2333,7 @@ assert kind == rewrite.FLAG_UNICODE addr = self.malloc_slowpath_unicode self.mc.MOV(edi, lengthloc) - self.mc.CALL(imm(addr)) + self.mc.CALL(imm(follow_jump(addr))) self.mc.JMP_l8(0) # jump to done, patched later jmp_location = self.mc.get_relative_pos() # diff --git a/rpython/jit/backend/x86/callbuilder.py b/rpython/jit/backend/x86/callbuilder.py --- a/rpython/jit/backend/x86/callbuilder.py +++ b/rpython/jit/backend/x86/callbuilder.py @@ -1,6 +1,7 @@ import sys from rpython.rlib.clibffi import FFI_DEFAULT_ABI from rpython.rlib.objectmodel import we_are_translated +from rpython.rlib.rarithmetic import intmask from rpython.jit.metainterp.history import INT, FLOAT from rpython.jit.backend.x86.arch import (WORD, IS_X86_64, IS_X86_32, PASS_ON_MY_FRAME, FRAME_FIXED_SIZE, @@ -25,6 +26,14 @@ def align_stack_words(words): return (words + CALL_ALIGN - 1) & ~(CALL_ALIGN-1) +def follow_jump(addr): + # If 'addr' is immediately starting with another JMP instruction, + # follow it now. 'addr' is an absolute address here + while rffi.cast(rffi.CCHARP, addr)[0] == '\xE9': # JMP <4 bytes> + addr += 5 + addr += intmask(rffi.cast(rffi.INTP, addr - 4)[0]) + return addr + class CallBuilderX86(AbstractCallBuilder): @@ -42,8 +51,11 @@ resloc, restype, ressize) # Avoid tons of issues with a non-immediate fnloc by sticking it # as an extra argument if needed - self.fnloc_is_immediate = isinstance(fnloc, ImmedLoc) - if not self.fnloc_is_immediate: + if isinstance(fnloc, ImmedLoc): + self.fnloc_is_immediate = True + self.fnloc = imm(follow_jump(fnloc.value)) + else: + self.fnloc_is_immediate = False self.fnloc = None self.arglocs = arglocs + [fnloc] self.current_esp = 0 # 0 or (usually) negative, counted in bytes @@ -203,7 +215,7 @@ tlofsreg = self.get_tlofs_reg() # => esi, callee-saved self.save_stack_position() # => edi, callee-saved mc.PUSH_m((tlofsreg.value, lasterror)) - mc.CALL(imm(SetLastError_addr)) + mc.CALL(imm(follow_jump(SetLastError_addr))) # restore the stack position without assuming a particular # calling convention of _SetLastError() self.mc.MOV(esp, self.saved_stack_position_reg) @@ -271,7 +283,7 @@ lasterror = llerrno.get_rpy_lasterror_offset(self.asm.cpu) self.save_result_value(save_edx=True) # save eax/edx/xmm0 self.result_value_saved_early = True - mc.CALL(imm(GetLastError_addr)) + mc.CALL(imm(follow_jump(GetLastError_addr))) # tlofsreg = self.get_tlofs_reg() # => esi (possibly reused) mc.MOV32_mr((tlofsreg.value, lasterror), eax.value) @@ -352,7 +364,7 @@ mc.MOV_sr(4, old_value.value) mc.MOV_sr(0, css_value.value) # on X86_64, they are already in the right registers - mc.CALL(imm(self.asm.reacqgil_addr)) + mc.CALL(imm(follow_jump(self.asm.reacqgil_addr))) if not self.result_value_saved_early: self.restore_result_value(save_edx=False) # diff --git a/rpython/jit/backend/x86/regloc.py b/rpython/jit/backend/x86/regloc.py --- a/rpython/jit/backend/x86/regloc.py +++ b/rpython/jit/backend/x86/regloc.py @@ -516,10 +516,7 @@ if code == possible_code: val = getattr(loc, "value_" + possible_code)() if possible_code == 'i': - # This is for CALL or JMP only. If target is - # immediately starting with another JMP instruction, - # follow it now. - val = self._follow_jump_instructions(val) + # This is for CALL or JMP only. if self.WORD == 4: _rx86_getattr(self, name + "_l")(val) self.add_pending_relocation() @@ -537,17 +534,6 @@ return func_with_new_name(INSN, "INSN_" + name) - _do_follow_jump_instructions = True - - def _follow_jump_instructions(self, addr): - if not self._do_follow_jump_instructions or addr == 0: # for tests - return addr - # 'addr' is an absolute address here - while rffi.cast(rffi.CCHARP, addr)[0] == '\xE9': # JMP <4 bytes> - addr += 5 - addr += intmask(rffi.cast(rffi.INTP, addr - 4)[0]) - return addr - def _addr_as_reg_offset(self, addr): # Encodes a (64-bit) address as an offset from the scratch register. # If we are within a "reuse_scratch_register" block, we remember the diff --git a/rpython/jit/backend/x86/test/test_callbuilder.py b/rpython/jit/backend/x86/test/test_callbuilder.py --- a/rpython/jit/backend/x86/test/test_callbuilder.py +++ b/rpython/jit/backend/x86/test/test_callbuilder.py @@ -20,7 +20,12 @@ def test_base_case(call_release_gil_mode=False): asm = FakeAssembler() - cb = callbuilder.CallBuilder64(asm, ImmedLoc(12345), [ebx, ebx]) + old_follow_jump = callbuilder.follow_jump + try: + callbuilder.follow_jump = lambda addr: addr + cb = callbuilder.CallBuilder64(asm, ImmedLoc(12345), [ebx, ebx]) + finally: + callbuilder.follow_jump = old_follow_jump if call_release_gil_mode: cb.select_call_release_gil_mode() cb.prepare_arguments() diff --git a/rpython/jit/backend/x86/test/test_regloc.py b/rpython/jit/backend/x86/test/test_regloc.py --- a/rpython/jit/backend/x86/test/test_regloc.py +++ b/rpython/jit/backend/x86/test/test_regloc.py @@ -5,6 +5,7 @@ from rpython.jit.backend.x86.assembler import heap from rpython.jit.backend.x86.arch import IS_X86_64, IS_X86_32 from rpython.jit.backend.x86 import codebuf +from rpython.jit.backend.x86.callbuilder import follow_jump from rpython.rlib.rarithmetic import intmask import py.test @@ -67,7 +68,6 @@ if target > sys.maxint: continue mc = codebuf.MachineCodeBlockWrapper() - mc._do_follow_jump_instructions = False mc.CALL(ImmedLoc(target)) length = mc.get_relative_pos() buf = lltype.malloc(rffi.CCHARP.TO, length, flavor='raw') @@ -104,6 +104,7 @@ mc.RET() mc.copy_to_raw_memory(raw) mc = codebuf.MachineCodeBlockWrapper(); mc.WORD = 4; mc.relocations = [] + assert follow_jump(raw) == raw mc.JMP(imm(raw)) mc.copy_to_raw_memory(raw + 20) assert buf[20] == '\xE9' # JMP @@ -112,21 +113,15 @@ assert buf[23] == '\xFF' assert buf[24] == '\xFF' mc = codebuf.MachineCodeBlockWrapper(); mc.WORD = 4; mc.relocations = [] - mc.JMP(imm(raw + 20)) + assert follow_jump(raw + 20) == raw + mc.JMP(imm(raw)) mc.copy_to_raw_memory(raw + 40) assert buf[40] == '\xE9' # JMP assert buf[41] == '\xD3' # -45 assert buf[42] == '\xFF' assert buf[43] == '\xFF' assert buf[44] == '\xFF' - mc = codebuf.MachineCodeBlockWrapper(); mc.WORD = 4; mc.relocations = [] - mc.CALL(imm(raw + 40)) - mc.copy_to_raw_memory(raw + 60) - assert buf[60] == '\xE8' # CALL - assert buf[61] == '\xBF' # -65 - assert buf[62] == '\xFF' - assert buf[63] == '\xFF' - assert buf[64] == '\xFF' + assert follow_jump(raw + 40) == raw lltype.free(buf, flavor='raw') From noreply at buildbot.pypy.org Wed Mar 4 02:03:13 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 4 Mar 2015 02:03:13 +0100 (CET) Subject: [pypy-commit] pypy default: merge heads Message-ID: <20150304010313.273891C0488@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76236:167ad5df04a2 Date: 2015-03-04 02:03 +0100 http://bitbucket.org/pypy/pypy/changeset/167ad5df04a2/ Log: merge heads diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py --- a/rpython/flowspace/test/test_objspace.py +++ b/rpython/flowspace/test/test_objspace.py @@ -204,6 +204,22 @@ def test_break_continue(self): x = self.codetest(self.break_continue) + def test_break_from_handler(self): + def f(x): + while True: + try: + x() + except TypeError: + if x: + raise + break + assert f(0) is None + graph = self.codetest(f) + simplify_graph(graph) + entrymap = mkentrymap(graph) + links = entrymap[graph.returnblock] + assert len(links) == 1 + #__________________________________________________________ def unpack_tuple(lst): a, b, c = lst @@ -245,6 +261,19 @@ def test_finallys(self): x = self.codetest(self.finallys) + def test_branching_in_finally(self): + def f(x, y): + try: + return x + finally: + if x: + x = 0 + if y > 0: + y -= 1 + return y + self.codetest(f) + + #__________________________________________________________ def const_pow(): return 2 ** 5 @@ -942,6 +971,22 @@ 'simple_call': 4, # __enter__, g and 2 possible calls to __exit__ } + def test_return_in_with(self): + def f(x): + with x: + return 1 + graph = self.codetest(f) + simplify_graph(graph) + assert self.all_operations(graph) == {'getattr': 2, 'simple_call': 2} + + def test_break_in_with(self): + def f(n, x): + for i in range(n): + with x: + break + return 1 + self.codetest(f) + def monkey_patch_code(self, code, stacksize, flags, codestring, names, varnames): c = code return types.CodeType(c.co_argcount, c.co_nlocals, stacksize, flags, From noreply at buildbot.pypy.org Wed Mar 4 10:40:01 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 4 Mar 2015 10:40:01 +0100 (CET) Subject: [pypy-commit] pypy default: Fix test after translation Message-ID: <20150304094001.1DA6C1C0096@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76237:7517b098f581 Date: 2015-03-04 10:39 +0100 http://bitbucket.org/pypy/pypy/changeset/7517b098f581/ Log: Fix test after translation diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py --- a/pypy/module/sys/test/test_sysmodule.py +++ b/pypy/module/sys/test/test_sysmodule.py @@ -521,9 +521,11 @@ def test_reload_doesnt_override_sys_executable(self): import sys - sys.executable = 'from_test_sysmodule' + if not hasattr(sys, 'executable'): # if not translated + sys.executable = 'from_test_sysmodule' + previous = sys.executable reload(sys) - assert sys.executable == 'from_test_sysmodule' + assert sys.executable == previous def test_settrace(self): import sys From noreply at buildbot.pypy.org Wed Mar 4 11:52:59 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Wed, 4 Mar 2015 11:52:59 +0100 (CET) Subject: [pypy-commit] pypy default: some more pure op variants for int_add and int_sub where one argument is constant Message-ID: <20150304105259.AE8A01C052B@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: Changeset: r76238:921ec8f176f5 Date: 2015-03-04 11:52 +0100 http://bitbucket.org/pypy/pypy/changeset/921ec8f176f5/ Log: some more pure op variants for int_add and int_sub where one argument is constant This allows the optimization of: l.append(x) ... y = l[-1] because the index manipulation is folded diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -866,13 +866,41 @@ return optpure = self.optpure if op.getopnum() == rop.INT_ADD: - optpure.pure(rop.INT_ADD, [op.getarg(1), op.getarg(0)], op.result) + arg0 = op.getarg(0) + arg1 = op.getarg(1) + optpure.pure(rop.INT_ADD, [arg1, arg0], op.result) # Synthesize the reverse op for optimize_default to reuse - optpure.pure(rop.INT_SUB, [op.result, op.getarg(1)], op.getarg(0)) - optpure.pure(rop.INT_SUB, [op.result, op.getarg(0)], op.getarg(1)) + optpure.pure(rop.INT_SUB, [op.result, arg1], arg0) + optpure.pure(rop.INT_SUB, [op.result, arg0], arg1) + if isinstance(arg0, ConstInt): + # invert the constant + i0 = arg0.getint() + inv_arg0 = ConstInt(-i0) + elif isinstance(arg1, ConstInt): + # commutative + i0 = arg1.getint() + inv_arg0 = ConstInt(-i0) + arg1 = arg0 + else: + return + optpure.pure(rop.INT_SUB, [arg1, inv_arg0], op.result) + optpure.pure(rop.INT_SUB, [arg1, op.result], inv_arg0) + optpure.pure(rop.INT_ADD, [op.result, inv_arg0], arg1) + optpure.pure(rop.INT_ADD, [inv_arg0, op.result], arg1) + elif op.getopnum() == rop.INT_SUB: - optpure.pure(rop.INT_ADD, [op.result, op.getarg(1)], op.getarg(0)) - optpure.pure(rop.INT_SUB, [op.getarg(0), op.result], op.getarg(1)) + arg0 = op.getarg(0) + arg1 = op.getarg(1) + optpure.pure(rop.INT_ADD, [op.result, arg1], arg0) + optpure.pure(rop.INT_SUB, [arg0, op.result], arg1) + if isinstance(arg1, ConstInt): + # invert the constant + i1 = arg1.getint() + inv_arg1 = ConstInt(-i1) + optpure.pure(rop.INT_ADD, [arg0, inv_arg1], op.result) + optpure.pure(rop.INT_ADD, [inv_arg1, arg0], op.result) + optpure.pure(rop.INT_SUB, [inv_arg1, op.result], arg0) + optpure.pure(rop.INT_SUB, [op.result, arg0], inv_arg1) elif op.getopnum() == rop.FLOAT_MUL: optpure.pure(rop.FLOAT_MUL, [op.getarg(1), op.getarg(0)], op.result) elif op.getopnum() == rop.FLOAT_NEG: diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -3703,6 +3703,52 @@ """ self.optimize_loop(ops, expected) + def test_int_add_sub_constants_inverse(self): + ops = """ + [i0, i10, i11, i12, i13] + i2 = int_add(1, i0) + i3 = int_add(-1, i2) + i4 = int_sub(i0, -1) + i5 = int_sub(i0, i2) + jump(i0, i2, i3, i4, i5) + """ + expected = """ + [i0, i10, i11, i12, i13] + i2 = int_add(1, i0) + jump(i0, i2, i0, i2, -1) + """ + self.optimize_loop(ops, expected) + ops = """ + [i0, i10, i11, i12, i13] + i2 = int_add(i0, 1) + i3 = int_add(-1, i2) + i4 = int_sub(i0, -1) + i5 = int_sub(i0, i2) + jump(i0, i2, i3, i4, i5) + """ + expected = """ + [i0, i10, i11, i12, i13] + i2 = int_add(i0, 1) + jump(i0, i2, i0, i2, -1) + """ + self.optimize_loop(ops, expected) + + ops = """ + [i0, i10, i11, i12, i13, i14] + i2 = int_sub(i0, 1) + i3 = int_add(-1, i0) + i4 = int_add(i0, -1) + i5 = int_sub(-1, i2) + i6 = int_sub(i2, i0) + jump(i0, i2, i3, i4, i5, i6) + """ + expected = """ + [i0, i10, i11, i12, i13, i14] + i2 = int_sub(i0, 1) + jump(i0, i2, i2, i2, i0, -1) + """ + self.optimize_loop(ops, expected) + def test_framestackdepth_overhead(self): ops = """ [p0, i22] From noreply at buildbot.pypy.org Wed Mar 4 14:55:58 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Wed, 4 Mar 2015 14:55:58 +0100 (CET) Subject: [pypy-commit] pypy default: some sanity checks about using nonsensical combinations of hints Message-ID: <20150304135558.EF1091C0292@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: Changeset: r76239:ff7c259bbcc6 Date: 2015-03-04 14:55 +0100 http://bitbucket.org/pypy/pypy/changeset/ff7c259bbcc6/ Log: some sanity checks about using nonsensical combinations of hints diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -45,6 +45,8 @@ else: assert oldresult == result return result + if getattr(func, '_jit_unroll_safe_', False): + raise TypeError("it does not make sense for %s to be both elidable and unroll_safe" % func) func._elidable_function_ = True return func @@ -83,6 +85,8 @@ """ Make sure the JIT does not trace inside decorated function (it becomes a call instead) """ + if getattr(func, '_jit_unroll_safe_', False): + raise TypeError("it does not make sense for %s to be both dont_look_inside and unroll_safe" % func) func._jit_look_inside_ = False return func @@ -97,6 +101,10 @@ """ JIT can safely unroll loops in this function and this will not lead to code explosion """ + if getattr(func, '_elidable_function_', False): + raise TypeError("it does not make sense for %s to be both elidable and unroll_safe" % func) + if not getattr(func, '_jit_look_inside_', True): + raise TypeError("it does not make sense for %s to be both elidable and dont_look_inside" % func) func._jit_unroll_safe_ = True return func diff --git a/rpython/rlib/test/test_jit.py b/rpython/rlib/test/test_jit.py --- a/rpython/rlib/test/test_jit.py +++ b/rpython/rlib/test/test_jit.py @@ -1,9 +1,10 @@ -import py +import py, pytest from rpython.conftest import option from rpython.annotator.model import UnionError from rpython.rlib.jit import (hint, we_are_jitted, JitDriver, elidable_promote, - JitHintError, oopspec, isconstant, conditional_call) + JitHintError, oopspec, isconstant, conditional_call, + elidable, unroll_safe, dont_look_inside) from rpython.rlib.rarithmetic import r_uint from rpython.rtyper.test.tool import BaseRtypingTest from rpython.rtyper.lltypesystem import lltype @@ -91,6 +92,28 @@ myjitdriver = JitDriver(greens=['n'], reds=[]) py.test.raises(JitHintError, fn, 100) +def test_invalid_hint_combinations_error(): + with pytest.raises(TypeError): + @unroll_safe + @elidable + def f(): + pass + with pytest.raises(TypeError): + @unroll_safe + @elidable + def f(): + pass + with pytest.raises(TypeError): + @unroll_safe + @dont_look_inside + def f(): + pass + with pytest.raises(TypeError): + @unroll_safe + @dont_look_inside + def f(): + pass + class TestJIT(BaseRtypingTest): def test_hint(self): def f(): From noreply at buildbot.pypy.org Wed Mar 4 15:02:47 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Wed, 4 Mar 2015 15:02:47 +0100 (CET) Subject: [pypy-commit] pypy default: add a comment Message-ID: <20150304140247.0E7051C0305@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: Changeset: r76240:1200b6d0c697 Date: 2015-03-04 15:02 +0100 http://bitbucket.org/pypy/pypy/changeset/1200b6d0c697/ Log: add a comment diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -14,6 +14,8 @@ def elidable(func): """ Decorate a function as "trace-elidable". Usually this means simply that the function is constant-foldable, i.e. is pure and has no side-effects. + This also has the effect that the inside of the function will never be + traced. In some situations it is ok to use this decorator if the function *has* side effects, as long as these side-effects are idempotent. A typical From noreply at buildbot.pypy.org Wed Mar 4 15:06:01 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Wed, 4 Mar 2015 15:06:01 +0100 (CET) Subject: [pypy-commit] pypy default: warn when using DeprecationWarning Message-ID: <20150304140601.C3B451C0305@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: Changeset: r76241:d748a4ca2557 Date: 2015-03-04 15:05 +0100 http://bitbucket.org/pypy/pypy/changeset/d748a4ca2557/ Log: warn when using DeprecationWarning diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -96,6 +96,8 @@ """ Make sure the JIT traces inside decorated function, even if the rest of the module is not visible to the JIT """ + import warnings + warnings.warn("look_inside is deprecated", DeprecationWarning) func._jit_look_inside_ = True return func From noreply at buildbot.pypy.org Wed Mar 4 15:32:43 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 4 Mar 2015 15:32:43 +0100 (CET) Subject: [pypy-commit] pypy default: Kill most of the fragile keepalive_until_here() logic. Replace it with a Message-ID: <20150304143243.BDC0F1C009F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76242:1a44b21df005 Date: 2015-03-04 15:32 +0100 http://bitbucket.org/pypy/pypy/changeset/1a44b21df005/ Log: Kill most of the fragile keepalive_until_here() logic. Replace it with a context manager API, declared on the W_CData objects themselves. diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py --- a/pypy/module/_cffi_backend/cbuffer.py +++ b/pypy/module/_cffi_backend/cbuffer.py @@ -81,4 +81,5 @@ if size < 0: raise oefmt(space.w_TypeError, "don't know the size pointed to by '%s'", ctype.name) - return space.wrap(MiniBuffer(LLBuffer(w_cdata._cdata, size), w_cdata)) + ptr = w_cdata.unsafe_escaping_ptr() # w_cdata kept alive by MiniBuffer() + return space.wrap(MiniBuffer(LLBuffer(ptr, size), w_cdata)) diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py --- a/pypy/module/_cffi_backend/ccallback.py +++ b/pypy/module/_cffi_backend/ccallback.py @@ -48,9 +48,12 @@ raise oefmt(space.w_NotImplementedError, "%s: callback with unsupported argument or " "return type or with '...'", self.getfunctype().name) - res = clibffi.c_ffi_prep_closure(self.get_closure(), cif_descr.cif, - invoke_callback, - rffi.cast(rffi.VOIDP, self.unique_id)) + with self as ptr: + closure_ptr = rffi.cast(clibffi.FFI_CLOSUREP, ptr) + unique_id = rffi.cast(rffi.VOIDP, self.unique_id) + res = clibffi.c_ffi_prep_closure(closure_ptr, cif_descr.cif, + invoke_callback, + unique_id) if rffi.cast(lltype.Signed, res) != clibffi.FFI_OK: raise OperationError(space.w_SystemError, space.wrap("libffi failed to build this callback")) @@ -62,12 +65,9 @@ from pypy.module.thread.os_thread import setup_threads setup_threads(space) - def get_closure(self): - return rffi.cast(clibffi.FFI_CLOSUREP, self._cdata) - #@rgc.must_be_light_finalizer def __del__(self): - clibffi.closureHeap.free(self.get_closure()) + clibffi.closureHeap.free(rffi.cast(clibffi.FFI_CLOSUREP, self._ptr)) if self.ll_error: lltype.free(self.ll_error, flavor='raw') @@ -106,7 +106,7 @@ fresult = self.getfunctype().ctitem if fresult.size > 0: misc._raw_memcopy(self.ll_error, ll_res, fresult.size) - keepalive_until_here(self) + keepalive_until_here(self) # to keep self.ll_error alive global_callback_mapping = rweakref.RWeakValueDictionary(int, W_CDataCallback) diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py --- a/pypy/module/_cffi_backend/cdataobj.py +++ b/pypy/module/_cffi_backend/cdataobj.py @@ -14,21 +14,37 @@ class W_CData(W_Root): - _attrs_ = ['space', '_cdata', 'ctype', '_lifeline_'] - _immutable_fields_ = ['_cdata', 'ctype'] - _cdata = lltype.nullptr(rffi.CCHARP.TO) + _attrs_ = ['space', '_ptr', 'ctype', '_lifeline_'] + _immutable_fields_ = ['_ptr', 'ctype'] + _ptr = lltype.nullptr(rffi.CCHARP.TO) - def __init__(self, space, cdata, ctype): + def __init__(self, space, ptr, ctype): from pypy.module._cffi_backend import ctypeobj - assert lltype.typeOf(cdata) == rffi.CCHARP + assert lltype.typeOf(ptr) == rffi.CCHARP assert isinstance(ctype, ctypeobj.W_CType) self.space = space - self._cdata = cdata # don't forget keepalive_until_here! + self._ptr = ptr # don't access directly! use "with cdata as ptr:" self.ctype = ctype + def __enter__(self): + """Use 'with cdata as ptr:' to access the raw memory. It will + stay alive at least until the end of the 'with' block. + """ + return self._ptr + + def __exit__(self, *args): + keepalive_until_here(self) + + def unsafe_escaping_ptr(self): + """Generally unsafe: escape the pointer to raw memory. + If 'self' is a subclass that frees the pointer in a destructor, + it may be freed under your feet at any time. + """ + return self._ptr + def _repr_extra(self): - extra = self.ctype.extra_repr(self._cdata) - keepalive_until_here(self) + with self as ptr: + extra = self.ctype.extra_repr(ptr) return extra def _repr_extra_owning(self): @@ -54,11 +70,13 @@ self.ctype.name, extra1, extra2)) def nonzero(self): - return self.space.wrap(bool(self._cdata)) + with self as ptr: + nonzero = bool(ptr) + return self.space.wrap(nonzero) def int(self, space): - w_result = self.ctype.cast_to_int(self._cdata) - keepalive_until_here(self) + with self as ptr: + w_result = self.ctype.cast_to_int(ptr) return w_result def long(self, space): @@ -69,8 +87,8 @@ return w_result def float(self): - w_result = self.ctype.float(self._cdata) - keepalive_until_here(self) + with self as ptr: + w_result = self.ctype.float(ptr) return w_result def len(self): @@ -88,20 +106,19 @@ def _cmp(self, w_other): from pypy.module._cffi_backend.ctypeprim import W_CTypePrimitive space = self.space - cdata1 = self._cdata - if isinstance(w_other, W_CData): - cdata2 = w_other._cdata - else: + if not isinstance(w_other, W_CData): return space.w_NotImplemented - if requires_ordering: - if (isinstance(self.ctype, W_CTypePrimitive) or - isinstance(w_other.ctype, W_CTypePrimitive)): - raise OperationError(space.w_TypeError, - space.wrap("cannot do comparison on a primitive cdata")) - cdata1 = rffi.cast(lltype.Unsigned, cdata1) - cdata2 = rffi.cast(lltype.Unsigned, cdata2) - return space.newbool(op(cdata1, cdata2)) + with self as ptr1, w_other as ptr2: + if requires_ordering: + if (isinstance(self.ctype, W_CTypePrimitive) or + isinstance(w_other.ctype, W_CTypePrimitive)): + raise OperationError(space.w_TypeError, space.wrap( + "cannot do comparison on a primitive cdata")) + ptr1 = rffi.cast(lltype.Unsigned, ptr1) + ptr2 = rffi.cast(lltype.Unsigned, ptr2) + result = op(ptr1, ptr2) + return space.newbool(result) # return func_with_new_name(_cmp, name) @@ -113,7 +130,8 @@ ge = _make_comparison('ge') def hash(self): - h = rffi.cast(lltype.Signed, self._cdata) + ptr = self.unsafe_escaping_ptr() + h = rffi.cast(lltype.Signed, ptr) # To hash pointers in dictionaries. Assumes that h shows some # alignment (to 4, 8, maybe 16 bytes), so we use the following # formula to avoid the trailing bits being always 0. @@ -128,26 +146,27 @@ i = space.getindex_w(w_index, space.w_IndexError) ctype = self.ctype._check_subscript_index(self, i) w_o = self._do_getitem(ctype, i) - keepalive_until_here(self) return w_o def _do_getitem(self, ctype, i): ctitem = ctype.ctitem - return ctitem.convert_to_object( - rffi.ptradd(self._cdata, i * ctitem.size)) + with self as ptr: + return ctitem.convert_to_object( + rffi.ptradd(ptr, i * ctitem.size)) def setitem(self, w_index, w_value): space = self.space if space.isinstance_w(w_index, space.w_slice): - self._do_setslice(w_index, w_value) + with self as ptr: + self._do_setslice(w_index, w_value, ptr) else: i = space.getindex_w(w_index, space.w_IndexError) ctype = self.ctype._check_subscript_index(self, i) ctitem = ctype.ctitem - ctitem.convert_from_object( - rffi.ptradd(self._cdata, i * ctitem.size), - w_value) - keepalive_until_here(self) + with self as ptr: + ctitem.convert_from_object( + rffi.ptradd(ptr, i * ctitem.size), + w_value) def _do_getslicearg(self, w_slice): from pypy.module._cffi_backend.ctypeptr import W_CTypePointer @@ -188,14 +207,15 @@ ctarray = newtype.new_array_type(space, ctptr, space.w_None) ctptr.cache_array_type = ctarray # - p = rffi.ptradd(self._cdata, start * ctarray.ctitem.size) - return W_CDataSliced(space, p, ctarray, length) + ptr = self.unsafe_escaping_ptr() + ptr = rffi.ptradd(ptr, start * ctarray.ctitem.size) + return W_CDataSliced(space, ptr, ctarray, length) - def _do_setslice(self, w_slice, w_value): + def _do_setslice(self, w_slice, w_value, ptr): ctptr, start, length = self._do_getslicearg(w_slice) ctitem = ctptr.ctitem ctitemsize = ctitem.size - cdata = rffi.ptradd(self._cdata, start * ctitemsize) + target = rffi.ptradd(ptr, start * ctitemsize) # if isinstance(w_value, W_CData): from pypy.module._cffi_backend import ctypearray @@ -204,9 +224,8 @@ ctv.ctitem is ctitem and w_value.get_array_length() == length): # fast path: copying from exactly the correct type - s = w_value._cdata - rffi.c_memcpy(cdata, s, ctitemsize * length) - keepalive_until_here(w_value) + with w_value as source: + rffi.c_memcpy(target, source, ctitemsize * length) return # # A fast path for [0:N] = "somestring". @@ -221,7 +240,7 @@ raise oefmt(space.w_ValueError, "need a string of length %d, got %d", length, len(value)) - copy_string_to_raw(llstr(value), cdata, 0, length) + copy_string_to_raw(llstr(value), target, 0, length) return # w_iter = space.iter(w_value) @@ -233,8 +252,8 @@ raise raise oefmt(space.w_ValueError, "need %d values to unpack, got %d", length, i) - ctitem.convert_from_object(cdata, w_item) - cdata = rffi.ptradd(cdata, ctitemsize) + ctitem.convert_from_object(target, w_item) + target = rffi.ptradd(target, ctitemsize) try: space.next(w_iter) except OperationError, e: @@ -247,7 +266,8 @@ def _add_or_sub(self, w_other, sign): space = self.space i = sign * space.getindex_w(w_other, space.w_OverflowError) - return self.ctype.add(self._cdata, i) + ptr = self.unsafe_escaping_ptr() + return self.ctype.add(ptr, i) def add(self, w_other): return self._add_or_sub(w_other, +1) @@ -268,9 +288,11 @@ self.ctype.name, ct.name) # itemsize = ct.ctitem.size - if itemsize <= 0: itemsize = 1 - diff = (rffi.cast(lltype.Signed, self._cdata) - - rffi.cast(lltype.Signed, w_other._cdata)) // itemsize + if itemsize <= 0: + itemsize = 1 + with self as ptr1, w_other as ptr2: + diff = (rffi.cast(lltype.Signed, ptr1) - + rffi.cast(lltype.Signed, ptr2)) // itemsize return space.wrap(diff) # return self._add_or_sub(w_other, -1) @@ -279,17 +301,19 @@ return self.ctype.getcfield(self.space.str_w(w_attr)) def getattr(self, w_attr): - w_res = self.getcfield(w_attr).read(self._cdata) - keepalive_until_here(self) + cfield = self.getcfield(w_attr) + with self as ptr: + w_res = cfield.read(ptr) return w_res def setattr(self, w_attr, w_value): - self.getcfield(w_attr).write(self._cdata, w_value) - keepalive_until_here(self) + cfield = self.getcfield(w_attr) + with self as ptr: + cfield.write(ptr, w_value) def call(self, args_w): - w_result = self.ctype.call(self._cdata, args_w) - keepalive_until_here(self) + with self as ptr: + w_result = self.ctype.call(ptr, args_w) return w_result def iter(self): @@ -311,21 +335,21 @@ @specialize.argtype(1) def write_raw_signed_data(self, source): - misc.write_raw_signed_data(self._cdata, source, self.ctype.size) - keepalive_until_here(self) + with self as ptr: + misc.write_raw_signed_data(ptr, source, self.ctype.size) @specialize.argtype(1) def write_raw_unsigned_data(self, source): - misc.write_raw_unsigned_data(self._cdata, source, self.ctype.size) - keepalive_until_here(self) + with self as ptr: + misc.write_raw_unsigned_data(ptr, source, self.ctype.size) def write_raw_float_data(self, source): - misc.write_raw_float_data(self._cdata, source, self.ctype.size) - keepalive_until_here(self) + with self as ptr: + misc.write_raw_float_data(ptr, source, self.ctype.size) def convert_to_object(self): - w_obj = self.ctype.convert_to_object(self._cdata) - keepalive_until_here(self) + with self as ptr: + w_obj = self.ctype.convert_to_object(ptr) return w_obj def get_array_length(self): @@ -353,7 +377,7 @@ @rgc.must_be_light_finalizer def __del__(self): - lltype.free(self._cdata, flavor='raw') + lltype.free(self._ptr, flavor='raw') class W_CDataNewOwning(W_CDataMem): diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py --- a/pypy/module/_cffi_backend/ctypearray.py +++ b/pypy/module/_cffi_backend/ctypearray.py @@ -8,7 +8,6 @@ from pypy.interpreter.typedef import TypeDef from rpython.rtyper.lltypesystem import rffi -from rpython.rlib.objectmodel import keepalive_until_here from rpython.rlib.rarithmetic import ovfcheck from pypy.module._cffi_backend import cdataobj @@ -49,8 +48,8 @@ cdata = cdataobj.W_CDataNewOwning(space, datasize, self) # if not space.is_w(w_init, space.w_None): - self.convert_from_object(cdata._cdata, w_init) - keepalive_until_here(cdata) + with cdata as ptr: + self.convert_from_object(ptr, w_init) return cdata def _check_subscript_index(self, w_cdata, i): @@ -119,8 +118,8 @@ self.ctitem = ctitem self.cdata = cdata length = cdata.get_array_length() - self._next = cdata._cdata - self._stop = rffi.ptradd(cdata._cdata, length * ctitem.size) + self._next = cdata.unsafe_escaping_ptr() + self._stop = rffi.ptradd(self._next, length * ctitem.size) def iter_w(self): return self.space.wrap(self) diff --git a/pypy/module/_cffi_backend/ctypeenum.py b/pypy/module/_cffi_backend/ctypeenum.py --- a/pypy/module/_cffi_backend/ctypeenum.py +++ b/pypy/module/_cffi_backend/ctypeenum.py @@ -2,8 +2,6 @@ Enums. """ -from rpython.rlib.objectmodel import keepalive_until_here - from pypy.module._cffi_backend import misc from pypy.module._cffi_backend.ctypeprim import (W_CTypePrimitiveSigned, W_CTypePrimitiveUnsigned) @@ -47,8 +45,8 @@ return '%s: %s' % (value, s) def string(self, cdataobj, maxlen): - value = self._get_value(cdataobj._cdata) - keepalive_until_here(cdataobj) + with cdataobj as ptr: + value = self._get_value(ptr) try: s = self.enumvalues2erators[value] except KeyError: diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py --- a/pypy/module/_cffi_backend/ctypeobj.py +++ b/pypy/module/_cffi_backend/ctypeobj.py @@ -177,8 +177,8 @@ raise oefmt(space.w_AttributeError, "cdata '%s' has no attribute '%s'", self.name, attr) - def copy_and_convert_to_object(self, cdata): - return self.convert_to_object(cdata) + def copy_and_convert_to_object(self, source): + return self.convert_to_object(source) # __________ app-level attributes __________ def dir(self): diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py --- a/pypy/module/_cffi_backend/ctypeprim.py +++ b/pypy/module/_cffi_backend/ctypeprim.py @@ -5,7 +5,6 @@ import sys from rpython.rlib.rarithmetic import r_uint, r_ulonglong, intmask -from rpython.rlib.objectmodel import keepalive_until_here from rpython.rlib import jit from rpython.rtyper.lltypesystem import lltype, rffi @@ -53,7 +52,8 @@ space = self.space if (isinstance(w_ob, cdataobj.W_CData) and isinstance(w_ob.ctype, ctypeptr.W_CTypePtrOrArray)): - value = rffi.cast(lltype.Signed, w_ob._cdata) + ptr = w_ob.unsafe_escaping_ptr() + value = rffi.cast(lltype.Signed, ptr) value = self._cast_result(value) elif space.isinstance_w(w_ob, space.w_str): value = self.cast_str(w_ob) @@ -81,8 +81,8 @@ def string(self, cdataobj, maxlen): if self.size == 1: - s = cdataobj._cdata[0] - keepalive_until_here(cdataobj) + with cdataobj as ptr: + s = ptr[0] return self.space.wrap(s) return W_CType.string(self, cdataobj, maxlen) @@ -116,7 +116,8 @@ return s[0] if (isinstance(w_ob, cdataobj.W_CData) and isinstance(w_ob.ctype, W_CTypePrimitiveChar)): - return w_ob._cdata[0] + with w_ob as ptr: + return ptr[0] raise self._convert_error("string of length 1", w_ob) def convert_from_object(self, cdata, w_ob): @@ -137,8 +138,8 @@ return self.space.wrap(s) def string(self, cdataobj, maxlen): - w_res = self.convert_to_object(cdataobj._cdata) - keepalive_until_here(cdataobj) + with cdataobj as ptr: + w_res = self.convert_to_object(ptr) return w_res def _convert_to_unichar(self, w_ob): @@ -149,7 +150,8 @@ return s[0] if (isinstance(w_ob, cdataobj.W_CData) and isinstance(w_ob.ctype, W_CTypePrimitiveUniChar)): - return rffi.cast(rffi.CWCHARP, w_ob._cdata)[0] + with w_ob as ptr: + return rffi.cast(rffi.CWCHARP, ptr)[0] raise self._convert_error("unicode string of length 1", w_ob) def convert_from_object(self, cdata, w_ob): @@ -219,13 +221,15 @@ if self.size == rffi.sizeof(rffi.LONG): from rpython.rlib.rrawarray import populate_list_from_raw_array res = [] - buf = rffi.cast(rffi.LONGP, w_cdata._cdata) length = w_cdata.get_array_length() - populate_list_from_raw_array(res, buf, length) + with w_cdata as ptr: + buf = rffi.cast(rffi.LONGP, ptr) + populate_list_from_raw_array(res, buf, length) return res elif self.value_smaller_than_long: res = [0] * w_cdata.get_array_length() - misc.unpack_list_from_raw_array(res, w_cdata._cdata, self.size) + with w_cdata as ptr: + misc.unpack_list_from_raw_array(res, ptr, self.size) return res return None @@ -308,8 +312,8 @@ def unpack_list_of_int_items(self, w_cdata): if self.value_fits_long: res = [0] * w_cdata.get_array_length() - misc.unpack_unsigned_list_from_raw_array(res, w_cdata._cdata, - self.size) + with w_cdata as ptr: + misc.unpack_unsigned_list_from_raw_array(res, ptr, self.size) return res return None @@ -363,8 +367,8 @@ if not isinstance(self, W_CTypePrimitiveLongDouble): w_cdata.write_raw_float_data(value) else: - self._to_longdouble_and_write(value, w_cdata._cdata) - keepalive_until_here(w_cdata) + with w_cdata as ptr: + self._to_longdouble_and_write(value, ptr) return w_cdata def cast_to_int(self, cdata): @@ -387,13 +391,15 @@ if self.size == rffi.sizeof(rffi.DOUBLE): from rpython.rlib.rrawarray import populate_list_from_raw_array res = [] - buf = rffi.cast(rffi.DOUBLEP, w_cdata._cdata) length = w_cdata.get_array_length() - populate_list_from_raw_array(res, buf, length) + with w_cdata as ptr: + buf = rffi.cast(rffi.DOUBLEP, ptr) + populate_list_from_raw_array(res, buf, length) return res elif self.size == rffi.sizeof(rffi.FLOAT): res = [0.0] * w_cdata.get_array_length() - misc.unpack_cfloat_list_from_raw_array(res, w_cdata._cdata) + with w_cdata as ptr: + misc.unpack_cfloat_list_from_raw_array(res, ptr) return res return None @@ -423,8 +429,8 @@ def cast(self, w_ob): if (isinstance(w_ob, cdataobj.W_CData) and isinstance(w_ob.ctype, W_CTypePrimitiveLongDouble)): - w_cdata = self.convert_to_object(w_ob._cdata) - keepalive_until_here(w_ob) + with w_ob as ptr: + w_cdata = self.convert_to_object(ptr) return w_cdata else: return W_CTypePrimitiveFloat.cast(self, w_ob) @@ -451,16 +457,16 @@ def convert_to_object(self, cdata): w_cdata = cdataobj.W_CDataMem(self.space, self.size, self) - self._copy_longdouble(cdata, w_cdata._cdata) - keepalive_until_here(w_cdata) + with w_cdata as ptr: + self._copy_longdouble(cdata, ptr) return w_cdata def convert_from_object(self, cdata, w_ob): space = self.space if (isinstance(w_ob, cdataobj.W_CData) and isinstance(w_ob.ctype, W_CTypePrimitiveLongDouble)): - self._copy_longdouble(w_ob._cdata, cdata) - keepalive_until_here(w_ob) + with w_ob as ptr: + self._copy_longdouble(ptr, cdata) else: value = space.float_w(space.float(w_ob)) self._to_longdouble_and_write(value, cdata) diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py --- a/pypy/module/_cffi_backend/ctypeptr.py +++ b/pypy/module/_cffi_backend/ctypeptr.py @@ -3,7 +3,6 @@ """ from rpython.rlib import rposix -from rpython.rlib.objectmodel import keepalive_until_here from rpython.rlib.rarithmetic import ovfcheck from rpython.rtyper.annlowlevel import llstr, llunicode from rpython.rtyper.lltypesystem import lltype, rffi @@ -49,7 +48,7 @@ space = self.space if (isinstance(w_ob, cdataobj.W_CData) and isinstance(w_ob.ctype, W_CTypePtrOrArray)): - value = w_ob._cdata + value = w_ob.unsafe_escaping_ptr() else: value = misc.as_unsigned_long(space, w_ob, strict=False) value = rffi.cast(rffi.CCHARP, value) @@ -108,34 +107,33 @@ def string(self, cdataobj, maxlen): space = self.space if isinstance(self.ctitem, ctypeprim.W_CTypePrimitive): - cdata = cdataobj._cdata - if not cdata: - raise oefmt(space.w_RuntimeError, "cannot use string() on %s", - space.str_w(cdataobj.repr())) - # - from pypy.module._cffi_backend import ctypearray - length = maxlen - if length < 0 and isinstance(self, ctypearray.W_CTypeArray): - length = cdataobj.get_array_length() - # - # pointer to a primitive type of size 1: builds and returns a str - if self.ctitem.size == rffi.sizeof(lltype.Char): - if length < 0: - s = rffi.charp2str(cdata) - else: - s = rffi.charp2strn(cdata, length) - keepalive_until_here(cdataobj) - return space.wrap(s) - # - # pointer to a wchar_t: builds and returns a unicode - if self.is_unichar_ptr_or_array(): - cdata = rffi.cast(rffi.CWCHARP, cdata) - if length < 0: - u = rffi.wcharp2unicode(cdata) - else: - u = rffi.wcharp2unicoden(cdata, length) - keepalive_until_here(cdataobj) - return space.wrap(u) + with cdataobj as ptr: + if not ptr: + raise oefmt(space.w_RuntimeError, + "cannot use string() on %s", + space.str_w(cdataobj.repr())) + # + from pypy.module._cffi_backend import ctypearray + length = maxlen + if length < 0 and isinstance(self, ctypearray.W_CTypeArray): + length = cdataobj.get_array_length() + # + # pointer to a primitive type of size 1: builds and returns a str + if self.ctitem.size == rffi.sizeof(lltype.Char): + if length < 0: + s = rffi.charp2str(ptr) + else: + s = rffi.charp2strn(ptr, length) + return space.wrap(s) + # + # pointer to a wchar_t: builds and returns a unicode + if self.is_unichar_ptr_or_array(): + cdata = rffi.cast(rffi.CWCHARP, ptr) + if length < 0: + u = rffi.wcharp2unicode(cdata) + else: + u = rffi.wcharp2unicoden(cdata, length) + return space.wrap(u) # return W_CType.string(self, cdataobj, maxlen) @@ -162,7 +160,7 @@ if not (self.can_cast_anything or other.can_cast_anything): raise self._convert_error("compatible pointer", w_ob) - rffi.cast(rffi.CCHARPP, cdata)[0] = w_ob._cdata + rffi.cast(rffi.CCHARPP, cdata)[0] = w_ob.unsafe_escaping_ptr() def _alignof(self): from pypy.module._cffi_backend import newtype @@ -206,8 +204,8 @@ lltype.nullptr(rffi.CCHARP.TO), w_init, datasize) # cdatastruct = cdataobj.W_CDataNewOwning(space, datasize, ctitem) - cdata = cdataobj.W_CDataPtrToStructOrUnion(space, - cdatastruct._cdata, + ptr = cdatastruct.unsafe_escaping_ptr() + cdata = cdataobj.W_CDataPtrToStructOrUnion(space, ptr, self, cdatastruct) else: if self.is_char_or_unichar_ptr_or_array(): @@ -215,8 +213,8 @@ cdata = cdataobj.W_CDataNewOwning(space, datasize, self) # if not space.is_w(w_init, space.w_None): - ctitem.convert_from_object(cdata._cdata, w_init) - keepalive_until_here(cdata) + with cdata as ptr: + ctitem.convert_from_object(ptr, w_init) return cdata def _check_subscript_index(self, w_cdata, i): @@ -332,8 +330,9 @@ ctype2 = cdata.ctype if (isinstance(ctype2, W_CTypeStructOrUnion) or isinstance(ctype2, W_CTypePtrOrArray)): - ptrdata = rffi.ptradd(cdata._cdata, offset) - return cdataobj.W_CData(space, ptrdata, self) + ptr = cdata.unsafe_escaping_ptr() + ptr = rffi.ptradd(ptr, offset) + return cdataobj.W_CData(space, ptr, self) else: raise OperationError(space.w_TypeError, space.wrap("expected a cdata struct/union/array/pointer" diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py --- a/pypy/module/_cffi_backend/ctypestruct.py +++ b/pypy/module/_cffi_backend/ctypestruct.py @@ -7,7 +7,6 @@ from pypy.interpreter.typedef import TypeDef, interp_attrproperty from rpython.rlib import jit -from rpython.rlib.objectmodel import keepalive_until_here from rpython.rlib.rarithmetic import r_uint, r_ulonglong, r_longlong, intmask from rpython.rlib.rarithmetic import ovfcheck from rpython.rtyper.lltypesystem import lltype, rffi @@ -57,12 +56,12 @@ self.check_complete() return cdataobj.W_CData(space, cdata, self) - def copy_and_convert_to_object(self, cdata): + def copy_and_convert_to_object(self, source): space = self.space self.check_complete() ob = cdataobj.W_CDataNewOwning(space, self.size, self) - misc._raw_memcopy(cdata, ob._cdata, self.size) - keepalive_until_here(ob) + with ob as target: + misc._raw_memcopy(source, target, self.size) return ob def typeoffsetof_field(self, fieldname, following): @@ -80,8 +79,8 @@ def _copy_from_same(self, cdata, w_ob): if isinstance(w_ob, cdataobj.W_CData): if w_ob.ctype is self and self.size >= 0: - misc._raw_memcopy(w_ob._cdata, cdata, self.size) - keepalive_until_here(w_ob) + with w_ob as ptr: + misc._raw_memcopy(ptr, cdata, self.size) return True return False diff --git a/pypy/module/_cffi_backend/ctypevoid.py b/pypy/module/_cffi_backend/ctypevoid.py --- a/pypy/module/_cffi_backend/ctypevoid.py +++ b/pypy/module/_cffi_backend/ctypevoid.py @@ -13,5 +13,5 @@ def __init__(self, space): W_CType.__init__(self, space, -1, "void", len("void")) - def copy_and_convert_to_object(self, cdata): + def copy_and_convert_to_object(self, source): return self.space.w_None diff --git a/pypy/module/_cffi_backend/handle.py b/pypy/module/_cffi_backend/handle.py --- a/pypy/module/_cffi_backend/handle.py +++ b/pypy/module/_cffi_backend/handle.py @@ -34,8 +34,9 @@ raise oefmt(space.w_TypeError, "expected a 'cdata' object with a 'void *' out of " "new_handle(), got '%s'", ctype.name) - index = rffi.cast(lltype.Signed, w_cdata._cdata) - original_cdataobj = get(space).fetch_handle(index - 1) + with w_cdata as ptr: + index = rffi.cast(lltype.Signed, ptr) + original_cdataobj = get(space).fetch_handle(index - 1) # if isinstance(original_cdataobj, cdataobj.W_CDataHandle): return original_cdataobj.w_keepalive diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py --- a/pypy/module/_cffi_backend/misc.py +++ b/pypy/module/_cffi_backend/misc.py @@ -3,7 +3,7 @@ from pypy.interpreter.error import OperationError from rpython.rlib import jit -from rpython.rlib.objectmodel import keepalive_until_here, specialize +from rpython.rlib.objectmodel import specialize from rpython.rlib.rarithmetic import r_uint, r_ulonglong from rpython.rlib.unroll import unrolling_iterable from rpython.rtyper.lltypesystem import lltype, llmemory, rffi @@ -272,11 +272,11 @@ from pypy.module._cffi_backend.ctypeprim import W_CTypePrimitiveLongDouble is_cdata = isinstance(w_ob, W_CData) if is_cdata and isinstance(w_ob.ctype, W_CTypePrimitiveFloat): - if isinstance(w_ob.ctype, W_CTypePrimitiveLongDouble): - result = is_nonnull_longdouble(w_ob._cdata) - else: - result = is_nonnull_float(w_ob._cdata, w_ob.ctype.size) - keepalive_until_here(w_ob) + with w_ob as ptr: + if isinstance(w_ob.ctype, W_CTypePrimitiveLongDouble): + result = is_nonnull_longdouble(ptr) + else: + result = is_nonnull_float(ptr, w_ob.ctype.size) return result # if not is_cdata and space.lookup(w_ob, '__float__') is not None: From noreply at buildbot.pypy.org Wed Mar 4 17:27:55 2015 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 4 Mar 2015 17:27:55 +0100 (CET) Subject: [pypy-commit] pypy default: update values Message-ID: <20150304162755.70E731C0292@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r76244:9093d490654c Date: 2015-03-04 18:28 +0200 http://bitbucket.org/pypy/pypy/changeset/9093d490654c/ Log: update values diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py --- a/pypy/module/micronumpy/test/test_zjit.py +++ b/pypy/module/micronumpy/test/test_zjit.py @@ -238,15 +238,15 @@ 'getfield_gc': 5, 'getfield_gc_pure': 51, 'guard_class': 3, - 'guard_false': 13, + 'guard_false': 12, 'guard_nonnull': 11, 'guard_nonnull_class': 3, 'guard_not_invalidated': 2, 'guard_true': 10, - 'guard_value': 5, + 'guard_value': 6, 'int_add': 13, 'int_ge': 4, - 'int_is_true': 4, + 'int_is_true': 3, 'int_is_zero': 4, 'int_le': 2, 'int_lt': 3, @@ -616,15 +616,15 @@ 'getfield_gc': 6, 'getfield_gc_pure': 63, 'guard_class': 5, - 'guard_false': 20, + 'guard_false': 19, 'guard_nonnull': 6, 'guard_nonnull_class': 1, 'guard_not_invalidated': 3, 'guard_true': 16, - 'guard_value': 2, + 'guard_value': 3, 'int_add': 24, 'int_ge': 4, - 'int_is_true': 6, + 'int_is_true': 5, 'int_is_zero': 4, 'int_le': 5, 'int_lt': 7, From noreply at buildbot.pypy.org Wed Mar 4 17:27:54 2015 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 4 Mar 2015 17:27:54 +0100 (CET) Subject: [pypy-commit] pypy default: change concrete.get_storage to a context manager API, issue #1994 Message-ID: <20150304162754.5271A1C0292@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r76243:5cfbc35539b9 Date: 2015-03-04 18:24 +0200 http://bitbucket.org/pypy/pypy/changeset/5cfbc35539b9/ Log: change concrete.get_storage to a context manager API, issue #1994 diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py --- a/pypy/module/micronumpy/concrete.py +++ b/pypy/module/micronumpy/concrete.py @@ -12,6 +12,7 @@ from pypy.module.micronumpy.strides import (Chunk, Chunks, NewAxisChunk, RecordChunk, calc_strides, calc_new_strides, shape_agreement, calculate_broadcast_strides, calc_backstrides) +from rpython.rlib.objectmodel import keepalive_until_here class BaseConcreteArray(object): @@ -312,12 +313,18 @@ l_w = [w_res.descr_getitem(space, space.wrap(d)) for d in range(nd)] return space.newtuple(l_w) - def get_storage_as_int(self, space): - return rffi.cast(lltype.Signed, self.storage) + self.start + def get_storage_as_int(self): + return rffi.cast(lltype.Signed, self.storage) + self.start - def get_storage(self): + ##def get_storage(self): + ## return self.storage + ## use a safer context manager + def __enter__(self): return self.storage + def __exit__(self, typ, value, traceback): + keepalive_until_here(self) + def get_buffer(self, space, readonly): return ArrayBuffer(self, readonly) diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py --- a/pypy/module/micronumpy/loop.py +++ b/pypy/module/micronumpy/loop.py @@ -605,7 +605,7 @@ w_res_str = W_NDimArray.from_shape(space, [1], arr.get_dtype(), order='C') itemsize = arr.get_dtype().elsize res_str_casted = rffi.cast(rffi.CArrayPtr(lltype.Char), - w_res_str.implementation.get_storage_as_int(space)) + w_res_str.implementation.get_storage_as_int()) while not iter.done(state): w_res_str.implementation.setitem(0, iter.getitem(state)) for i in range(itemsize): diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -532,7 +532,7 @@ self.get_dtype(), storage_bytes=sz, w_base=self) def descr_array_iface(self, space): - addr = self.implementation.get_storage_as_int(space) + addr = self.implementation.get_storage_as_int() # will explode if it can't w_d = space.newdict() space.setitem_str(w_d, 'data', @@ -1165,7 +1165,8 @@ builder.append(box.raw_str()) state = iter.next(state) else: - builder.append_charpsize(self.implementation.get_storage(), + with self.implementation as storage: + builder.append_charpsize(storage, self.implementation.get_storage_size()) state = space.newtuple([ diff --git a/pypy/module/micronumpy/selection.py b/pypy/module/micronumpy/selection.py --- a/pypy/module/micronumpy/selection.py +++ b/pypy/module/micronumpy/selection.py @@ -33,14 +33,14 @@ self.values = values self.indexes = indexes - def getitem(self, item): + def getitem(self, idx): if count < 2: - v = raw_storage_getitem(TP, self.values, item * self.stride_size + v = raw_storage_getitem(TP, self.values, idx * self.stride_size + self.start) else: v = [] for i in range(count): - _v = raw_storage_getitem(TP, self.values, item * self.stride_size + _v = raw_storage_getitem(TP, self.values, idx * self.stride_size + self.start + step * i) v.append(_v) if comp_type == 'int': @@ -52,7 +52,7 @@ else: raise NotImplementedError('cannot reach') return (v, raw_storage_getitem(lltype.Signed, self.indexes, - item * self.index_stride_size + + idx * self.index_stride_size + self.index_start)) def setitem(self, idx, item): @@ -134,37 +134,37 @@ # create array of indexes dtype = descriptor.get_dtype_cache(space).w_longdtype index_arr = W_NDimArray.from_shape(space, arr.get_shape(), dtype) - storage = index_arr.implementation.get_storage() - if len(arr.get_shape()) == 1: - for i in range(arr.get_size()): - raw_storage_setitem(storage, i * INT_SIZE, i) - r = Repr(INT_SIZE, itemsize, arr.get_size(), arr.get_storage(), - storage, 0, arr.start) - ArgSort(r).sort() - else: - shape = arr.get_shape() - if axis < 0: - axis = len(shape) + axis - if axis < 0 or axis >= len(shape): - raise oefmt(space.w_IndexError, "Wrong axis %d", axis) - arr_iter = AllButAxisIter(arr, axis) - arr_state = arr_iter.reset() - index_impl = index_arr.implementation - index_iter = AllButAxisIter(index_impl, axis) - index_state = index_iter.reset() - stride_size = arr.strides[axis] - index_stride_size = index_impl.strides[axis] - axis_size = arr.shape[axis] - while not arr_iter.done(arr_state): - for i in range(axis_size): - raw_storage_setitem(storage, i * index_stride_size + - index_state.offset, i) - r = Repr(index_stride_size, stride_size, axis_size, - arr.get_storage(), storage, index_state.offset, arr_state.offset) + with index_arr.implementation as storage, arr as arr_storage: + if len(arr.get_shape()) == 1: + for i in range(arr.get_size()): + raw_storage_setitem(storage, i * INT_SIZE, i) + r = Repr(INT_SIZE, itemsize, arr.get_size(), arr_storage, + storage, 0, arr.start) ArgSort(r).sort() - arr_state = arr_iter.next(arr_state) - index_state = index_iter.next(index_state) - return index_arr + else: + shape = arr.get_shape() + if axis < 0: + axis = len(shape) + axis + if axis < 0 or axis >= len(shape): + raise oefmt(space.w_IndexError, "Wrong axis %d", axis) + arr_iter = AllButAxisIter(arr, axis) + arr_state = arr_iter.reset() + index_impl = index_arr.implementation + index_iter = AllButAxisIter(index_impl, axis) + index_state = index_iter.reset() + stride_size = arr.strides[axis] + index_stride_size = index_impl.strides[axis] + axis_size = arr.shape[axis] + while not arr_iter.done(arr_state): + for i in range(axis_size): + raw_storage_setitem(storage, i * index_stride_size + + index_state.offset, i) + r = Repr(index_stride_size, stride_size, axis_size, + arr_storage, storage, index_state.offset, arr_state.offset) + ArgSort(r).sort() + arr_state = arr_iter.next(arr_state) + index_state = index_iter.next(index_state) + return index_arr return argsort @@ -282,25 +282,25 @@ axis = -1 else: axis = space.int_w(w_axis) - # create array of indexes - if len(arr.get_shape()) == 1: - r = Repr(itemsize, arr.get_size(), arr.get_storage(), - arr.start) - ArgSort(r).sort() - else: - shape = arr.get_shape() - if axis < 0: - axis = len(shape) + axis - if axis < 0 or axis >= len(shape): - raise oefmt(space.w_IndexError, "Wrong axis %d", axis) - arr_iter = AllButAxisIter(arr, axis) - arr_state = arr_iter.reset() - stride_size = arr.strides[axis] - axis_size = arr.shape[axis] - while not arr_iter.done(arr_state): - r = Repr(stride_size, axis_size, arr.get_storage(), arr_state.offset) + with arr as storage: + if len(arr.get_shape()) == 1: + r = Repr(itemsize, arr.get_size(), storage, + arr.start) ArgSort(r).sort() - arr_state = arr_iter.next(arr_state) + else: + shape = arr.get_shape() + if axis < 0: + axis = len(shape) + axis + if axis < 0 or axis >= len(shape): + raise oefmt(space.w_IndexError, "Wrong axis %d", axis) + arr_iter = AllButAxisIter(arr, axis) + arr_state = arr_iter.reset() + stride_size = arr.strides[axis] + axis_size = arr.shape[axis] + while not arr_iter.done(arr_state): + r = Repr(stride_size, axis_size, storage, arr_state.offset) + ArgSort(r).sort() + arr_state = arr_iter.next(arr_state) return sort diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py --- a/pypy/module/micronumpy/ufuncs.py +++ b/pypy/module/micronumpy/ufuncs.py @@ -1407,7 +1407,7 @@ space.wrap("cannot mix ndarray and %r (arg %d) in call to ufunc" % ( arg_i, i))) raw_storage_setitem(dataps, CCHARP_SIZE * i, - rffi.cast(rffi.CCHARP, arg_i.implementation.get_storage_as_int(space))) + rffi.cast(rffi.CCHARP, arg_i.implementation.get_storage_as_int())) #This assumes we iterate over the whole array (it should be a view...) raw_storage_setitem(self.dims, LONG_SIZE * i, rffi.cast(rffi.LONG, arg_i.get_size())) raw_storage_setitem(self.steps, LONG_SIZE * i, rffi.cast(rffi.LONG, arg_i.get_dtype().elsize)) @@ -1416,7 +1416,7 @@ arg_i = args_w[i] assert isinstance(arg_i, W_NDimArray) raw_storage_setitem(dataps, CCHARP_SIZE * i, - rffi.cast(rffi.CCHARP, arg_i.implementation.get_storage_as_int(space))) + rffi.cast(rffi.CCHARP, arg_i.implementation.get_storage_as_int())) try: arg1 = rffi.cast(rffi.CArrayPtr(rffi.CCHARP), dataps) arg2 = rffi.cast(npy_intpp, self.dims) From noreply at buildbot.pypy.org Wed Mar 4 20:14:20 2015 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 4 Mar 2015 20:14:20 +0100 (CET) Subject: [pypy-commit] pypy default: more "x._cdata -> with x as ptr" conversions Message-ID: <20150304191420.716531C014B@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r76245:e5ad346030ac Date: 2015-03-04 21:13 +0200 http://bitbucket.org/pypy/pypy/changeset/e5ad346030ac/ Log: more "x._cdata -> with x as ptr" conversions diff --git a/pypy/module/cppyy/capi/loadable_capi.py b/pypy/module/cppyy/capi/loadable_capi.py --- a/pypy/module/cppyy/capi/loadable_capi.py +++ b/pypy/module/cppyy/capi/loadable_capi.py @@ -259,7 +259,8 @@ if not objectmodel.we_are_translated(): leakfinder.remember_free(c_call.ctype.cif_descr._obj0) state.capi_calls[name] = c_call - return c_call.ctype.rcall(c_call._cdata, args) + with c_call as ptr: + return c_call.ctype.rcall(ptr, args) def _cdata_to_cobject(space, w_cdata): return rffi.cast(C_OBJECT, space.uint_w(w_cdata)) @@ -271,8 +272,8 @@ return rffi.cast(rffi.LONG, space.int_w(w_cdata)) def _cdata_to_ptr(space, w_cdata): # TODO: this is both a hack and dreadfully slow - return rffi.cast(rffi.VOIDP, - space.interp_w(cdataobj.W_CData, w_cdata, can_be_None=False)._cdata) + with space.interp_w(cdataobj.W_CData, w_cdata, can_be_None=False) as ptr: + return rffi.cast(rffi.VOIDP, ptr) def c_load_dictionary(name): return libffi.CDLL(name) From noreply at buildbot.pypy.org Wed Mar 4 20:14:21 2015 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 4 Mar 2015 20:14:21 +0100 (CET) Subject: [pypy-commit] pypy default: more carefully guard access to storage by using a context manager Message-ID: <20150304191421.9F9101C014B@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r76246:6cba1a6d7ad1 Date: 2015-03-04 21:15 +0200 http://bitbucket.org/pypy/pypy/changeset/6cba1a6d7ad1/ Log: more carefully guard access to storage by using a context manager diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py --- a/pypy/module/micronumpy/base.py +++ b/pypy/module/micronumpy/base.py @@ -129,6 +129,9 @@ def get_order(self): return self.implementation.order + def get_start(self): + return self.implementation.start + def ndims(self): return len(self.get_shape()) ndims._always_inline_ = True diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py --- a/pypy/module/micronumpy/concrete.py +++ b/pypy/module/micronumpy/concrete.py @@ -313,9 +313,6 @@ l_w = [w_res.descr_getitem(space, space.wrap(d)) for d in range(nd)] return space.newtuple(l_w) - def get_storage_as_int(self): - return rffi.cast(lltype.Signed, self.storage) + self.start - ##def get_storage(self): ## return self.storage ## use a safer context manager diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py --- a/pypy/module/micronumpy/ctors.py +++ b/pypy/module/micronumpy/ctors.py @@ -100,10 +100,11 @@ elems_w[i] = w_object.implementation.getitem(i * elsize) else: imp = w_object.implementation - sz = support.product(w_object.get_shape()) * dtype.elsize - return W_NDimArray.from_shape_and_storage(space, - w_object.get_shape(),imp.storage, dtype, storage_bytes=sz, - w_base=w_object, start=imp.start) + with imp as storage: + sz = support.product(w_object.get_shape()) * dtype.elsize + return W_NDimArray.from_shape_and_storage(space, + w_object.get_shape(), storage, dtype, storage_bytes=sz, + w_base=w_object, start=imp.start) else: # not an array shape, elems_w = strides.find_shape_and_elems(space, w_object, dtype) diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py --- a/pypy/module/micronumpy/loop.py +++ b/pypy/module/micronumpy/loop.py @@ -604,14 +604,15 @@ iter, state = arr.create_iter() w_res_str = W_NDimArray.from_shape(space, [1], arr.get_dtype(), order='C') itemsize = arr.get_dtype().elsize - res_str_casted = rffi.cast(rffi.CArrayPtr(lltype.Char), - w_res_str.implementation.get_storage_as_int()) - while not iter.done(state): - w_res_str.implementation.setitem(0, iter.getitem(state)) - for i in range(itemsize): - builder.append(res_str_casted[i]) - state = iter.next(state) - return builder.build() + with w_res_str.implementation as storage: + res_str_casted = rffi.cast(rffi.CArrayPtr(lltype.Char), + support.get_storage_as_int(storage)) + while not iter.done(state): + w_res_str.implementation.setitem(0, iter.getitem(state)) + for i in range(itemsize): + builder.append(res_str_casted[i]) + state = iter.next(state) + return builder.build() getitem_int_driver = jit.JitDriver(name = 'numpy_getitem_int', greens = ['shapelen', 'indexlen', diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -532,20 +532,25 @@ self.get_dtype(), storage_bytes=sz, w_base=self) def descr_array_iface(self, space): - addr = self.implementation.get_storage_as_int() - # will explode if it can't - w_d = space.newdict() - space.setitem_str(w_d, 'data', - space.newtuple([space.wrap(addr), space.w_False])) - space.setitem_str(w_d, 'shape', self.descr_get_shape(space)) - space.setitem_str(w_d, 'typestr', self.get_dtype().descr_get_str(space)) - if self.implementation.order == 'C': - # Array is contiguous, no strides in the interface. - strides = space.w_None - else: - strides = self.descr_get_strides(space) - space.setitem_str(w_d, 'strides', strides) - return w_d + ''' + Note: arr.__array__.data[0] is a pointer so arr must be kept alive + while it is in use + ''' + with self.implementation as storage: + addr = support.get_storage_as_int(storage, self.get_start()) + # will explode if it can't + w_d = space.newdict() + space.setitem_str(w_d, 'data', + space.newtuple([space.wrap(addr), space.w_False])) + space.setitem_str(w_d, 'shape', self.descr_get_shape(space)) + space.setitem_str(w_d, 'typestr', self.get_dtype().descr_get_str(space)) + if self.implementation.order == 'C': + # Array is contiguous, no strides in the interface. + strides = space.w_None + else: + strides = self.descr_get_strides(space) + space.setitem_str(w_d, 'strides', strides) + return w_d w_pypy_data = None diff --git a/pypy/module/micronumpy/support.py b/pypy/module/micronumpy/support.py --- a/pypy/module/micronumpy/support.py +++ b/pypy/module/micronumpy/support.py @@ -1,6 +1,7 @@ from pypy.interpreter.error import OperationError, oefmt from rpython.rlib import jit from rpython.rlib.rarithmetic import ovfcheck +from rpython.rtyper.lltypesystem import rffi, lltype def issequence_w(space, w_obj): @@ -147,3 +148,7 @@ if cur_core_dim == 0: ufunc.core_enabled = False return 0 # for historical reasons, any failures will raise + +def get_storage_as_int(storage, start=0): + return rffi.cast(lltype.Signed, storage) + start + diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -180,13 +180,16 @@ raw_storage_setitem_unaligned(storage, i + offset, value) def read(self, arr, i, offset, dtype=None): - return self.box(self._read(arr.storage, i, offset)) + with arr as storage: + return self.box(self._read(storage, i, offset)) def read_bool(self, arr, i, offset): - return bool(self.for_computation(self._read(arr.storage, i, offset))) + with arr as storage: + return bool(self.for_computation(self._read(storage, i, offset))) def store(self, arr, i, offset, box): - self._write(arr.storage, i, offset, self.unbox(box)) + with arr as storage: + self._write(storage, i, offset, self.unbox(box)) def fill(self, storage, width, box, start, stop, offset): value = self.unbox(box) @@ -1080,8 +1083,9 @@ return bool(real) or bool(imag) def read_bool(self, arr, i, offset): - v = self.for_computation(self._read(arr.storage, i, offset)) - return bool(v[0]) or bool(v[1]) + with arr as storage: + v = self.for_computation(self._read(storage, i, offset)) + return bool(v[0]) or bool(v[1]) def get_element_size(self): return 2 * rffi.sizeof(self.T) @@ -1132,8 +1136,9 @@ return real, imag def read(self, arr, i, offset, dtype=None): - real, imag = self._read(arr.storage, i, offset) - return self.box_complex(real, imag) + with arr as storage: + real, imag = self._read(storage, i, offset) + return self.box_complex(real, imag) def _write(self, storage, i, offset, value): real, imag = value @@ -1144,7 +1149,8 @@ raw_storage_setitem_unaligned(storage, i + offset + rffi.sizeof(self.T), imag) def store(self, arr, i, offset, box): - self._write(arr.storage, i, offset, self.unbox(box)) + with arr as storage: + self._write(storage, i, offset, self.unbox(box)) def fill(self, storage, width, box, start, stop, offset): value = self.unbox(box) @@ -1633,13 +1639,14 @@ assert isinstance(item, boxes.W_FlexibleBox) i = item.ofs end = i + item.dtype.elsize - while i < end: - assert isinstance(item.arr.storage[i], str) - if item.arr.storage[i] == '\x00': - break - builder.append(item.arr.storage[i]) - i += 1 - return builder.build() + with item.arr as storage: + while i < end: + assert isinstance(storage[i], str) + if storage[i] == '\x00': + break + builder.append(storage[i]) + i += 1 + return builder.build() def str_unary_op(func): specialize.argtype(1)(func) @@ -1669,23 +1676,26 @@ w_item = space.wrap('') arg = space.str_w(space.str(w_item)) arr = VoidBoxStorage(dtype.elsize, dtype) - j = min(len(arg), dtype.elsize) - for i in range(j): - arr.storage[i] = arg[i] - for j in range(j, dtype.elsize): - arr.storage[j] = '\x00' - return boxes.W_StringBox(arr, 0, arr.dtype) + with arr as storage: + j = min(len(arg), dtype.elsize) + for i in range(j): + storage[i] = arg[i] + for j in range(j, dtype.elsize): + storage[j] = '\x00' + return boxes.W_StringBox(arr, 0, arr.dtype) def store(self, arr, i, offset, box): assert isinstance(box, boxes.W_StringBox) size = min(arr.dtype.elsize - offset, box.arr.size - box.ofs) - return self._store(arr.storage, i, offset, box, size) + with arr as storage: + return self._store(storage, i, offset, box, size) @jit.unroll_safe def _store(self, storage, i, offset, box, size): assert isinstance(box, boxes.W_StringBox) - for k in range(size): - storage[k + offset + i] = box.arr.storage[k + box.ofs] + with box.arr as box_storage: + for k in range(size): + storage[k + offset + i] = box_storage[k + box.ofs] def read(self, arr, i, offset, dtype=None): if dtype is None: @@ -1802,8 +1812,9 @@ assert i == 0 assert isinstance(box, boxes.W_VoidBox) assert box.dtype is box.arr.dtype - for k in range(box.arr.dtype.elsize): - arr.storage[k + ofs] = box.arr.storage[k + box.ofs] + with arr as arr_storage, box.arr as box_storage: + for k in range(box.arr.dtype.elsize): + arr_storage[k + ofs] = box_storage[k + box.ofs] def readarray(self, arr, i, offset, dtype=None): from pypy.module.micronumpy.base import W_NDimArray @@ -1893,12 +1904,14 @@ def store(self, arr, i, ofs, box): assert isinstance(box, boxes.W_VoidBox) - self._store(arr.storage, i, ofs, box, box.dtype.elsize) + with arr as storage: + self._store(storage, i, ofs, box, box.dtype.elsize) @jit.unroll_safe def _store(self, storage, i, ofs, box, size): - for k in range(size): - storage[k + i + ofs] = box.arr.storage[k + box.ofs] + with box.arr as box_storage: + for k in range(size): + storage[k + i + ofs] = box_storage[k + box.ofs] def fill(self, storage, width, box, start, stop, offset): assert isinstance(box, boxes.W_VoidBox) @@ -1944,9 +1957,10 @@ s1 = v1.dtype.elsize s2 = v2.dtype.elsize assert s1 == s2 - for i in range(s1): - if v1.arr.storage[v1.ofs + i] != v2.arr.storage[v2.ofs + i]: - return False + with v1.arr as v1_storage, v2.arr as v2_storage: + for i in range(s1): + if v1_storage[v1.ofs + i] != v2_storage[v2.ofs + i]: + return False return True def ne(self, v1, v2): diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py --- a/pypy/module/micronumpy/ufuncs.py +++ b/pypy/module/micronumpy/ufuncs.py @@ -13,11 +13,12 @@ from pypy.module.micronumpy.ctors import numpify from pypy.module.micronumpy.nditer import W_NDIter, coalesce_iter from pypy.module.micronumpy.strides import shape_agreement -from pypy.module.micronumpy.support import _parse_signature, product +from pypy.module.micronumpy.support import _parse_signature, product, get_storage_as_int from rpython.rlib.rawstorage import (raw_storage_setitem, free_raw_storage, alloc_raw_storage) from rpython.rtyper.lltypesystem import rffi, lltype from rpython.rlib.rarithmetic import LONG_BIT, _get_bitsize +from rpython.rlib.objectmodel import keepalive_until_here def done_if_true(dtype, val): @@ -98,7 +99,9 @@ if out is not None and not isinstance(out, W_NDimArray): raise OperationError(space.w_TypeError, space.wrap( 'output must be an array')) - return self.call(space, args_w, sig, casting, extobj) + retval = self.call(space, args_w, sig, casting, extobj) + keepalive_until_here(args_w) + return retval def descr_accumulate(self, space, w_obj, w_axis=None, w_dtype=None, w_out=None): if space.is_none(w_axis): @@ -804,11 +807,12 @@ assert isinstance(curarg, W_NDimArray) if len(arg_shapes[i]) != curarg.ndims(): # reshape + sz = product(curarg.get_shape()) * curarg.get_dtype().elsize - inargs[i] = W_NDimArray.from_shape_and_storage( - space, arg_shapes[i], curarg.implementation.storage, - curarg.get_dtype(), storage_bytes=sz, w_base=curarg) - pass + with curarg.implementation as storage: + inargs[i] = W_NDimArray.from_shape_and_storage( + space, arg_shapes[i], storage, + curarg.get_dtype(), storage_bytes=sz, w_base=curarg) need_to_cast.append(curarg.get_dtype() != dtypes[i]) for i in range(len(outargs)): j = self.nin + i @@ -819,9 +823,10 @@ elif len(arg_shapes[i]) != curarg.ndims(): # reshape sz = product(curarg.get_shape()) * curarg.get_dtype().elsize - outargs[i] = W_NDimArray.from_shape_and_storage( - space, arg_shapes[i], curarg.implementation.storage, - curarg.get_dtype(), storage_bytes=sz, w_base=curarg) + with curarg.implementation as storage: + outargs[i] = W_NDimArray.from_shape_and_storage( + space, arg_shapes[i], storage, + curarg.get_dtype(), storage_bytes=sz, w_base=curarg) curarg = outargs[i] assert isinstance(curarg, W_NDimArray) need_to_cast.append(curarg.get_dtype() != dtypes[j]) @@ -1406,8 +1411,9 @@ raise OperationError(space.w_NotImplementedError, space.wrap("cannot mix ndarray and %r (arg %d) in call to ufunc" % ( arg_i, i))) - raw_storage_setitem(dataps, CCHARP_SIZE * i, - rffi.cast(rffi.CCHARP, arg_i.implementation.get_storage_as_int())) + with arg_i.implementation as storage: + addr = get_storage_as_int(storage, arg_i.get_start()) + raw_storage_setitem(dataps, CCHARP_SIZE * i, rffi.cast(rffi.CCHARP, addr)) #This assumes we iterate over the whole array (it should be a view...) raw_storage_setitem(self.dims, LONG_SIZE * i, rffi.cast(rffi.LONG, arg_i.get_size())) raw_storage_setitem(self.steps, LONG_SIZE * i, rffi.cast(rffi.LONG, arg_i.get_dtype().elsize)) @@ -1415,8 +1421,9 @@ for i in range(len(args_w)): arg_i = args_w[i] assert isinstance(arg_i, W_NDimArray) - raw_storage_setitem(dataps, CCHARP_SIZE * i, - rffi.cast(rffi.CCHARP, arg_i.implementation.get_storage_as_int())) + with arg_i.implementation as storage: + addr = get_storage_as_int(storage, arg_i.get_start()) + raw_storage_setitem(dataps, CCHARP_SIZE * i, rffi.cast(rffi.CCHARP, addr)) try: arg1 = rffi.cast(rffi.CArrayPtr(rffi.CCHARP), dataps) arg2 = rffi.cast(npy_intpp, self.dims) @@ -1424,6 +1431,7 @@ self.func(arg1, arg2, arg3, self.data) finally: free_raw_storage(dataps, track_allocation=False) + keepalive_until_here(args_w) def set_dims_and_steps(self, space, dims, steps): if not isinstance(dims, list) or not isinstance(steps, list): From noreply at buildbot.pypy.org Wed Mar 4 20:28:44 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 4 Mar 2015 20:28:44 +0100 (CET) Subject: [pypy-commit] pypy default: Use unsafe_escaping_ptr() in this case because the pointer value is escaping. Message-ID: <20150304192844.E5F901C009F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76247:19539c5ed150 Date: 2015-03-04 20:28 +0100 http://bitbucket.org/pypy/pypy/changeset/19539c5ed150/ Log: Use unsafe_escaping_ptr() in this case because the pointer value is escaping. diff --git a/pypy/module/cppyy/capi/loadable_capi.py b/pypy/module/cppyy/capi/loadable_capi.py --- a/pypy/module/cppyy/capi/loadable_capi.py +++ b/pypy/module/cppyy/capi/loadable_capi.py @@ -272,8 +272,9 @@ return rffi.cast(rffi.LONG, space.int_w(w_cdata)) def _cdata_to_ptr(space, w_cdata): # TODO: this is both a hack and dreadfully slow - with space.interp_w(cdataobj.W_CData, w_cdata, can_be_None=False) as ptr: - return rffi.cast(rffi.VOIDP, ptr) + w_cdata = space.interp_w(cdataobj.W_CData, w_cdata, can_be_None=False) + ptr = w_cdata.unsafe_escaping_ptr() + return rffi.cast(rffi.VOIDP, ptr) def c_load_dictionary(name): return libffi.CDLL(name) From noreply at buildbot.pypy.org Thu Mar 5 13:43:28 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 5 Mar 2015 13:43:28 +0100 (CET) Subject: [pypy-commit] pypy default: fix for _ctype -> _ptr Message-ID: <20150305124328.76AEB1C0F16@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r76248:246bbfec1305 Date: 2015-03-05 14:43 +0200 http://bitbucket.org/pypy/pypy/changeset/246bbfec1305/ Log: fix for _ctype -> _ptr diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py --- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py +++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py @@ -341,7 +341,7 @@ raw_store(i119, 0, i160, descr=) raw_store(i119, 2, i160, descr=) raw_store(i119, 4, i160, descr=) - setfield_gc(p167, i119, descr=) + setfield_gc(p167, i119, descr=) i123 = arraylen_gc(p67, descr=) jump(..., descr=...) """) From noreply at buildbot.pypy.org Thu Mar 5 15:19:06 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Thu, 5 Mar 2015 15:19:06 +0100 (CET) Subject: [pypy-commit] pypy default: the optimization does not actually work for -maxint-1 Message-ID: <20150305141906.79D1C1C04A7@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: Changeset: r76249:58bec7621db5 Date: 2015-03-05 15:17 +0100 http://bitbucket.org/pypy/pypy/changeset/58bec7621db5/ Log: the optimization does not actually work for -maxint-1 diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -862,6 +862,7 @@ return resbox.constbox() def pure_reverse(self, op): + import sys if self.optpure is None: return optpure = self.optpure @@ -875,10 +876,14 @@ if isinstance(arg0, ConstInt): # invert the constant i0 = arg0.getint() + if i0 == -sys.maxint - 1: + return inv_arg0 = ConstInt(-i0) elif isinstance(arg1, ConstInt): # commutative i0 = arg1.getint() + if i0 == -sys.maxint - 1: + return inv_arg0 = ConstInt(-i0) arg1 = arg0 else: @@ -896,6 +901,8 @@ if isinstance(arg1, ConstInt): # invert the constant i1 = arg1.getint() + if i1 == -sys.maxint - 1: + return inv_arg1 = ConstInt(-i1) optpure.pure(rop.INT_ADD, [arg0, inv_arg1], op.result) optpure.pure(rop.INT_ADD, [inv_arg1, arg0], op.result) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -3704,6 +3704,7 @@ self.optimize_loop(ops, expected) def test_int_add_sub_constants_inverse(self): + import sys ops = """ [i0, i10, i11, i12, i13] i2 = int_add(1, i0) @@ -3748,6 +3749,14 @@ jump(i0, i2, i2, i2, i0, -1) """ self.optimize_loop(ops, expected) + ops = """ + [i0, i10, i11, i12] + i2 = int_add(%s, i0) + i3 = int_add(i2, %s) + i4 = int_sub(i0, %s) + jump(i0, i2, i3, i4) + """ % ((-sys.maxint - 1, ) * 3) + self.optimize_loop(ops, ops) # does not crash def test_framestackdepth_overhead(self): ops = """ From noreply at buildbot.pypy.org Thu Mar 5 22:50:39 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 5 Mar 2015 22:50:39 +0100 (CET) Subject: [pypy-commit] pypy default: fix more by induction than tdd, fix test too Message-ID: <20150305215039.7B22C1C0096@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r76250:a4663675b84a Date: 2015-03-05 23:51 +0200 http://bitbucket.org/pypy/pypy/changeset/a4663675b84a/ Log: fix more by induction than tdd, fix test too diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -906,7 +906,7 @@ inv_arg1 = ConstInt(-i1) optpure.pure(rop.INT_ADD, [arg0, inv_arg1], op.result) optpure.pure(rop.INT_ADD, [inv_arg1, arg0], op.result) - optpure.pure(rop.INT_SUB, [inv_arg1, op.result], arg0) + optpure.pure(rop.INT_SUB, [op.result, inv_arg1], arg0) optpure.pure(rop.INT_SUB, [op.result, arg0], inv_arg1) elif op.getopnum() == rop.FLOAT_MUL: optpure.pure(rop.FLOAT_MUL, [op.getarg(1), op.getarg(0)], op.result) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -3739,7 +3739,7 @@ i2 = int_sub(i0, 1) i3 = int_add(-1, i0) i4 = int_add(i0, -1) - i5 = int_sub(-1, i2) + i5 = int_sub(i2, -1) i6 = int_sub(i2, i0) jump(i0, i2, i3, i4, i5, i6) """ From noreply at buildbot.pypy.org Fri Mar 6 13:46:55 2015 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 6 Mar 2015 13:46:55 +0100 (CET) Subject: [pypy-commit] pypy recent-pure-ops: fix the rest of the tests, this branch either goes in or dies Message-ID: <20150306124655.50BFD1C00FC@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: recent-pure-ops Changeset: r76251:9bf72dc42671 Date: 2015-03-06 14:46 +0200 http://bitbucket.org/pypy/pypy/changeset/9bf72dc42671/ Log: fix the rest of the tests, this branch either goes in or dies diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py --- a/rpython/jit/metainterp/optimizeopt/intbounds.py +++ b/rpython/jit/metainterp/optimizeopt/intbounds.py @@ -214,15 +214,15 @@ # Else, synthesize the non overflowing op for optimize_default to # reuse, as well as the reverse op elif opnum == rop.INT_ADD_OVF: - self.pure(rop.INT_ADD, args[:], result) + #self.pure(rop.INT_ADD, args[:], result) self.pure(rop.INT_SUB, [result, args[1]], args[0]) self.pure(rop.INT_SUB, [result, args[0]], args[1]) elif opnum == rop.INT_SUB_OVF: - self.pure(rop.INT_SUB, args[:], result) + #self.pure(rop.INT_SUB, args[:], result) self.pure(rop.INT_ADD, [result, args[1]], args[0]) self.pure(rop.INT_SUB, [args[0], result], args[1]) - elif opnum == rop.INT_MUL_OVF: - self.pure(rop.INT_MUL, args[:], result) + #elif opnum == rop.INT_MUL_OVF: + # self.pure(rop.INT_MUL, args[:], result) self.emit_operation(op) def optimize_GUARD_OVERFLOW(self, op): diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -1,7 +1,7 @@ from rpython.jit.metainterp import jitprof, resume, compile from rpython.jit.metainterp.executor import execute_nonspec from rpython.jit.metainterp.history import BoxInt, BoxFloat, Const, ConstInt,\ - REF, BoxPtr, ConstPtr, ConstFloat + REF, BoxPtr, ConstPtr, ConstFloat, Box from rpython.jit.metainterp.optimizeopt.intutils import IntBound, IntUnbounded,\ IntLowerBound, MININT,\ MAXINT @@ -528,6 +528,7 @@ class Optimizer(Optimization): exporting_state = False + emitting_dissabled = False def __init__(self, metainterp_sd, jitdriver_sd, loop, optimizations=None): self.metainterp_sd = metainterp_sd diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py --- a/rpython/jit/metainterp/optimizeopt/pure.py +++ b/rpython/jit/metainterp/optimizeopt/pure.py @@ -53,6 +53,7 @@ self._pure_operations = [None] * (rop._ALWAYS_PURE_LAST - rop._ALWAYS_PURE_FIRST) self.call_pure_positions = [] + self.extra_call_pure = [] def propagate_forward(self, op): dispatch_opt(self, op) @@ -104,7 +105,10 @@ self.emit_operation(nextop) def getrecentops(self, opnum): - opnum = opnum - rop._ALWAYS_PURE_FIRST + if rop._OVF_FIRST <= opnum <= rop._OVF_LAST: + opnum = opnum - rop._OVF_FIRST + else: + opnum = opnum - rop._ALWAYS_PURE_FIRST assert 0 <= opnum < len(self._pure_operations) recentops = self._pure_operations[opnum] if recentops is None: @@ -122,22 +126,41 @@ # Step 2: check if all arguments are the same as a previous # CALL_PURE. - args = self.optimizer.make_args_key(op) - oldvalue = self.pure_operations.get(args, None) - if oldvalue is not None: - # this removes a CALL_PURE that has the same (non-constant) - # arguments as a previous CALL_PURE. - self.make_equal_to(op.result, oldvalue) - self.last_emitted_operation = REMOVED - return - else: - self.pure_operations[args] = self.getvalue(op.result) + for pos in self.call_pure_positions: + old_op = self.optimizer._newoperations[pos] + if self.optimize_call_pure(op, old_op): + return + for old_op in self.extra_call_pure: + if self.optimize_call_pure(op, old_op): + return # replace CALL_PURE with just CALL args = op.getarglist() self.emit_operation(ResOperation(rop.CALL, args, op.result, op.getdescr())) - self.call_pure_positions.append(len(self.optimizer._newoperations) - 1) + if self.optimizer.emitting_dissabled: + self.extra_call_pure.append(op) # XXX + else: + self.call_pure_positions.append(len(self.optimizer._newoperations) + - 1) + + def optimize_call_pure(self, op, old_op): + if (op.numargs() != old_op.numargs() or + op.getdescr() is not old_op.getdescr()): + return False + for i, box in enumerate(old_op.getarglist()): + if not self.get_box_replacement(op.getarg(i)).same_box(box): + break + else: + # all identical + # this removes a CALL_PURE that has the same (non-constant) + # arguments as a previous CALL_PURE. + oldvalue = self.getvalue(old_op.result) + self.make_equal_to(op.result, oldvalue) + self.last_emitted_operation = REMOVED + return True + return False + def optimize_GUARD_NO_EXCEPTION(self, op): if self.last_emitted_operation is REMOVED: diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -32,9 +32,9 @@ dispatch_opt(self, op) def try_boolinvers(self, op, targs): - op = self.get_pure_result(targs) - if op is not None: - value = self.getvalue(op.result) + oldop = self.get_pure_result(targs) + if oldop is not None: + value = self.getvalue(oldop.result) if value.is_constant(): if value.box.same_constant(CONST_1): self.make_constant(op.result, CONST_0) @@ -57,11 +57,11 @@ oldopnum = op.boolreflex # FIXME: add INT_ADD, INT_MUL if oldopnum != -1: - top = ResOperation(oldopnum, [arg0, arg1], None) + top = ResOperation(oldopnum, [arg1, arg0], None) oldop = self.get_pure_result(top) if oldop is not None: - self.optimizer.make_equal_to(op.result, self.getvalue(oldop), - True) + self.optimizer.make_equal_to(op.result, + self.getvalue(oldop.result), True) return True if op.boolreflex == -1: diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -4762,6 +4762,7 @@ self.optimize_loop(ops, expected) def test_complains_getfieldpure_setfield(self): + py.test.skip("disabled for now") from rpython.jit.metainterp.optimizeopt.heap import BogusPureField ops = """ [p3] diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -550,7 +550,7 @@ '_CANRAISE_LAST', # ----- end of can_raise operations ----- '_OVF_FIRST', # ----- start of is_ovf operations ----- - 'INT_ADD_OVF/2', + 'INT_ADD_OVF/2', # note that the orded has to match INT_ADD order 'INT_SUB_OVF/2', 'INT_MUL_OVF/2', '_OVF_LAST', # ----- end of is_ovf operations ----- From noreply at buildbot.pypy.org Fri Mar 6 14:10:44 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 6 Mar 2015 14:10:44 +0100 (CET) Subject: [pypy-commit] pypy.org extradoc: update the values Message-ID: <20150306131044.71AD31C00FC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r575:15d893b04f4c Date: 2015-03-06 14:11 +0100 http://bitbucket.org/pypy/pypy.org/changeset/15d893b04f4c/ Log: update the values diff --git a/don1.html b/don1.html --- a/don1.html +++ b/don1.html @@ -9,13 +9,13 @@ - $58764 of $105000 (56.0%) + $58873 of $105000 (56.1%)
diff --git a/don3.html b/don3.html --- a/don3.html +++ b/don3.html @@ -15,7 +15,7 @@ - $51316 of $60000 (85.5%) + $51321 of $60000 (85.5%)
diff --git a/don4.html b/don4.html --- a/don4.html +++ b/don4.html @@ -9,7 +9,7 @@ @@ -17,7 +17,7 @@ 2nd call: - $22357 of $80000 (27.9%) + $22463 of $80000 (28.1%)
From noreply at buildbot.pypy.org Fri Mar 6 16:02:48 2015 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 6 Mar 2015 16:02:48 +0100 (CET) Subject: [pypy-commit] pypy default: fix egg version Message-ID: <20150306150248.028E21C02AD@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r76252:c4c364117fe8 Date: 2015-03-06 09:27 +0200 http://bitbucket.org/pypy/pypy/changeset/c4c364117fe8/ Log: fix egg version diff --git a/lib_pypy/cffi.egg-info b/lib_pypy/cffi.egg-info --- a/lib_pypy/cffi.egg-info +++ b/lib_pypy/cffi.egg-info @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: cffi -Version: 0.8.6+ +Version: 0.9.0 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski From noreply at buildbot.pypy.org Fri Mar 6 16:02:52 2015 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 6 Mar 2015 16:02:52 +0100 (CET) Subject: [pypy-commit] pypy object-dtype2: reflect attributes of object, start to write gc test Message-ID: <20150306150252.948601C02AD@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: object-dtype2 Changeset: r76254:a4a38e2aec66 Date: 2015-03-06 17:03 +0200 http://bitbucket.org/pypy/pypy/changeset/a4a38e2aec66/ Log: reflect attributes of object, start to write gc test diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py --- a/pypy/module/micronumpy/boxes.py +++ b/pypy/module/micronumpy/boxes.py @@ -616,6 +616,8 @@ def convert_to(self, space, dtype): return self # XXX + def descr__getattr__(self, space, w_key): + return space.getattr(self.w_obj, w_key) W_GenericBox.typedef = TypeDef("numpy.generic", __new__ = interp2app(W_GenericBox.descr__new__.im_func), @@ -865,3 +867,9 @@ __new__ = interp2app(W_UnicodeBox.descr__new__unicode_box.im_func), __len__ = interp2app(W_UnicodeBox.descr_len), ) + +W_ObjectBox.typedef = TypeDef("numpy.object", W_ObjectBox.typedef, + __new__ = interp2app(W_ObjectBox.descr__new__.im_func), + __getattr__ = interp2app(W_ObjectBox.descr__getattr__), +) + diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -473,7 +473,12 @@ pass for o in [object, O]: print np.dtype(o).byteorder - assert np.dtype(o).str == '|O8' + if self.ptr_size == 4: + assert np.dtype(o).str == '|O4' + elif self.ptr_size == 8: + assert np.dtype(o).str == '|O8' + else: + assert False,'self._ptr_size unknown' class AppTestTypes(BaseAppTestDtypes): def test_abstract_types(self): @@ -1350,9 +1355,11 @@ from numpy import array import sys class Polynomial(object): - pass + def whatami(self): + return 'an object' a = array(Polynomial()) assert a.shape == () + assert a.sum().whatami() == 'an object' def test_uninitialized_object_array_is_filled_by_None(self): import numpy as np @@ -1369,3 +1376,16 @@ res = a + b assert res[0] == "foobar" + + def test_keep_object_alive(self): + import numpy as np + import gc + class O(object): + def whatami(self): + return 'an object' + fiveOs = [O()] * 5 + a = np.array(fiveOs, dtype=object) + del fiveOs + gc.collect() + gc.collect() + assert a[2].whatami() == 'an object' From noreply at buildbot.pypy.org Fri Mar 6 16:02:51 2015 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 6 Mar 2015 16:02:51 +0100 (CET) Subject: [pypy-commit] pypy object-dtype2: merge default into branch Message-ID: <20150306150251.60C761C02AD@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: object-dtype2 Changeset: r76253:cb72282b1ea6 Date: 2015-03-06 09:28 +0200 http://bitbucket.org/pypy/pypy/changeset/cb72282b1ea6/ Log: merge default into branch diff too long, truncating to 2000 out of 3202 lines diff --git a/lib_pypy/cffi.egg-info b/lib_pypy/cffi.egg-info --- a/lib_pypy/cffi.egg-info +++ b/lib_pypy/cffi.egg-info @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: cffi -Version: 0.8.6+ +Version: 0.9.0 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "0.8.6+" -__version_info__ = (0, 8, 6, "plus") +__version__ = "0.9.0" +__version_info__ = (0, 9, 0) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/pyrepl/readline.py b/lib_pypy/pyrepl/readline.py --- a/lib_pypy/pyrepl/readline.py +++ b/lib_pypy/pyrepl/readline.py @@ -73,7 +73,6 @@ assume_immutable_completions = False use_brackets = False sort_in_column = True - tab_insert_spaces_if_stem_is_empty = False def error(self, msg="none"): pass # don't show error messages by default @@ -87,7 +86,7 @@ return ''.join(b[p+1:self.pos]) def get_completions(self, stem): - if len(stem) == 0 and self.tab_insert_spaces_if_stem_is_empty: + if len(stem) == 0 and self.more_lines is not None: b = self.buffer p = self.pos while p > 0 and b[p - 1] != '\n': @@ -141,12 +140,16 @@ def collect_keymap(self): return super(ReadlineAlikeReader, self).collect_keymap() + ( - (r'\n', 'maybe-accept'),) + (r'\n', 'maybe-accept'), + (r'\', 'backspace-dedent'), + ) def __init__(self, console): super(ReadlineAlikeReader, self).__init__(console) self.commands['maybe_accept'] = maybe_accept self.commands['maybe-accept'] = maybe_accept + self.commands['backspace_dedent'] = backspace_dedent + self.commands['backspace-dedent'] = backspace_dedent def after_command(self, cmd): super(ReadlineAlikeReader, self).after_command(cmd) @@ -164,6 +167,28 @@ if self.pos > len(self.buffer): self.pos = len(self.buffer) +def _get_this_line_indent(buffer, pos): + indent = 0 + while pos > 0 and buffer[pos - 1] in " \t": + indent += 1 + pos -= 1 + if pos > 0 and buffer[pos - 1] == "\n": + return indent + return 0 + +def _get_previous_line_indent(buffer, pos): + prevlinestart = pos + while prevlinestart > 0 and buffer[prevlinestart - 1] != "\n": + prevlinestart -= 1 + prevlinetext = prevlinestart + while prevlinetext < pos and buffer[prevlinetext] in " \t": + prevlinetext += 1 + if prevlinetext == pos: + indent = None + else: + indent = prevlinetext - prevlinestart + return prevlinestart, indent + class maybe_accept(commands.Command): def do(self): r = self.reader @@ -172,13 +197,39 @@ # if there are already several lines and the cursor # is not on the last one, always insert a new \n. text = r.get_unicode() - if "\n" in r.buffer[r.pos:]: + if ("\n" in r.buffer[r.pos:] or + (r.more_lines is not None and r.more_lines(text))): + # + # auto-indent the next line like the previous line + prevlinestart, indent = _get_previous_line_indent(r.buffer, r.pos) r.insert("\n") - elif r.more_lines is not None and r.more_lines(text): - r.insert("\n") + if indent: + for i in range(prevlinestart, prevlinestart + indent): + r.insert(r.buffer[i]) else: self.finish = 1 +class backspace_dedent(commands.Command): + def do(self): + r = self.reader + b = r.buffer + if r.pos > 0: + repeat = 1 + if b[r.pos - 1] != "\n": + indent = _get_this_line_indent(b, r.pos) + if indent > 0: + ls = r.pos - indent + while ls > 0: + ls, pi = _get_previous_line_indent(b, ls - 1) + if pi is not None and pi < indent: + repeat = indent - pi + break + r.pos -= repeat + del b[r.pos:r.pos + repeat] + r.dirty = 1 + else: + self.reader.error("can't backspace at start") + # ____________________________________________________________ class _ReadlineWrapper(object): @@ -212,15 +263,14 @@ boolean value is true. """ reader = self.get_reader() - saved = reader.more_lines, reader.tab_insert_spaces_if_stem_is_empty + saved = reader.more_lines try: reader.more_lines = more_lines reader.ps1 = reader.ps2 = ps1 reader.ps3 = reader.ps4 = ps2 - reader.tab_insert_spaces_if_stem_is_empty = True return reader.readline(returns_unicode=returns_unicode) finally: - reader.more_lines, reader.tab_insert_spaces_if_stem_is_empty = saved + reader.more_lines = saved def parse_and_bind(self, string): pass # XXX we don't support parsing GNU-readline-style init files diff --git a/pypy/doc/jit-hooks.rst b/pypy/doc/jit-hooks.rst --- a/pypy/doc/jit-hooks.rst +++ b/pypy/doc/jit-hooks.rst @@ -39,3 +39,30 @@ Reason is a string, the meaning of other arguments is the same as attributes on JitLoopInfo object +.. function:: enable_debug() + + Start recording debugging counters for ``get_stats_snapshot`` + +.. function:: disable_debug() + + Stop recording debugging counters for ``get_stats_snapshot`` + +.. function:: get_stats_snapshot() + + Get the jit status in the specific moment in time. Note that this + is eager - the attribute access is not lazy, if you need new stats + you need to call this function again. You might want to call + ``enable_debug`` to get more information. It returns an instance + of ``JitInfoSnapshot`` + +.. class:: JitInfoSnapshot + + A class describing current snapshot. Usable attributes: + + * ``counters`` - internal JIT integer counters + + * ``counter_times`` - internal JIT float counters, notably time spent + TRACING and in the JIT BACKEND + + * ``loop_run_times`` - counters for number of times loops are run, only + works when ``enable_debug`` is called. diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -517,6 +517,10 @@ elif not sys.stdout.isatty(): set_fully_buffered_io() + if we_are_translated(): + import __pypy__ + __pypy__.save_module_content_for_future_reload(sys) + mainmodule = type(sys)('__main__') sys.modules['__main__'] = mainmodule diff --git a/pypy/interpreter/mixedmodule.py b/pypy/interpreter/mixedmodule.py --- a/pypy/interpreter/mixedmodule.py +++ b/pypy/interpreter/mixedmodule.py @@ -55,7 +55,10 @@ if self.w_initialdict is None: Module.init(self, space) if not self.lazy and self.w_initialdict is None: - self.w_initialdict = space.call_method(self.w_dict, 'items') + self.save_module_content_for_future_reload() + + def save_module_content_for_future_reload(self): + self.w_initialdict = self.space.call_method(self.w_dict, 'items') def get_applevel_name(cls): @@ -119,7 +122,7 @@ w_value = self.get(name) space.setitem(self.w_dict, space.new_interned_str(name), w_value) self.lazy = False - self.w_initialdict = space.call_method(self.w_dict, 'items') + self.save_module_content_for_future_reload() return self.w_dict def _cleanup_(self): diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -82,6 +82,8 @@ 'strategy' : 'interp_magic.strategy', # dict,set,list 'set_debug' : 'interp_magic.set_debug', 'locals_to_fast' : 'interp_magic.locals_to_fast', + 'save_module_content_for_future_reload': + 'interp_magic.save_module_content_for_future_reload', } if sys.platform == 'win32': interpleveldefs['get_console_cp'] = 'interp_magic.get_console_cp' diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -1,6 +1,7 @@ from pypy.interpreter.error import OperationError, wrap_oserror from pypy.interpreter.gateway import unwrap_spec from pypy.interpreter.pyframe import PyFrame +from pypy.interpreter.mixedmodule import MixedModule from rpython.rlib.objectmodel import we_are_translated from pypy.objspace.std.dictmultiobject import W_DictMultiObject from pypy.objspace.std.listobject import W_ListObject @@ -130,3 +131,7 @@ def locals_to_fast(space, w_frame): assert isinstance(w_frame, PyFrame) w_frame.locals2fast() + + at unwrap_spec(w_module=MixedModule) +def save_module_content_for_future_reload(space, w_module): + w_module.save_module_content_for_future_reload() diff --git a/pypy/module/__pypy__/test/test_magic.py b/pypy/module/__pypy__/test/test_magic.py new file mode 100644 --- /dev/null +++ b/pypy/module/__pypy__/test/test_magic.py @@ -0,0 +1,15 @@ + +class AppTestMagic: + spaceconfig = dict(usemodules=['__pypy__']) + + def test_save_module_content_for_future_reload(self): + import sys, __pypy__ + d = sys.dont_write_bytecode + sys.dont_write_bytecode = "hello world" + __pypy__.save_module_content_for_future_reload(sys) + sys.dont_write_bytecode = d + reload(sys) + assert sys.dont_write_bytecode == "hello world" + # + sys.dont_write_bytecode = d + __pypy__.save_module_content_for_future_reload(sys) diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -8,7 +8,7 @@ appleveldefs = { } interpleveldefs = { - '__version__': 'space.wrap("0.8.6+")', + '__version__': 'space.wrap("0.9.0")', 'load_library': 'libraryobj.load_library', diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py --- a/pypy/module/_cffi_backend/cbuffer.py +++ b/pypy/module/_cffi_backend/cbuffer.py @@ -81,4 +81,5 @@ if size < 0: raise oefmt(space.w_TypeError, "don't know the size pointed to by '%s'", ctype.name) - return space.wrap(MiniBuffer(LLBuffer(w_cdata._cdata, size), w_cdata)) + ptr = w_cdata.unsafe_escaping_ptr() # w_cdata kept alive by MiniBuffer() + return space.wrap(MiniBuffer(LLBuffer(ptr, size), w_cdata)) diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py --- a/pypy/module/_cffi_backend/ccallback.py +++ b/pypy/module/_cffi_backend/ccallback.py @@ -48,9 +48,12 @@ raise oefmt(space.w_NotImplementedError, "%s: callback with unsupported argument or " "return type or with '...'", self.getfunctype().name) - res = clibffi.c_ffi_prep_closure(self.get_closure(), cif_descr.cif, - invoke_callback, - rffi.cast(rffi.VOIDP, self.unique_id)) + with self as ptr: + closure_ptr = rffi.cast(clibffi.FFI_CLOSUREP, ptr) + unique_id = rffi.cast(rffi.VOIDP, self.unique_id) + res = clibffi.c_ffi_prep_closure(closure_ptr, cif_descr.cif, + invoke_callback, + unique_id) if rffi.cast(lltype.Signed, res) != clibffi.FFI_OK: raise OperationError(space.w_SystemError, space.wrap("libffi failed to build this callback")) @@ -62,12 +65,9 @@ from pypy.module.thread.os_thread import setup_threads setup_threads(space) - def get_closure(self): - return rffi.cast(clibffi.FFI_CLOSUREP, self._cdata) - #@rgc.must_be_light_finalizer def __del__(self): - clibffi.closureHeap.free(self.get_closure()) + clibffi.closureHeap.free(rffi.cast(clibffi.FFI_CLOSUREP, self._ptr)) if self.ll_error: lltype.free(self.ll_error, flavor='raw') @@ -106,7 +106,7 @@ fresult = self.getfunctype().ctitem if fresult.size > 0: misc._raw_memcopy(self.ll_error, ll_res, fresult.size) - keepalive_until_here(self) + keepalive_until_here(self) # to keep self.ll_error alive global_callback_mapping = rweakref.RWeakValueDictionary(int, W_CDataCallback) diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py --- a/pypy/module/_cffi_backend/cdataobj.py +++ b/pypy/module/_cffi_backend/cdataobj.py @@ -14,21 +14,37 @@ class W_CData(W_Root): - _attrs_ = ['space', '_cdata', 'ctype', '_lifeline_'] - _immutable_fields_ = ['_cdata', 'ctype'] - _cdata = lltype.nullptr(rffi.CCHARP.TO) + _attrs_ = ['space', '_ptr', 'ctype', '_lifeline_'] + _immutable_fields_ = ['_ptr', 'ctype'] + _ptr = lltype.nullptr(rffi.CCHARP.TO) - def __init__(self, space, cdata, ctype): + def __init__(self, space, ptr, ctype): from pypy.module._cffi_backend import ctypeobj - assert lltype.typeOf(cdata) == rffi.CCHARP + assert lltype.typeOf(ptr) == rffi.CCHARP assert isinstance(ctype, ctypeobj.W_CType) self.space = space - self._cdata = cdata # don't forget keepalive_until_here! + self._ptr = ptr # don't access directly! use "with cdata as ptr:" self.ctype = ctype + def __enter__(self): + """Use 'with cdata as ptr:' to access the raw memory. It will + stay alive at least until the end of the 'with' block. + """ + return self._ptr + + def __exit__(self, *args): + keepalive_until_here(self) + + def unsafe_escaping_ptr(self): + """Generally unsafe: escape the pointer to raw memory. + If 'self' is a subclass that frees the pointer in a destructor, + it may be freed under your feet at any time. + """ + return self._ptr + def _repr_extra(self): - extra = self.ctype.extra_repr(self._cdata) - keepalive_until_here(self) + with self as ptr: + extra = self.ctype.extra_repr(ptr) return extra def _repr_extra_owning(self): @@ -54,11 +70,13 @@ self.ctype.name, extra1, extra2)) def nonzero(self): - return self.space.wrap(bool(self._cdata)) + with self as ptr: + nonzero = bool(ptr) + return self.space.wrap(nonzero) def int(self, space): - w_result = self.ctype.cast_to_int(self._cdata) - keepalive_until_here(self) + with self as ptr: + w_result = self.ctype.cast_to_int(ptr) return w_result def long(self, space): @@ -69,8 +87,8 @@ return w_result def float(self): - w_result = self.ctype.float(self._cdata) - keepalive_until_here(self) + with self as ptr: + w_result = self.ctype.float(ptr) return w_result def len(self): @@ -88,20 +106,19 @@ def _cmp(self, w_other): from pypy.module._cffi_backend.ctypeprim import W_CTypePrimitive space = self.space - cdata1 = self._cdata - if isinstance(w_other, W_CData): - cdata2 = w_other._cdata - else: + if not isinstance(w_other, W_CData): return space.w_NotImplemented - if requires_ordering: - if (isinstance(self.ctype, W_CTypePrimitive) or - isinstance(w_other.ctype, W_CTypePrimitive)): - raise OperationError(space.w_TypeError, - space.wrap("cannot do comparison on a primitive cdata")) - cdata1 = rffi.cast(lltype.Unsigned, cdata1) - cdata2 = rffi.cast(lltype.Unsigned, cdata2) - return space.newbool(op(cdata1, cdata2)) + with self as ptr1, w_other as ptr2: + if requires_ordering: + if (isinstance(self.ctype, W_CTypePrimitive) or + isinstance(w_other.ctype, W_CTypePrimitive)): + raise OperationError(space.w_TypeError, space.wrap( + "cannot do comparison on a primitive cdata")) + ptr1 = rffi.cast(lltype.Unsigned, ptr1) + ptr2 = rffi.cast(lltype.Unsigned, ptr2) + result = op(ptr1, ptr2) + return space.newbool(result) # return func_with_new_name(_cmp, name) @@ -113,7 +130,8 @@ ge = _make_comparison('ge') def hash(self): - h = rffi.cast(lltype.Signed, self._cdata) + ptr = self.unsafe_escaping_ptr() + h = rffi.cast(lltype.Signed, ptr) # To hash pointers in dictionaries. Assumes that h shows some # alignment (to 4, 8, maybe 16 bytes), so we use the following # formula to avoid the trailing bits being always 0. @@ -128,26 +146,27 @@ i = space.getindex_w(w_index, space.w_IndexError) ctype = self.ctype._check_subscript_index(self, i) w_o = self._do_getitem(ctype, i) - keepalive_until_here(self) return w_o def _do_getitem(self, ctype, i): ctitem = ctype.ctitem - return ctitem.convert_to_object( - rffi.ptradd(self._cdata, i * ctitem.size)) + with self as ptr: + return ctitem.convert_to_object( + rffi.ptradd(ptr, i * ctitem.size)) def setitem(self, w_index, w_value): space = self.space if space.isinstance_w(w_index, space.w_slice): - self._do_setslice(w_index, w_value) + with self as ptr: + self._do_setslice(w_index, w_value, ptr) else: i = space.getindex_w(w_index, space.w_IndexError) ctype = self.ctype._check_subscript_index(self, i) ctitem = ctype.ctitem - ctitem.convert_from_object( - rffi.ptradd(self._cdata, i * ctitem.size), - w_value) - keepalive_until_here(self) + with self as ptr: + ctitem.convert_from_object( + rffi.ptradd(ptr, i * ctitem.size), + w_value) def _do_getslicearg(self, w_slice): from pypy.module._cffi_backend.ctypeptr import W_CTypePointer @@ -188,14 +207,15 @@ ctarray = newtype.new_array_type(space, ctptr, space.w_None) ctptr.cache_array_type = ctarray # - p = rffi.ptradd(self._cdata, start * ctarray.ctitem.size) - return W_CDataSliced(space, p, ctarray, length) + ptr = self.unsafe_escaping_ptr() + ptr = rffi.ptradd(ptr, start * ctarray.ctitem.size) + return W_CDataSliced(space, ptr, ctarray, length) - def _do_setslice(self, w_slice, w_value): + def _do_setslice(self, w_slice, w_value, ptr): ctptr, start, length = self._do_getslicearg(w_slice) ctitem = ctptr.ctitem ctitemsize = ctitem.size - cdata = rffi.ptradd(self._cdata, start * ctitemsize) + target = rffi.ptradd(ptr, start * ctitemsize) # if isinstance(w_value, W_CData): from pypy.module._cffi_backend import ctypearray @@ -204,9 +224,8 @@ ctv.ctitem is ctitem and w_value.get_array_length() == length): # fast path: copying from exactly the correct type - s = w_value._cdata - rffi.c_memcpy(cdata, s, ctitemsize * length) - keepalive_until_here(w_value) + with w_value as source: + rffi.c_memcpy(target, source, ctitemsize * length) return # # A fast path for [0:N] = "somestring". @@ -221,7 +240,7 @@ raise oefmt(space.w_ValueError, "need a string of length %d, got %d", length, len(value)) - copy_string_to_raw(llstr(value), cdata, 0, length) + copy_string_to_raw(llstr(value), target, 0, length) return # w_iter = space.iter(w_value) @@ -233,8 +252,8 @@ raise raise oefmt(space.w_ValueError, "need %d values to unpack, got %d", length, i) - ctitem.convert_from_object(cdata, w_item) - cdata = rffi.ptradd(cdata, ctitemsize) + ctitem.convert_from_object(target, w_item) + target = rffi.ptradd(target, ctitemsize) try: space.next(w_iter) except OperationError, e: @@ -247,7 +266,8 @@ def _add_or_sub(self, w_other, sign): space = self.space i = sign * space.getindex_w(w_other, space.w_OverflowError) - return self.ctype.add(self._cdata, i) + ptr = self.unsafe_escaping_ptr() + return self.ctype.add(ptr, i) def add(self, w_other): return self._add_or_sub(w_other, +1) @@ -268,9 +288,11 @@ self.ctype.name, ct.name) # itemsize = ct.ctitem.size - if itemsize <= 0: itemsize = 1 - diff = (rffi.cast(lltype.Signed, self._cdata) - - rffi.cast(lltype.Signed, w_other._cdata)) // itemsize + if itemsize <= 0: + itemsize = 1 + with self as ptr1, w_other as ptr2: + diff = (rffi.cast(lltype.Signed, ptr1) - + rffi.cast(lltype.Signed, ptr2)) // itemsize return space.wrap(diff) # return self._add_or_sub(w_other, -1) @@ -279,17 +301,19 @@ return self.ctype.getcfield(self.space.str_w(w_attr)) def getattr(self, w_attr): - w_res = self.getcfield(w_attr).read(self._cdata) - keepalive_until_here(self) + cfield = self.getcfield(w_attr) + with self as ptr: + w_res = cfield.read(ptr) return w_res def setattr(self, w_attr, w_value): - self.getcfield(w_attr).write(self._cdata, w_value) - keepalive_until_here(self) + cfield = self.getcfield(w_attr) + with self as ptr: + cfield.write(ptr, w_value) def call(self, args_w): - w_result = self.ctype.call(self._cdata, args_w) - keepalive_until_here(self) + with self as ptr: + w_result = self.ctype.call(ptr, args_w) return w_result def iter(self): @@ -311,21 +335,21 @@ @specialize.argtype(1) def write_raw_signed_data(self, source): - misc.write_raw_signed_data(self._cdata, source, self.ctype.size) - keepalive_until_here(self) + with self as ptr: + misc.write_raw_signed_data(ptr, source, self.ctype.size) @specialize.argtype(1) def write_raw_unsigned_data(self, source): - misc.write_raw_unsigned_data(self._cdata, source, self.ctype.size) - keepalive_until_here(self) + with self as ptr: + misc.write_raw_unsigned_data(ptr, source, self.ctype.size) def write_raw_float_data(self, source): - misc.write_raw_float_data(self._cdata, source, self.ctype.size) - keepalive_until_here(self) + with self as ptr: + misc.write_raw_float_data(ptr, source, self.ctype.size) def convert_to_object(self): - w_obj = self.ctype.convert_to_object(self._cdata) - keepalive_until_here(self) + with self as ptr: + w_obj = self.ctype.convert_to_object(ptr) return w_obj def get_array_length(self): @@ -353,7 +377,7 @@ @rgc.must_be_light_finalizer def __del__(self): - lltype.free(self._cdata, flavor='raw') + lltype.free(self._ptr, flavor='raw') class W_CDataNewOwning(W_CDataMem): diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py --- a/pypy/module/_cffi_backend/ctypearray.py +++ b/pypy/module/_cffi_backend/ctypearray.py @@ -8,7 +8,6 @@ from pypy.interpreter.typedef import TypeDef from rpython.rtyper.lltypesystem import rffi -from rpython.rlib.objectmodel import keepalive_until_here from rpython.rlib.rarithmetic import ovfcheck from pypy.module._cffi_backend import cdataobj @@ -49,8 +48,8 @@ cdata = cdataobj.W_CDataNewOwning(space, datasize, self) # if not space.is_w(w_init, space.w_None): - self.convert_from_object(cdata._cdata, w_init) - keepalive_until_here(cdata) + with cdata as ptr: + self.convert_from_object(ptr, w_init) return cdata def _check_subscript_index(self, w_cdata, i): @@ -119,8 +118,8 @@ self.ctitem = ctitem self.cdata = cdata length = cdata.get_array_length() - self._next = cdata._cdata - self._stop = rffi.ptradd(cdata._cdata, length * ctitem.size) + self._next = cdata.unsafe_escaping_ptr() + self._stop = rffi.ptradd(self._next, length * ctitem.size) def iter_w(self): return self.space.wrap(self) diff --git a/pypy/module/_cffi_backend/ctypeenum.py b/pypy/module/_cffi_backend/ctypeenum.py --- a/pypy/module/_cffi_backend/ctypeenum.py +++ b/pypy/module/_cffi_backend/ctypeenum.py @@ -2,8 +2,6 @@ Enums. """ -from rpython.rlib.objectmodel import keepalive_until_here - from pypy.module._cffi_backend import misc from pypy.module._cffi_backend.ctypeprim import (W_CTypePrimitiveSigned, W_CTypePrimitiveUnsigned) @@ -47,8 +45,8 @@ return '%s: %s' % (value, s) def string(self, cdataobj, maxlen): - value = self._get_value(cdataobj._cdata) - keepalive_until_here(cdataobj) + with cdataobj as ptr: + value = self._get_value(ptr) try: s = self.enumvalues2erators[value] except KeyError: diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py --- a/pypy/module/_cffi_backend/ctypeobj.py +++ b/pypy/module/_cffi_backend/ctypeobj.py @@ -177,8 +177,8 @@ raise oefmt(space.w_AttributeError, "cdata '%s' has no attribute '%s'", self.name, attr) - def copy_and_convert_to_object(self, cdata): - return self.convert_to_object(cdata) + def copy_and_convert_to_object(self, source): + return self.convert_to_object(source) # __________ app-level attributes __________ def dir(self): diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py --- a/pypy/module/_cffi_backend/ctypeprim.py +++ b/pypy/module/_cffi_backend/ctypeprim.py @@ -5,7 +5,6 @@ import sys from rpython.rlib.rarithmetic import r_uint, r_ulonglong, intmask -from rpython.rlib.objectmodel import keepalive_until_here from rpython.rlib import jit from rpython.rtyper.lltypesystem import lltype, rffi @@ -53,7 +52,8 @@ space = self.space if (isinstance(w_ob, cdataobj.W_CData) and isinstance(w_ob.ctype, ctypeptr.W_CTypePtrOrArray)): - value = rffi.cast(lltype.Signed, w_ob._cdata) + ptr = w_ob.unsafe_escaping_ptr() + value = rffi.cast(lltype.Signed, ptr) value = self._cast_result(value) elif space.isinstance_w(w_ob, space.w_str): value = self.cast_str(w_ob) @@ -81,8 +81,8 @@ def string(self, cdataobj, maxlen): if self.size == 1: - s = cdataobj._cdata[0] - keepalive_until_here(cdataobj) + with cdataobj as ptr: + s = ptr[0] return self.space.wrap(s) return W_CType.string(self, cdataobj, maxlen) @@ -116,7 +116,8 @@ return s[0] if (isinstance(w_ob, cdataobj.W_CData) and isinstance(w_ob.ctype, W_CTypePrimitiveChar)): - return w_ob._cdata[0] + with w_ob as ptr: + return ptr[0] raise self._convert_error("string of length 1", w_ob) def convert_from_object(self, cdata, w_ob): @@ -137,8 +138,8 @@ return self.space.wrap(s) def string(self, cdataobj, maxlen): - w_res = self.convert_to_object(cdataobj._cdata) - keepalive_until_here(cdataobj) + with cdataobj as ptr: + w_res = self.convert_to_object(ptr) return w_res def _convert_to_unichar(self, w_ob): @@ -149,7 +150,8 @@ return s[0] if (isinstance(w_ob, cdataobj.W_CData) and isinstance(w_ob.ctype, W_CTypePrimitiveUniChar)): - return rffi.cast(rffi.CWCHARP, w_ob._cdata)[0] + with w_ob as ptr: + return rffi.cast(rffi.CWCHARP, ptr)[0] raise self._convert_error("unicode string of length 1", w_ob) def convert_from_object(self, cdata, w_ob): @@ -219,13 +221,15 @@ if self.size == rffi.sizeof(rffi.LONG): from rpython.rlib.rrawarray import populate_list_from_raw_array res = [] - buf = rffi.cast(rffi.LONGP, w_cdata._cdata) length = w_cdata.get_array_length() - populate_list_from_raw_array(res, buf, length) + with w_cdata as ptr: + buf = rffi.cast(rffi.LONGP, ptr) + populate_list_from_raw_array(res, buf, length) return res elif self.value_smaller_than_long: res = [0] * w_cdata.get_array_length() - misc.unpack_list_from_raw_array(res, w_cdata._cdata, self.size) + with w_cdata as ptr: + misc.unpack_list_from_raw_array(res, ptr, self.size) return res return None @@ -308,8 +312,8 @@ def unpack_list_of_int_items(self, w_cdata): if self.value_fits_long: res = [0] * w_cdata.get_array_length() - misc.unpack_unsigned_list_from_raw_array(res, w_cdata._cdata, - self.size) + with w_cdata as ptr: + misc.unpack_unsigned_list_from_raw_array(res, ptr, self.size) return res return None @@ -363,8 +367,8 @@ if not isinstance(self, W_CTypePrimitiveLongDouble): w_cdata.write_raw_float_data(value) else: - self._to_longdouble_and_write(value, w_cdata._cdata) - keepalive_until_here(w_cdata) + with w_cdata as ptr: + self._to_longdouble_and_write(value, ptr) return w_cdata def cast_to_int(self, cdata): @@ -387,13 +391,15 @@ if self.size == rffi.sizeof(rffi.DOUBLE): from rpython.rlib.rrawarray import populate_list_from_raw_array res = [] - buf = rffi.cast(rffi.DOUBLEP, w_cdata._cdata) length = w_cdata.get_array_length() - populate_list_from_raw_array(res, buf, length) + with w_cdata as ptr: + buf = rffi.cast(rffi.DOUBLEP, ptr) + populate_list_from_raw_array(res, buf, length) return res elif self.size == rffi.sizeof(rffi.FLOAT): res = [0.0] * w_cdata.get_array_length() - misc.unpack_cfloat_list_from_raw_array(res, w_cdata._cdata) + with w_cdata as ptr: + misc.unpack_cfloat_list_from_raw_array(res, ptr) return res return None @@ -423,8 +429,8 @@ def cast(self, w_ob): if (isinstance(w_ob, cdataobj.W_CData) and isinstance(w_ob.ctype, W_CTypePrimitiveLongDouble)): - w_cdata = self.convert_to_object(w_ob._cdata) - keepalive_until_here(w_ob) + with w_ob as ptr: + w_cdata = self.convert_to_object(ptr) return w_cdata else: return W_CTypePrimitiveFloat.cast(self, w_ob) @@ -451,16 +457,16 @@ def convert_to_object(self, cdata): w_cdata = cdataobj.W_CDataMem(self.space, self.size, self) - self._copy_longdouble(cdata, w_cdata._cdata) - keepalive_until_here(w_cdata) + with w_cdata as ptr: + self._copy_longdouble(cdata, ptr) return w_cdata def convert_from_object(self, cdata, w_ob): space = self.space if (isinstance(w_ob, cdataobj.W_CData) and isinstance(w_ob.ctype, W_CTypePrimitiveLongDouble)): - self._copy_longdouble(w_ob._cdata, cdata) - keepalive_until_here(w_ob) + with w_ob as ptr: + self._copy_longdouble(ptr, cdata) else: value = space.float_w(space.float(w_ob)) self._to_longdouble_and_write(value, cdata) diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py --- a/pypy/module/_cffi_backend/ctypeptr.py +++ b/pypy/module/_cffi_backend/ctypeptr.py @@ -3,7 +3,6 @@ """ from rpython.rlib import rposix -from rpython.rlib.objectmodel import keepalive_until_here from rpython.rlib.rarithmetic import ovfcheck from rpython.rtyper.annlowlevel import llstr, llunicode from rpython.rtyper.lltypesystem import lltype, rffi @@ -49,7 +48,7 @@ space = self.space if (isinstance(w_ob, cdataobj.W_CData) and isinstance(w_ob.ctype, W_CTypePtrOrArray)): - value = w_ob._cdata + value = w_ob.unsafe_escaping_ptr() else: value = misc.as_unsigned_long(space, w_ob, strict=False) value = rffi.cast(rffi.CCHARP, value) @@ -108,34 +107,33 @@ def string(self, cdataobj, maxlen): space = self.space if isinstance(self.ctitem, ctypeprim.W_CTypePrimitive): - cdata = cdataobj._cdata - if not cdata: - raise oefmt(space.w_RuntimeError, "cannot use string() on %s", - space.str_w(cdataobj.repr())) - # - from pypy.module._cffi_backend import ctypearray - length = maxlen - if length < 0 and isinstance(self, ctypearray.W_CTypeArray): - length = cdataobj.get_array_length() - # - # pointer to a primitive type of size 1: builds and returns a str - if self.ctitem.size == rffi.sizeof(lltype.Char): - if length < 0: - s = rffi.charp2str(cdata) - else: - s = rffi.charp2strn(cdata, length) - keepalive_until_here(cdataobj) - return space.wrap(s) - # - # pointer to a wchar_t: builds and returns a unicode - if self.is_unichar_ptr_or_array(): - cdata = rffi.cast(rffi.CWCHARP, cdata) - if length < 0: - u = rffi.wcharp2unicode(cdata) - else: - u = rffi.wcharp2unicoden(cdata, length) - keepalive_until_here(cdataobj) - return space.wrap(u) + with cdataobj as ptr: + if not ptr: + raise oefmt(space.w_RuntimeError, + "cannot use string() on %s", + space.str_w(cdataobj.repr())) + # + from pypy.module._cffi_backend import ctypearray + length = maxlen + if length < 0 and isinstance(self, ctypearray.W_CTypeArray): + length = cdataobj.get_array_length() + # + # pointer to a primitive type of size 1: builds and returns a str + if self.ctitem.size == rffi.sizeof(lltype.Char): + if length < 0: + s = rffi.charp2str(ptr) + else: + s = rffi.charp2strn(ptr, length) + return space.wrap(s) + # + # pointer to a wchar_t: builds and returns a unicode + if self.is_unichar_ptr_or_array(): + cdata = rffi.cast(rffi.CWCHARP, ptr) + if length < 0: + u = rffi.wcharp2unicode(cdata) + else: + u = rffi.wcharp2unicoden(cdata, length) + return space.wrap(u) # return W_CType.string(self, cdataobj, maxlen) @@ -162,7 +160,7 @@ if not (self.can_cast_anything or other.can_cast_anything): raise self._convert_error("compatible pointer", w_ob) - rffi.cast(rffi.CCHARPP, cdata)[0] = w_ob._cdata + rffi.cast(rffi.CCHARPP, cdata)[0] = w_ob.unsafe_escaping_ptr() def _alignof(self): from pypy.module._cffi_backend import newtype @@ -206,8 +204,8 @@ lltype.nullptr(rffi.CCHARP.TO), w_init, datasize) # cdatastruct = cdataobj.W_CDataNewOwning(space, datasize, ctitem) - cdata = cdataobj.W_CDataPtrToStructOrUnion(space, - cdatastruct._cdata, + ptr = cdatastruct.unsafe_escaping_ptr() + cdata = cdataobj.W_CDataPtrToStructOrUnion(space, ptr, self, cdatastruct) else: if self.is_char_or_unichar_ptr_or_array(): @@ -215,8 +213,8 @@ cdata = cdataobj.W_CDataNewOwning(space, datasize, self) # if not space.is_w(w_init, space.w_None): - ctitem.convert_from_object(cdata._cdata, w_init) - keepalive_until_here(cdata) + with cdata as ptr: + ctitem.convert_from_object(ptr, w_init) return cdata def _check_subscript_index(self, w_cdata, i): @@ -332,8 +330,9 @@ ctype2 = cdata.ctype if (isinstance(ctype2, W_CTypeStructOrUnion) or isinstance(ctype2, W_CTypePtrOrArray)): - ptrdata = rffi.ptradd(cdata._cdata, offset) - return cdataobj.W_CData(space, ptrdata, self) + ptr = cdata.unsafe_escaping_ptr() + ptr = rffi.ptradd(ptr, offset) + return cdataobj.W_CData(space, ptr, self) else: raise OperationError(space.w_TypeError, space.wrap("expected a cdata struct/union/array/pointer" diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py --- a/pypy/module/_cffi_backend/ctypestruct.py +++ b/pypy/module/_cffi_backend/ctypestruct.py @@ -7,7 +7,6 @@ from pypy.interpreter.typedef import TypeDef, interp_attrproperty from rpython.rlib import jit -from rpython.rlib.objectmodel import keepalive_until_here from rpython.rlib.rarithmetic import r_uint, r_ulonglong, r_longlong, intmask from rpython.rlib.rarithmetic import ovfcheck from rpython.rtyper.lltypesystem import lltype, rffi @@ -57,12 +56,12 @@ self.check_complete() return cdataobj.W_CData(space, cdata, self) - def copy_and_convert_to_object(self, cdata): + def copy_and_convert_to_object(self, source): space = self.space self.check_complete() ob = cdataobj.W_CDataNewOwning(space, self.size, self) - misc._raw_memcopy(cdata, ob._cdata, self.size) - keepalive_until_here(ob) + with ob as target: + misc._raw_memcopy(source, target, self.size) return ob def typeoffsetof_field(self, fieldname, following): @@ -80,8 +79,8 @@ def _copy_from_same(self, cdata, w_ob): if isinstance(w_ob, cdataobj.W_CData): if w_ob.ctype is self and self.size >= 0: - misc._raw_memcopy(w_ob._cdata, cdata, self.size) - keepalive_until_here(w_ob) + with w_ob as ptr: + misc._raw_memcopy(ptr, cdata, self.size) return True return False diff --git a/pypy/module/_cffi_backend/ctypevoid.py b/pypy/module/_cffi_backend/ctypevoid.py --- a/pypy/module/_cffi_backend/ctypevoid.py +++ b/pypy/module/_cffi_backend/ctypevoid.py @@ -13,5 +13,5 @@ def __init__(self, space): W_CType.__init__(self, space, -1, "void", len("void")) - def copy_and_convert_to_object(self, cdata): + def copy_and_convert_to_object(self, source): return self.space.w_None diff --git a/pypy/module/_cffi_backend/handle.py b/pypy/module/_cffi_backend/handle.py --- a/pypy/module/_cffi_backend/handle.py +++ b/pypy/module/_cffi_backend/handle.py @@ -34,8 +34,9 @@ raise oefmt(space.w_TypeError, "expected a 'cdata' object with a 'void *' out of " "new_handle(), got '%s'", ctype.name) - index = rffi.cast(lltype.Signed, w_cdata._cdata) - original_cdataobj = get(space).fetch_handle(index - 1) + with w_cdata as ptr: + index = rffi.cast(lltype.Signed, ptr) + original_cdataobj = get(space).fetch_handle(index - 1) # if isinstance(original_cdataobj, cdataobj.W_CDataHandle): return original_cdataobj.w_keepalive diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py --- a/pypy/module/_cffi_backend/misc.py +++ b/pypy/module/_cffi_backend/misc.py @@ -3,7 +3,7 @@ from pypy.interpreter.error import OperationError from rpython.rlib import jit -from rpython.rlib.objectmodel import keepalive_until_here, specialize +from rpython.rlib.objectmodel import specialize from rpython.rlib.rarithmetic import r_uint, r_ulonglong from rpython.rlib.unroll import unrolling_iterable from rpython.rtyper.lltypesystem import lltype, llmemory, rffi @@ -272,11 +272,11 @@ from pypy.module._cffi_backend.ctypeprim import W_CTypePrimitiveLongDouble is_cdata = isinstance(w_ob, W_CData) if is_cdata and isinstance(w_ob.ctype, W_CTypePrimitiveFloat): - if isinstance(w_ob.ctype, W_CTypePrimitiveLongDouble): - result = is_nonnull_longdouble(w_ob._cdata) - else: - result = is_nonnull_float(w_ob._cdata, w_ob.ctype.size) - keepalive_until_here(w_ob) + with w_ob as ptr: + if isinstance(w_ob.ctype, W_CTypePrimitiveLongDouble): + result = is_nonnull_longdouble(ptr) + else: + result = is_nonnull_float(ptr, w_ob.ctype.size) return result # if not is_cdata and space.lookup(w_ob, '__float__') is not None: diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -3249,4 +3249,4 @@ def test_version(): # this test is here mostly for PyPy - assert __version__ == "0.8.6+" + assert __version__ == "0.9.0" diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -1,4 +1,4 @@ -from rpython.rlib import jit +from rpython.rlib import jit, rgc from rpython.rlib.buffer import Buffer from rpython.rlib.objectmodel import keepalive_until_here from rpython.rlib.rarithmetic import ovfcheck, widen @@ -698,11 +698,9 @@ self.space.wrap(msg)) return result + @rgc.must_be_light_finalizer def __del__(self): - # note that we don't call clear_all_weakrefs here because - # an array with freed buffer is ok to see - it's just empty with 0 - # length - self.setlen(0) + lltype.free(self.buffer, flavor='raw') def setlen(self, size, zero=False, overallocate=True): if size > 0: diff --git a/pypy/module/cppyy/capi/loadable_capi.py b/pypy/module/cppyy/capi/loadable_capi.py --- a/pypy/module/cppyy/capi/loadable_capi.py +++ b/pypy/module/cppyy/capi/loadable_capi.py @@ -259,7 +259,8 @@ if not objectmodel.we_are_translated(): leakfinder.remember_free(c_call.ctype.cif_descr._obj0) state.capi_calls[name] = c_call - return c_call.ctype.rcall(c_call._cdata, args) + with c_call as ptr: + return c_call.ctype.rcall(ptr, args) def _cdata_to_cobject(space, w_cdata): return rffi.cast(C_OBJECT, space.uint_w(w_cdata)) @@ -271,8 +272,9 @@ return rffi.cast(rffi.LONG, space.int_w(w_cdata)) def _cdata_to_ptr(space, w_cdata): # TODO: this is both a hack and dreadfully slow - return rffi.cast(rffi.VOIDP, - space.interp_w(cdataobj.W_CData, w_cdata, can_be_None=False)._cdata) + w_cdata = space.interp_w(cdataobj.W_CData, w_cdata, can_be_None=False) + ptr = w_cdata.unsafe_escaping_ptr() + return rffi.cast(rffi.VOIDP, ptr) def c_load_dictionary(name): return libffi.CDLL(name) diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py --- a/pypy/module/micronumpy/base.py +++ b/pypy/module/micronumpy/base.py @@ -48,7 +48,7 @@ @staticmethod def from_shape_and_storage(space, shape, storage, dtype, storage_bytes=-1, order='C', owning=False, w_subtype=None, - w_base=None, writable=True, strides=None): + w_base=None, writable=True, strides=None, start=0): from pypy.module.micronumpy import concrete from pypy.module.micronumpy.strides import (calc_strides, calc_backstrides) @@ -77,8 +77,9 @@ raise OperationError(space.w_ValueError, space.wrap("Cannot have owning=True when specifying a buffer")) if writable: - impl = concrete.ConcreteArrayWithBase(shape, dtype, order, strides, - backstrides, storage, w_base) + impl = concrete.ConcreteArrayWithBase(shape, dtype, order, + strides, backstrides, storage, w_base, + start=start) else: impl = concrete.ConcreteNonWritableArrayWithBase(shape, dtype, order, strides, backstrides, @@ -130,6 +131,9 @@ def get_order(self): return self.implementation.order + def get_start(self): + return self.implementation.start + def ndims(self): return len(self.get_shape()) ndims._always_inline_ = True diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py --- a/pypy/module/micronumpy/concrete.py +++ b/pypy/module/micronumpy/concrete.py @@ -12,6 +12,7 @@ from pypy.module.micronumpy.strides import (Chunk, Chunks, NewAxisChunk, RecordChunk, calc_strides, calc_new_strides, shape_agreement, calculate_broadcast_strides, calc_backstrides) +from rpython.rlib.objectmodel import keepalive_until_here class BaseConcreteArray(object): @@ -312,12 +313,15 @@ l_w = [w_res.descr_getitem(space, space.wrap(d)) for d in range(nd)] return space.newtuple(l_w) - def get_storage_as_int(self, space): - return rffi.cast(lltype.Signed, self.storage) + self.start - - def get_storage(self): + ##def get_storage(self): + ## return self.storage + ## use a safer context manager + def __enter__(self): return self.storage + def __exit__(self, typ, value, traceback): + keepalive_until_here(self) + def get_buffer(self, space, readonly): return ArrayBuffer(self, readonly) @@ -331,7 +335,7 @@ class ConcreteArrayNotOwning(BaseConcreteArray): - def __init__(self, shape, dtype, order, strides, backstrides, storage): + def __init__(self, shape, dtype, order, strides, backstrides, storage, start=0): make_sure_not_resized(shape) make_sure_not_resized(strides) make_sure_not_resized(backstrides) @@ -342,6 +346,7 @@ self.strides = strides self.backstrides = backstrides self.storage = storage + self.start = start def fill(self, space, box): self.dtype.itemtype.fill(self.storage, self.dtype.elsize, @@ -350,7 +355,7 @@ def set_shape(self, space, orig_array, new_shape): strides, backstrides = calc_strides(new_shape, self.dtype, self.order) - return SliceArray(0, strides, backstrides, new_shape, self, + return SliceArray(self.start, strides, backstrides, new_shape, self, orig_array) def set_dtype(self, space, dtype): @@ -384,9 +389,10 @@ class ConcreteArrayWithBase(ConcreteArrayNotOwning): - def __init__(self, shape, dtype, order, strides, backstrides, storage, orig_base): + def __init__(self, shape, dtype, order, strides, backstrides, storage, + orig_base, start=0): ConcreteArrayNotOwning.__init__(self, shape, dtype, order, - strides, backstrides, storage) + strides, backstrides, storage, start) self.orig_base = orig_base def base(self): diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py --- a/pypy/module/micronumpy/ctors.py +++ b/pypy/module/micronumpy/ctors.py @@ -99,10 +99,12 @@ for i in range(w_object.get_size()): elems_w[i] = w_object.implementation.getitem(i * elsize) else: - sz = support.product(w_object.get_shape()) * dtype.elsize - return W_NDimArray.from_shape_and_storage(space, - w_object.get_shape(),w_object.implementation.storage, - dtype, storage_bytes=sz, w_base=w_object) + imp = w_object.implementation + with imp as storage: + sz = support.product(w_object.get_shape()) * dtype.elsize + return W_NDimArray.from_shape_and_storage(space, + w_object.get_shape(), storage, dtype, storage_bytes=sz, + w_base=w_object, start=imp.start) else: # not an array shape, elems_w = strides.find_shape_and_elems(space, w_object, dtype) diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py --- a/pypy/module/micronumpy/loop.py +++ b/pypy/module/micronumpy/loop.py @@ -604,14 +604,15 @@ iter, state = arr.create_iter() w_res_str = W_NDimArray.from_shape(space, [1], arr.get_dtype(), order='C') itemsize = arr.get_dtype().elsize - res_str_casted = rffi.cast(rffi.CArrayPtr(lltype.Char), - w_res_str.implementation.get_storage_as_int(space)) - while not iter.done(state): - w_res_str.implementation.setitem(0, iter.getitem(state)) - for i in range(itemsize): - builder.append(res_str_casted[i]) - state = iter.next(state) - return builder.build() + with w_res_str.implementation as storage: + res_str_casted = rffi.cast(rffi.CArrayPtr(lltype.Char), + support.get_storage_as_int(storage)) + while not iter.done(state): + w_res_str.implementation.setitem(0, iter.getitem(state)) + for i in range(itemsize): + builder.append(res_str_casted[i]) + state = iter.next(state) + return builder.build() getitem_int_driver = jit.JitDriver(name = 'numpy_getitem_int', greens = ['shapelen', 'indexlen', diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -532,20 +532,25 @@ self.get_dtype(), storage_bytes=sz, w_base=self) def descr_array_iface(self, space): - addr = self.implementation.get_storage_as_int(space) - # will explode if it can't - w_d = space.newdict() - space.setitem_str(w_d, 'data', - space.newtuple([space.wrap(addr), space.w_False])) - space.setitem_str(w_d, 'shape', self.descr_get_shape(space)) - space.setitem_str(w_d, 'typestr', self.get_dtype().descr_get_str(space)) - if self.implementation.order == 'C': - # Array is contiguous, no strides in the interface. - strides = space.w_None - else: - strides = self.descr_get_strides(space) - space.setitem_str(w_d, 'strides', strides) - return w_d + ''' + Note: arr.__array__.data[0] is a pointer so arr must be kept alive + while it is in use + ''' + with self.implementation as storage: + addr = support.get_storage_as_int(storage, self.get_start()) + # will explode if it can't + w_d = space.newdict() + space.setitem_str(w_d, 'data', + space.newtuple([space.wrap(addr), space.w_False])) + space.setitem_str(w_d, 'shape', self.descr_get_shape(space)) + space.setitem_str(w_d, 'typestr', self.get_dtype().descr_get_str(space)) + if self.implementation.order == 'C': + # Array is contiguous, no strides in the interface. + strides = space.w_None + else: + strides = self.descr_get_strides(space) + space.setitem_str(w_d, 'strides', strides) + return w_d w_pypy_data = None @@ -1165,7 +1170,8 @@ builder.append(box.raw_str()) state = iter.next(state) else: - builder.append_charpsize(self.implementation.get_storage(), + with self.implementation as storage: + builder.append_charpsize(storage, self.implementation.get_storage_size()) state = space.newtuple([ diff --git a/pypy/module/micronumpy/selection.py b/pypy/module/micronumpy/selection.py --- a/pypy/module/micronumpy/selection.py +++ b/pypy/module/micronumpy/selection.py @@ -33,14 +33,14 @@ self.values = values self.indexes = indexes - def getitem(self, item): + def getitem(self, idx): if count < 2: - v = raw_storage_getitem(TP, self.values, item * self.stride_size + v = raw_storage_getitem(TP, self.values, idx * self.stride_size + self.start) else: v = [] for i in range(count): - _v = raw_storage_getitem(TP, self.values, item * self.stride_size + _v = raw_storage_getitem(TP, self.values, idx * self.stride_size + self.start + step * i) v.append(_v) if comp_type == 'int': @@ -52,7 +52,7 @@ else: raise NotImplementedError('cannot reach') return (v, raw_storage_getitem(lltype.Signed, self.indexes, - item * self.index_stride_size + + idx * self.index_stride_size + self.index_start)) def setitem(self, idx, item): @@ -134,37 +134,37 @@ # create array of indexes dtype = descriptor.get_dtype_cache(space).w_longdtype index_arr = W_NDimArray.from_shape(space, arr.get_shape(), dtype) - storage = index_arr.implementation.get_storage() - if len(arr.get_shape()) == 1: - for i in range(arr.get_size()): - raw_storage_setitem(storage, i * INT_SIZE, i) - r = Repr(INT_SIZE, itemsize, arr.get_size(), arr.get_storage(), - storage, 0, arr.start) - ArgSort(r).sort() - else: - shape = arr.get_shape() - if axis < 0: - axis = len(shape) + axis - if axis < 0 or axis >= len(shape): - raise oefmt(space.w_IndexError, "Wrong axis %d", axis) - arr_iter = AllButAxisIter(arr, axis) - arr_state = arr_iter.reset() - index_impl = index_arr.implementation - index_iter = AllButAxisIter(index_impl, axis) - index_state = index_iter.reset() - stride_size = arr.strides[axis] - index_stride_size = index_impl.strides[axis] - axis_size = arr.shape[axis] - while not arr_iter.done(arr_state): - for i in range(axis_size): - raw_storage_setitem(storage, i * index_stride_size + - index_state.offset, i) - r = Repr(index_stride_size, stride_size, axis_size, - arr.get_storage(), storage, index_state.offset, arr_state.offset) + with index_arr.implementation as storage, arr as arr_storage: + if len(arr.get_shape()) == 1: + for i in range(arr.get_size()): + raw_storage_setitem(storage, i * INT_SIZE, i) + r = Repr(INT_SIZE, itemsize, arr.get_size(), arr_storage, + storage, 0, arr.start) ArgSort(r).sort() - arr_state = arr_iter.next(arr_state) - index_state = index_iter.next(index_state) - return index_arr + else: + shape = arr.get_shape() + if axis < 0: + axis = len(shape) + axis + if axis < 0 or axis >= len(shape): + raise oefmt(space.w_IndexError, "Wrong axis %d", axis) + arr_iter = AllButAxisIter(arr, axis) + arr_state = arr_iter.reset() + index_impl = index_arr.implementation + index_iter = AllButAxisIter(index_impl, axis) + index_state = index_iter.reset() + stride_size = arr.strides[axis] + index_stride_size = index_impl.strides[axis] + axis_size = arr.shape[axis] + while not arr_iter.done(arr_state): + for i in range(axis_size): + raw_storage_setitem(storage, i * index_stride_size + + index_state.offset, i) + r = Repr(index_stride_size, stride_size, axis_size, + arr_storage, storage, index_state.offset, arr_state.offset) + ArgSort(r).sort() + arr_state = arr_iter.next(arr_state) + index_state = index_iter.next(index_state) + return index_arr return argsort @@ -282,25 +282,25 @@ axis = -1 else: axis = space.int_w(w_axis) - # create array of indexes - if len(arr.get_shape()) == 1: - r = Repr(itemsize, arr.get_size(), arr.get_storage(), - arr.start) - ArgSort(r).sort() - else: - shape = arr.get_shape() - if axis < 0: - axis = len(shape) + axis - if axis < 0 or axis >= len(shape): - raise oefmt(space.w_IndexError, "Wrong axis %d", axis) - arr_iter = AllButAxisIter(arr, axis) - arr_state = arr_iter.reset() - stride_size = arr.strides[axis] - axis_size = arr.shape[axis] - while not arr_iter.done(arr_state): - r = Repr(stride_size, axis_size, arr.get_storage(), arr_state.offset) + with arr as storage: + if len(arr.get_shape()) == 1: + r = Repr(itemsize, arr.get_size(), storage, + arr.start) ArgSort(r).sort() - arr_state = arr_iter.next(arr_state) + else: + shape = arr.get_shape() + if axis < 0: + axis = len(shape) + axis + if axis < 0 or axis >= len(shape): + raise oefmt(space.w_IndexError, "Wrong axis %d", axis) + arr_iter = AllButAxisIter(arr, axis) + arr_state = arr_iter.reset() + stride_size = arr.strides[axis] + axis_size = arr.shape[axis] + while not arr_iter.done(arr_state): + r = Repr(stride_size, axis_size, storage, arr_state.offset) + ArgSort(r).sort() + arr_state = arr_iter.next(arr_state) return sort diff --git a/pypy/module/micronumpy/support.py b/pypy/module/micronumpy/support.py --- a/pypy/module/micronumpy/support.py +++ b/pypy/module/micronumpy/support.py @@ -1,6 +1,7 @@ from pypy.interpreter.error import OperationError, oefmt from rpython.rlib import jit from rpython.rlib.rarithmetic import ovfcheck +from rpython.rtyper.lltypesystem import rffi, lltype def issequence_w(space, w_obj): @@ -147,3 +148,7 @@ if cur_core_dim == 0: ufunc.core_enabled = False return 0 # for historical reasons, any failures will raise + +def get_storage_as_int(storage, start=0): + return rffi.cast(lltype.Signed, storage) + start + diff --git a/pypy/module/micronumpy/test/test_subtype.py b/pypy/module/micronumpy/test/test_subtype.py --- a/pypy/module/micronumpy/test/test_subtype.py +++ b/pypy/module/micronumpy/test/test_subtype.py @@ -2,7 +2,7 @@ class AppTestSupport(BaseNumpyAppTest): - spaceconfig = dict(usemodules=["micronumpy", "struct", "binascii"]) + spaceconfig = dict(usemodules=["micronumpy", "struct", "binascii", "mmap"]) def setup_class(cls): BaseNumpyAppTest.setup_class.im_func(cls) @@ -476,3 +476,120 @@ a = self.SubType(array([[1, 2], [3, 4]])) b = array(a, subok=False) assert type(b) is ndarray + + def test_numpypy_mmap(self): + # issue #21 on pypy/numpy + from numpy import array, ndarray, arange, dtype as dtypedescr + import mmap + import os.path + from tempfile import mkdtemp + import os.path as path + valid_filemodes = ["r", "c", "r+", "w+"] + writeable_filemodes = ["r+", "w+"] + mode_equivalents = { + "readonly":"r", + "copyonwrite":"c", + "readwrite":"r+", + "write":"w+" + } + + class memmap(ndarray): + def __new__(subtype, filename, dtype='uint8', mode='r+', offset=0, shape=None, order='C'): + # Import here to minimize 'import numpy' overhead + try: + mode = mode_equivalents[mode] + except KeyError: + if mode not in valid_filemodes: + raise ValueError("mode must be one of %s" % + (valid_filemodes + list(mode_equivalents.keys()))) + + if hasattr(filename, 'read'): + fid = filename + own_file = False + else: + fid = open(filename, (mode == 'c' and 'r' or mode)+'b') + own_file = True + + if (mode == 'w+') and shape is None: + raise ValueError("shape must be given") + + fid.seek(0, 2) + flen = fid.tell() + descr = dtypedescr(dtype) + _dbytes = descr.itemsize + + if shape is None: + bytes = flen - offset + if (bytes % _dbytes): + fid.close() + raise ValueError("Size of available data is not a " + "multiple of the data-type size.") + size = bytes // _dbytes + shape = (size,) + else: + if not isinstance(shape, tuple): + shape = (shape,) + size = 1 + for k in shape: + size *= k + + bytes = long(offset + size*_dbytes) + + if mode == 'w+' or (mode == 'r+' and flen < bytes): + fid.seek(bytes - 1, 0) + fid.write('\0') + fid.flush() + + if mode == 'c': + acc = mmap.ACCESS_COPY + elif mode == 'r': + acc = mmap.ACCESS_READ + else: + acc = mmap.ACCESS_WRITE + + start = offset - offset % mmap.ALLOCATIONGRANULARITY + bytes -= start + offset -= start + mm = mmap.mmap(fid.fileno(), bytes, access=acc, offset=start) + + self = ndarray.__new__(subtype, shape, dtype=descr, buffer=mm, + offset=offset, order=order) + self._mmap = mm + self.offset = offset + self.mode = mode + + if isinstance(filename, basestring): + self.filename = os.path.abspath(filename) + # py3 returns int for TemporaryFile().name + elif (hasattr(filename, "name") and + isinstance(filename.name, basestring)): + self.filename = os.path.abspath(filename.name) + # same as memmap copies (e.g. memmap + 1) + else: + self.filename = None + + if own_file: + fid.close() + + return self + + def flush(self): + if self.base is not None and hasattr(self.base, 'flush'): + self.base.flush() + + def asarray(obj, itemsize=None, order=None): + return array(obj, itemsize, copy=False, order=order) + + filename = path.join(mkdtemp(), 'newfile.dat') + data = arange(10*10*36).reshape(10, 10, 36) + fp = memmap(filename, dtype='float32', mode='w+', shape=data.shape) + vals = [ 242, 507, 255, 505, 315, 316, 308, 506, + 309, 255, 211, 505, 315, 316, 308, 506, + 309, 255, 255, 711, 194, 232, 711, 711, + 709, 710, 709, 710, 882, 897, 711, 245, + 711, 711, 168, 245] + fp[:] = data + fp[5:6][:,4] = vals + a = asarray(fp[5:6][:,4]) + assert (a == vals).all() + diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py --- a/pypy/module/micronumpy/test/test_zjit.py +++ b/pypy/module/micronumpy/test/test_zjit.py @@ -238,15 +238,15 @@ 'getfield_gc': 5, 'getfield_gc_pure': 51, 'guard_class': 3, - 'guard_false': 13, + 'guard_false': 12, 'guard_nonnull': 11, 'guard_nonnull_class': 3, 'guard_not_invalidated': 2, 'guard_true': 10, - 'guard_value': 5, + 'guard_value': 6, 'int_add': 13, 'int_ge': 4, - 'int_is_true': 4, + 'int_is_true': 3, 'int_is_zero': 4, 'int_le': 2, 'int_lt': 3, @@ -616,15 +616,15 @@ 'getfield_gc': 6, 'getfield_gc_pure': 63, 'guard_class': 5, - 'guard_false': 20, + 'guard_false': 19, 'guard_nonnull': 6, 'guard_nonnull_class': 1, 'guard_not_invalidated': 3, 'guard_true': 16, - 'guard_value': 2, + 'guard_value': 3, 'int_add': 24, 'int_ge': 4, - 'int_is_true': 6, + 'int_is_true': 5, 'int_is_zero': 4, 'int_le': 5, 'int_lt': 7, diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -185,13 +185,16 @@ raw_storage_setitem_unaligned(storage, i + offset, value) def read(self, arr, i, offset, dtype=None): - return self.box(self._read(arr.storage, i, offset)) + with arr as storage: + return self.box(self._read(storage, i, offset)) def read_bool(self, arr, i, offset): - return bool(self.for_computation(self._read(arr.storage, i, offset))) + with arr as storage: + return bool(self.for_computation(self._read(storage, i, offset))) def store(self, arr, i, offset, box): - self._write(arr.storage, i, offset, self.unbox(box)) + with arr as storage: + self._write(storage, i, offset, self.unbox(box)) def fill(self, storage, width, box, start, stop, offset): value = self.unbox(box) @@ -1085,8 +1088,9 @@ return bool(real) or bool(imag) def read_bool(self, arr, i, offset): - v = self.for_computation(self._read(arr.storage, i, offset)) - return bool(v[0]) or bool(v[1]) + with arr as storage: + v = self.for_computation(self._read(storage, i, offset)) + return bool(v[0]) or bool(v[1]) def get_element_size(self): return 2 * rffi.sizeof(self.T) @@ -1137,8 +1141,9 @@ return real, imag def read(self, arr, i, offset, dtype=None): - real, imag = self._read(arr.storage, i, offset) - return self.box_complex(real, imag) + with arr as storage: + real, imag = self._read(storage, i, offset) + return self.box_complex(real, imag) def _write(self, storage, i, offset, value): real, imag = value @@ -1149,7 +1154,8 @@ raw_storage_setitem_unaligned(storage, i + offset + rffi.sizeof(self.T), imag) def store(self, arr, i, offset, box): - self._write(arr.storage, i, offset, self.unbox(box)) + with arr as storage: + self._write(storage, i, offset, self.unbox(box)) def fill(self, storage, width, box, start, stop, offset): value = self.unbox(box) @@ -1722,13 +1728,14 @@ assert isinstance(item, boxes.W_FlexibleBox) i = item.ofs end = i + item.dtype.elsize - while i < end: - assert isinstance(item.arr.storage[i], str) - if item.arr.storage[i] == '\x00': - break - builder.append(item.arr.storage[i]) - i += 1 - return builder.build() + with item.arr as storage: + while i < end: + assert isinstance(storage[i], str) + if storage[i] == '\x00': + break + builder.append(storage[i]) + i += 1 + return builder.build() def str_unary_op(func): specialize.argtype(1)(func) @@ -1758,23 +1765,26 @@ w_item = space.wrap('') arg = space.str_w(space.str(w_item)) arr = VoidBoxStorage(dtype.elsize, dtype) - j = min(len(arg), dtype.elsize) - for i in range(j): - arr.storage[i] = arg[i] - for j in range(j, dtype.elsize): - arr.storage[j] = '\x00' - return boxes.W_StringBox(arr, 0, arr.dtype) + with arr as storage: + j = min(len(arg), dtype.elsize) + for i in range(j): + storage[i] = arg[i] + for j in range(j, dtype.elsize): + storage[j] = '\x00' + return boxes.W_StringBox(arr, 0, arr.dtype) def store(self, arr, i, offset, box): assert isinstance(box, boxes.W_StringBox) size = min(arr.dtype.elsize - offset, box.arr.size - box.ofs) - return self._store(arr.storage, i, offset, box, size) + with arr as storage: + return self._store(storage, i, offset, box, size) @jit.unroll_safe def _store(self, storage, i, offset, box, size): assert isinstance(box, boxes.W_StringBox) - for k in range(size): - storage[k + offset + i] = box.arr.storage[k + box.ofs] + with box.arr as box_storage: + for k in range(size): + storage[k + offset + i] = box_storage[k + box.ofs] def read(self, arr, i, offset, dtype=None): if dtype is None: @@ -1891,8 +1901,9 @@ assert i == 0 assert isinstance(box, boxes.W_VoidBox) assert box.dtype is box.arr.dtype - for k in range(box.arr.dtype.elsize): - arr.storage[k + ofs] = box.arr.storage[k + box.ofs] + with arr as arr_storage, box.arr as box_storage: + for k in range(box.arr.dtype.elsize): + arr_storage[k + ofs] = box_storage[k + box.ofs] def readarray(self, arr, i, offset, dtype=None): from pypy.module.micronumpy.base import W_NDimArray @@ -1982,12 +1993,14 @@ def store(self, arr, i, ofs, box): assert isinstance(box, boxes.W_VoidBox) - self._store(arr.storage, i, ofs, box, box.dtype.elsize) + with arr as storage: + self._store(storage, i, ofs, box, box.dtype.elsize) @jit.unroll_safe def _store(self, storage, i, ofs, box, size): - for k in range(size): - storage[k + i + ofs] = box.arr.storage[k + box.ofs] + with box.arr as box_storage: + for k in range(size): + storage[k + i + ofs] = box_storage[k + box.ofs] def fill(self, storage, width, box, start, stop, offset): assert isinstance(box, boxes.W_VoidBox) @@ -2033,9 +2046,10 @@ s1 = v1.dtype.elsize s2 = v2.dtype.elsize assert s1 == s2 - for i in range(s1): - if v1.arr.storage[v1.ofs + i] != v2.arr.storage[v2.ofs + i]: - return False + with v1.arr as v1_storage, v2.arr as v2_storage: + for i in range(s1): + if v1_storage[v1.ofs + i] != v2_storage[v2.ofs + i]: + return False return True def ne(self, v1, v2): diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py --- a/pypy/module/micronumpy/ufuncs.py +++ b/pypy/module/micronumpy/ufuncs.py @@ -13,11 +13,12 @@ from pypy.module.micronumpy.ctors import numpify from pypy.module.micronumpy.nditer import W_NDIter, coalesce_iter from pypy.module.micronumpy.strides import shape_agreement -from pypy.module.micronumpy.support import _parse_signature, product +from pypy.module.micronumpy.support import _parse_signature, product, get_storage_as_int from rpython.rlib.rawstorage import (raw_storage_setitem, free_raw_storage, alloc_raw_storage) from rpython.rtyper.lltypesystem import rffi, lltype from rpython.rlib.rarithmetic import LONG_BIT, _get_bitsize +from rpython.rlib.objectmodel import keepalive_until_here def done_if_true(dtype, val): @@ -98,7 +99,9 @@ if out is not None and not isinstance(out, W_NDimArray): raise OperationError(space.w_TypeError, space.wrap( 'output must be an array')) - return self.call(space, args_w, sig, casting, extobj) + retval = self.call(space, args_w, sig, casting, extobj) + keepalive_until_here(args_w) + return retval def descr_accumulate(self, space, w_obj, w_axis=None, w_dtype=None, w_out=None): if space.is_none(w_axis): @@ -806,11 +809,12 @@ assert isinstance(curarg, W_NDimArray) if len(arg_shapes[i]) != curarg.ndims(): # reshape + sz = product(curarg.get_shape()) * curarg.get_dtype().elsize - inargs[i] = W_NDimArray.from_shape_and_storage( - space, arg_shapes[i], curarg.implementation.storage, - curarg.get_dtype(), storage_bytes=sz, w_base=curarg) - pass + with curarg.implementation as storage: + inargs[i] = W_NDimArray.from_shape_and_storage( + space, arg_shapes[i], storage, + curarg.get_dtype(), storage_bytes=sz, w_base=curarg) need_to_cast.append(curarg.get_dtype() != dtypes[i]) for i in range(len(outargs)): j = self.nin + i @@ -821,9 +825,10 @@ elif len(arg_shapes[i]) != curarg.ndims(): # reshape sz = product(curarg.get_shape()) * curarg.get_dtype().elsize - outargs[i] = W_NDimArray.from_shape_and_storage( - space, arg_shapes[i], curarg.implementation.storage, - curarg.get_dtype(), storage_bytes=sz, w_base=curarg) + with curarg.implementation as storage: + outargs[i] = W_NDimArray.from_shape_and_storage( + space, arg_shapes[i], storage, + curarg.get_dtype(), storage_bytes=sz, w_base=curarg) curarg = outargs[i] assert isinstance(curarg, W_NDimArray) need_to_cast.append(curarg.get_dtype() != dtypes[j]) @@ -1414,8 +1419,9 @@ raise OperationError(space.w_NotImplementedError, space.wrap("cannot mix ndarray and %r (arg %d) in call to ufunc" % ( arg_i, i))) - raw_storage_setitem(dataps, CCHARP_SIZE * i, - rffi.cast(rffi.CCHARP, arg_i.implementation.get_storage_as_int(space))) + with arg_i.implementation as storage: + addr = get_storage_as_int(storage, arg_i.get_start()) + raw_storage_setitem(dataps, CCHARP_SIZE * i, rffi.cast(rffi.CCHARP, addr)) #This assumes we iterate over the whole array (it should be a view...) raw_storage_setitem(self.dims, LONG_SIZE * i, rffi.cast(rffi.LONG, arg_i.get_size())) raw_storage_setitem(self.steps, LONG_SIZE * i, rffi.cast(rffi.LONG, arg_i.get_dtype().elsize)) @@ -1423,8 +1429,9 @@ for i in range(len(args_w)): arg_i = args_w[i] assert isinstance(arg_i, W_NDimArray) - raw_storage_setitem(dataps, CCHARP_SIZE * i, - rffi.cast(rffi.CCHARP, arg_i.implementation.get_storage_as_int(space))) + with arg_i.implementation as storage: + addr = get_storage_as_int(storage, arg_i.get_start()) + raw_storage_setitem(dataps, CCHARP_SIZE * i, rffi.cast(rffi.CCHARP, addr)) try: arg1 = rffi.cast(rffi.CArrayPtr(rffi.CCHARP), dataps) arg2 = rffi.cast(npy_intpp, self.dims) @@ -1432,6 +1439,7 @@ self.func(arg1, arg2, arg3, self.data) finally: free_raw_storage(dataps, track_allocation=False) + keepalive_until_here(args_w) def set_dims_and_steps(self, space, dims, steps): if not isinstance(dims, list) or not isinstance(steps, list): diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py --- a/pypy/module/mmap/interp_mmap.py +++ b/pypy/module/mmap/interp_mmap.py @@ -22,6 +22,10 @@ self.check_valid() return MMapBuffer(self.space, self.mmap, True) + def writebuf_w(self, space): + self.check_writeable() + return MMapBuffer(self.space, self.mmap, False) + def close(self): self.mmap.close() diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py --- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py +++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py @@ -341,7 +341,7 @@ raw_store(i119, 0, i160, descr=) raw_store(i119, 2, i160, descr=) raw_store(i119, 4, i160, descr=) - setfield_gc(p167, i119, descr=) + setfield_gc(p167, i119, descr=) i123 = arraylen_gc(p67, descr=) jump(..., descr=...) """) diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py --- a/pypy/module/sys/test/test_sysmodule.py +++ b/pypy/module/sys/test/test_sysmodule.py @@ -521,9 +521,11 @@ def test_reload_doesnt_override_sys_executable(self): import sys - sys.executable = 'from_test_sysmodule' + if not hasattr(sys, 'executable'): # if not translated + sys.executable = 'from_test_sysmodule' + previous = sys.executable reload(sys) - assert sys.executable == 'from_test_sysmodule' + assert sys.executable == previous def test_settrace(self): import sys diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py --- a/rpython/annotator/annrpython.py +++ b/rpython/annotator/annrpython.py @@ -6,8 +6,8 @@ from rpython.tool.pairtype import pair from rpython.tool.error import (format_blocked_annotation_error, gather_error, source_lines) -from rpython.flowspace.model import (Variable, Constant, FunctionGraph, - c_last_exception, checkgraph) +from rpython.flowspace.model import ( + Variable, Constant, FunctionGraph, checkgraph) from rpython.translator import simplify, transform from rpython.annotator import model as annmodel, signature from rpython.annotator.argument import simple_args @@ -407,8 +407,7 @@ self.bookkeeper.leave() except BlockedInference as e: - if (e.op is block.operations[-1] and - block.exitswitch == c_last_exception): + if e.op is block.raising_op: # this is the case where the last operation of the block will # always raise an exception which is immediately caught by # an exception handler. We then only follow the exceptional @@ -450,8 +449,8 @@ # filter out those exceptions which cannot # occour for this specific, typed operation. - if block.exitswitch == c_last_exception: - op = block.operations[-1] + if block.canraise: + op = block.raising_op From noreply at buildbot.pypy.org Fri Mar 6 16:22:34 2015 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 6 Mar 2015 16:22:34 +0100 (CET) Subject: [pypy-commit] pypy optresult: merge recent-ops Message-ID: <20150306152234.A152E1C02AD@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult Changeset: r76255:456f069a2ff4 Date: 2015-03-06 17:22 +0200 http://bitbucket.org/pypy/pypy/changeset/456f069a2ff4/ Log: merge recent-ops diff --git a/lib_pypy/pyrepl/readline.py b/lib_pypy/pyrepl/readline.py --- a/lib_pypy/pyrepl/readline.py +++ b/lib_pypy/pyrepl/readline.py @@ -73,7 +73,6 @@ assume_immutable_completions = False use_brackets = False sort_in_column = True - tab_insert_spaces_if_stem_is_empty = False def error(self, msg="none"): pass # don't show error messages by default @@ -87,7 +86,7 @@ return ''.join(b[p+1:self.pos]) def get_completions(self, stem): - if len(stem) == 0 and self.tab_insert_spaces_if_stem_is_empty: + if len(stem) == 0 and self.more_lines is not None: b = self.buffer p = self.pos while p > 0 and b[p - 1] != '\n': @@ -141,12 +140,16 @@ def collect_keymap(self): return super(ReadlineAlikeReader, self).collect_keymap() + ( - (r'\n', 'maybe-accept'),) + (r'\n', 'maybe-accept'), + (r'\', 'backspace-dedent'), + ) def __init__(self, console): super(ReadlineAlikeReader, self).__init__(console) self.commands['maybe_accept'] = maybe_accept self.commands['maybe-accept'] = maybe_accept + self.commands['backspace_dedent'] = backspace_dedent + self.commands['backspace-dedent'] = backspace_dedent def after_command(self, cmd): super(ReadlineAlikeReader, self).after_command(cmd) @@ -164,6 +167,28 @@ if self.pos > len(self.buffer): self.pos = len(self.buffer) +def _get_this_line_indent(buffer, pos): + indent = 0 + while pos > 0 and buffer[pos - 1] in " \t": + indent += 1 + pos -= 1 + if pos > 0 and buffer[pos - 1] == "\n": + return indent + return 0 + +def _get_previous_line_indent(buffer, pos): + prevlinestart = pos + while prevlinestart > 0 and buffer[prevlinestart - 1] != "\n": + prevlinestart -= 1 + prevlinetext = prevlinestart + while prevlinetext < pos and buffer[prevlinetext] in " \t": + prevlinetext += 1 + if prevlinetext == pos: + indent = None + else: + indent = prevlinetext - prevlinestart + return prevlinestart, indent + class maybe_accept(commands.Command): def do(self): r = self.reader @@ -172,13 +197,39 @@ # if there are already several lines and the cursor # is not on the last one, always insert a new \n. text = r.get_unicode() - if "\n" in r.buffer[r.pos:]: + if ("\n" in r.buffer[r.pos:] or + (r.more_lines is not None and r.more_lines(text))): + # + # auto-indent the next line like the previous line + prevlinestart, indent = _get_previous_line_indent(r.buffer, r.pos) r.insert("\n") - elif r.more_lines is not None and r.more_lines(text): - r.insert("\n") + if indent: + for i in range(prevlinestart, prevlinestart + indent): + r.insert(r.buffer[i]) else: self.finish = 1 +class backspace_dedent(commands.Command): + def do(self): + r = self.reader + b = r.buffer + if r.pos > 0: + repeat = 1 + if b[r.pos - 1] != "\n": + indent = _get_this_line_indent(b, r.pos) + if indent > 0: + ls = r.pos - indent + while ls > 0: + ls, pi = _get_previous_line_indent(b, ls - 1) + if pi is not None and pi < indent: + repeat = indent - pi + break + r.pos -= repeat + del b[r.pos:r.pos + repeat] + r.dirty = 1 + else: + self.reader.error("can't backspace at start") + # ____________________________________________________________ class _ReadlineWrapper(object): @@ -212,15 +263,14 @@ boolean value is true. """ reader = self.get_reader() - saved = reader.more_lines, reader.tab_insert_spaces_if_stem_is_empty + saved = reader.more_lines try: reader.more_lines = more_lines reader.ps1 = reader.ps2 = ps1 reader.ps3 = reader.ps4 = ps2 - reader.tab_insert_spaces_if_stem_is_empty = True return reader.readline(returns_unicode=returns_unicode) finally: - reader.more_lines, reader.tab_insert_spaces_if_stem_is_empty = saved + reader.more_lines = saved def parse_and_bind(self, string): pass # XXX we don't support parsing GNU-readline-style init files diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py --- a/rpython/jit/metainterp/optimizeopt/intbounds.py +++ b/rpython/jit/metainterp/optimizeopt/intbounds.py @@ -211,15 +211,15 @@ # Else, synthesize the non overflowing op for optimize_default to # reuse, as well as the reverse op elif opnum == rop.INT_ADD_OVF: - self.pure(rop.INT_ADD, args[:], result) + #self.pure(rop.INT_ADD, args[:], result) self.pure(rop.INT_SUB, [result, args[1]], args[0]) self.pure(rop.INT_SUB, [result, args[0]], args[1]) elif opnum == rop.INT_SUB_OVF: - self.pure(rop.INT_SUB, args[:], result) + #self.pure(rop.INT_SUB, args[:], result) self.pure(rop.INT_ADD, [result, args[1]], args[0]) self.pure(rop.INT_SUB, [args[0], result], args[1]) - elif opnum == rop.INT_MUL_OVF: - self.pure(rop.INT_MUL, args[:], result) + #elif opnum == rop.INT_MUL_OVF: + # self.pure(rop.INT_MUL, args[:], result) self.emit_operation(op) def optimize_GUARD_OVERFLOW(self, op): diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -323,6 +323,9 @@ def replace_op_with(self, op, newopnum, args=None, descr=None): return self.optimizer.replace_op_with(op, newopnum, args, descr) + def get_box_replacement(self, box): + return self.optimizer.get_box_replacement(box) + def make_constant(self, box, constbox): return self.optimizer.make_constant(box, constbox) @@ -399,6 +402,7 @@ class Optimizer(Optimization): exporting_state = False + emitting_dissabled = False def __init__(self, metainterp_sd, jitdriver_sd, loop, optimizations=None): self.metainterp_sd = metainterp_sd @@ -609,7 +613,7 @@ if clear: self.clear_newoperations() for op in self.loop.operations: - self._last_emitted_op = None + self._really_emitted_operation = None self.first_optimization.propagate_forward(op) self.loop.operations = self.get_newoperations() self.loop.quasi_immutable_deps = self.quasi_immutable_deps @@ -651,9 +655,12 @@ op = self.store_final_boxes_in_guard(guard_op, pendingfields) elif op.can_raise(): self.exception_might_have_happened = True - self._last_emitted_op = op + self._really_emitted_operation = op self._newoperations.append(op) + def getlastop(self): + return self._really_emitted_operation + def replace_guard_op(self, old_op_pos, new_op): old_op = self._newoperations[old_op_pos] assert old_op.is_guard() @@ -705,16 +712,6 @@ descr.make_a_counter_per_value(op) return op - def make_args_key(self, opnum, arglist, descr): - n = len(arglist) - args = [None] * (n + 2) - for i in range(n): - arg = self.get_box_replacement(arglist[i]) - args[i] = arg - args[n] = ConstInt(opnum) - args[n + 1] = descr - return args - def optimize_default(self, op): self.emit_operation(op) diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py --- a/rpython/jit/metainterp/optimizeopt/pure.py +++ b/rpython/jit/metainterp/optimizeopt/pure.py @@ -1,13 +1,59 @@ from rpython.jit.metainterp.optimizeopt.optimizer import Optimization, REMOVED -from rpython.jit.metainterp.resoperation import rop, ResOperation, OpHelpers -from rpython.jit.metainterp.optimizeopt.util import (make_dispatcher_method, - args_dict) +from rpython.jit.metainterp.resoperation import rop, ResOperation +from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method + + +class RecentPureOps(object): + REMEMBER_LIMIT = 16 + + def __init__(self): + self.lst = [None] * self.REMEMBER_LIMIT + self.next_index = 0 + + def add(self, op): + next_index = self.next_index + self.next_index = (next_index + 1) % self.REMEMBER_LIMIT + self.lst[next_index] = op + + def lookup1(self, box0, descr): + for i in range(self.REMEMBER_LIMIT): + op = self.lst[i] + if op is None: + break + if op.getarg(0).same_box(box0) and op.getdescr() is descr: + return op + return None + + def lookup2(self, box0, box1, descr): + for i in range(self.REMEMBER_LIMIT): + op = self.lst[i] + if op is None: + break + if (op.getarg(0).same_box(box0) and op.getarg(1).same_box(box1) + and op.getdescr() is descr): + return op + return None + + def lookup(self, optimizer, op): + numargs = op.numargs() + if numargs == 1: + return self.lookup1(optimizer.get_box_replacement(op.getarg(0)), + op.getdescr()) + elif numargs == 2: + return self.lookup2(optimizer.get_box_replacement(op.getarg(0)), + optimizer.get_box_replacement(op.getarg(1)), + op.getdescr()) + else: + assert False + class OptPure(Optimization): def __init__(self): self.postponed_op = None - self.pure_operations = args_dict() + self._pure_operations = [None] * (rop._ALWAYS_PURE_LAST - + rop._ALWAYS_PURE_FIRST) self.call_pure_positions = [] + self.extra_call_pure = [] def propagate_forward(self, op): dispatch_opt(self, op) @@ -25,7 +71,7 @@ else: nextop = None - args = None + save = False if canfold: for i in range(op.numargs()): if self.get_constant_box(op.getarg(i)) is None: @@ -39,22 +85,34 @@ return # did we do the exact same operation already? - if 0: - args = self.optimizer.make_args_key(op.getopnum(), - op.getarglist(), op.getdescr()) - oldval = self.pure_operations.get(args, None) - if oldval is not None: - self.optimizer.make_equal_to(op, oldval) - return + recentops = self.getrecentops(op.getopnum()) + save = True + oldop = recentops.lookup(self.optimizer, op) + if oldop is not None: + self.optimizer.make_equal_to(op, oldop) + return # otherwise, the operation remains self.emit_operation(op) if op.returns_bool_result(): self.getintbound(op).make_bool() + if save: + realop = self.get_box_replacement(op) + recentops = self.getrecentops(realop.getopnum()) + recentops.add(realop) if nextop: self.emit_operation(nextop) - #if args is not None: - # self.pure_operations[args] = self.getvalue(op) + + def getrecentops(self, opnum): + if rop._OVF_FIRST <= opnum <= rop._OVF_LAST: + opnum = opnum - rop._OVF_FIRST + else: + opnum = opnum - rop._ALWAYS_PURE_FIRST + assert 0 <= opnum < len(self._pure_operations) + recentops = self._pure_operations[opnum] + if recentops is None: + self._pure_operations[opnum] = recentops = RecentPureOps() + return recentops def optimize_CALL_PURE_I(self, op): # Step 1: check if all arguments are constant @@ -67,28 +125,45 @@ # Step 2: check if all arguments are the same as a previous # CALL_PURE. - args = self.optimizer.make_args_key(op.getopnum(), op.getarglist(), - op.getdescr()) - oldval = self.pure_operations.get(args, None) - if oldval is not None: - # this removes a CALL_PURE that has the same (non-constant) - # arguments as a previous CALL_PURE. - self.make_equal_to(op, oldval) - self.last_emitted_operation = REMOVED - return - else: - self.pure_operations[args] = self.getvalue(op) + for pos in self.call_pure_positions: + old_op = self.optimizer._newoperations[pos] + if self.optimize_call_pure(op, old_op): + return + for old_op in self.extra_call_pure: + if self.optimize_call_pure(op, old_op): + return # replace CALL_PURE with just CALL args = op.getarglist() opnum = OpHelpers.call_for_descr(op.getdescr()) newop = self.optimizer.replace_op_with(op, opnum) self.emit_operation(newop) - self.call_pure_positions.append(len(self.optimizer._newoperations) - 1) + if self.optimizer.emitting_dissabled: + self.extra_call_pure.append(op) # XXX + else: + self.call_pure_positions.append(len(self.optimizer._newoperations) + - 1) optimize_CALL_PURE_R = optimize_CALL_PURE_I optimize_CALL_PURE_F = optimize_CALL_PURE_I optimize_CALL_PURE_N = optimize_CALL_PURE_I + def optimize_call_pure(self, op, old_op): + if (op.numargs() != old_op.numargs() or + op.getdescr() is not old_op.getdescr()): + return False + for i, box in enumerate(old_op.getarglist()): + if not self.get_box_replacement(op.getarg(i)).same_box(box): + break + else: + # all identical + # this removes a CALL_PURE that has the same (non-constant) + # arguments as a previous CALL_PURE. + oldvalue = self.getvalue(old_op.result) + self.make_equal_to(op.result, oldvalue) + self.last_emitted_operation = REMOVED + return True + return False + def optimize_GUARD_NO_EXCEPTION(self, op): if self.last_emitted_operation is REMOVED: # it was a CALL_PURE that was killed; so we also kill the @@ -102,18 +177,18 @@ def setup(self): self.optimizer.optpure = self - def pure(self, opnum, args, result): - return # XXX - key = self.optimizer.make_args_key(opnum, args, None) - if key not in self.pure_operations: - self.pure_operations[key] = self.getvalue(result) + def pure(self, opnum, args, op): + op = self.get_box_replacement(op) + recentops = self.getrecentops(opnum) + recentops.add(op) def has_pure_result(self, opnum, args, descr): - key = self.optimizer.make_args_key(opnum, args, descr) - return self.pure_operations.get(key, None) is not None + return False + # XXX - def get_pure_result(self, key): - return self.pure_operations.get(key, None) + def get_pure_result(self, op): + recentops = self.getrecentops(op.getopnum()) + return recentops.lookup(self.optimizer, op) def produce_potential_short_preamble_ops(self, sb): ops = sb.optimizer._newoperations diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -27,16 +27,15 @@ def propagate_forward(self, op): if op.boolinverse != -1 or op.boolreflex != -1: - args = self.optimizer.make_args_key(op.getopnum(), - op.getarglist(), op.getdescr()) - if self.find_rewritable_bool(op, args): + if self.find_rewritable_bool(op): return dispatch_opt(self, op) def try_boolinvers(self, op, targs): - value = self.get_pure_result(targs) - if value is not None: + oldop = self.get_pure_result(targs) + if oldop is not None: + value = self.getvalue(oldop.result) if value.is_constant(): if value.box.same_constant(CONST_1): self.make_constant(op, CONST_0) @@ -48,30 +47,30 @@ return False - def find_rewritable_bool(self, op, args): + def find_rewritable_bool(self, op): oldopnum = op.boolinverse + arg0 = op.getarg(0) + arg1 = op.getarg(1) if oldopnum != -1: - targs = self.optimizer.make_args_key(oldopnum, [args[0], args[1]], - None) - if self.try_boolinvers(op, targs): + top = ResOperation(oldopnum, [arg0, arg1], None) + if self.try_boolinvers(op, top): return True oldopnum = op.boolreflex # FIXME: add INT_ADD, INT_MUL if oldopnum != -1: - targs = self.optimizer.make_args_key(oldopnum, [args[1], args[0]], - None) - oldval = self.get_pure_result(targs) - if oldval is not None: - self.make_equal_to(op, oldval) + top = ResOperation(oldopnum, [arg1, arg0], None) + oldop = self.get_pure_result(top) + if oldop is not None: + self.optimizer.make_equal_to(op.result, + self.getvalue(oldop.result), True) return True if op.boolreflex == -1: return False oldopnum = opclasses[op.boolreflex].boolinverse if oldopnum != -1: - targs = self.optimizer.make_args_key(oldopnum, [args[1], args[0]], - None) - if self.try_boolinvers(op, targs): + top = ResOperation(oldopnum, [arg1, arg0], None) + if self.try_boolinvers(op, top): return True return False diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -4759,6 +4759,7 @@ self.optimize_loop(ops, expected) def test_complains_getfieldpure_setfield(self): + py.test.skip("disabled for now") from rpython.jit.metainterp.optimizeopt.heap import BogusPureField ops = """ [p3] diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -779,7 +779,7 @@ '_CANRAISE_LAST', # ----- end of can_raise operations ----- '_OVF_FIRST', # ----- start of is_ovf operations ----- - 'INT_ADD_OVF/2/i', + 'INT_ADD_OVF/2/i', # note that the orded has to match INT_ADD order 'INT_SUB_OVF/2/i', 'INT_MUL_OVF/2/i', '_OVF_LAST', # ----- end of is_ovf operations ----- From noreply at buildbot.pypy.org Fri Mar 6 17:13:31 2015 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 6 Mar 2015 17:13:31 +0100 (CET) Subject: [pypy-commit] pypy optresult: more tests passing, really just whacking at it Message-ID: <20150306161331.7A0E91C0E95@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult Changeset: r76257:5a1e7576366b Date: 2015-03-06 17:29 +0200 http://bitbucket.org/pypy/pypy/changeset/5a1e7576366b/ Log: more tests passing, really just whacking at it diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py --- a/rpython/jit/metainterp/optimizeopt/info.py +++ b/rpython/jit/metainterp/optimizeopt/info.py @@ -78,6 +78,12 @@ def __init__(self, const): self._const = const + def is_null(self): + return not bool(self._const.getref_base()) + + def is_nonnull(self): + return bool(self._const.getref_base()) + def get_known_class(self, cpu): if not self._const.nonnull(): return None diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -262,24 +262,26 @@ # self.optimizer.optheap.value_updated(value, self.getvalue(constbox)) def optimize_GUARD_ISNULL(self, op): - value = self.getvalue(op.getarg(0)) - if value.is_null(): - return - elif value.is_nonnull(): - r = self.optimizer.metainterp_sd.logger_ops.repr_of_resop(op) - raise InvalidLoop('A GUARD_ISNULL (%s) was proven to always fail' - % r) + info = self.getptrinfo(op.getarg(0)) + if info is not None: + if info.is_null(): + return + elif info.is_nonnull(): + r = self.optimizer.metainterp_sd.logger_ops.repr_of_resop(op) + raise InvalidLoop('A GUARD_ISNULL (%s) was proven to always ' + 'fail' % r) self.emit_operation(op) - value.make_constant(self.optimizer.cpu.ts.CONST_NULL) + self.make_constant(op.getarg(0), self.optimizer.cpu.ts.CONST_NULL) def optimize_GUARD_NONNULL(self, op): opinfo = self.getptrinfo(op.getarg(0)) - if opinfo.is_nonnull(): - return - elif opinfo.is_null(): - r = self.optimizer.metainterp_sd.logger_ops.repr_of_resop(op) - raise InvalidLoop('A GUARD_NONNULL (%s) was proven to always fail' - % r) + if opinfo is not None: + if opinfo.is_nonnull(): + return + elif opinfo.is_null(): + r = self.optimizer.metainterp_sd.logger_ops.repr_of_resop(op) + raise InvalidLoop('A GUARD_NONNULL (%s) was proven to always ' + 'fail' % r) self.emit_operation(op) self.make_nonnull(op.getarg(0)) From noreply at buildbot.pypy.org Fri Mar 6 17:13:30 2015 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 6 Mar 2015 17:13:30 +0100 (CET) Subject: [pypy-commit] pypy optresult: make more tests pass Message-ID: <20150306161330.57E781C0E95@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult Changeset: r76256:443a81259c45 Date: 2015-03-06 17:24 +0200 http://bitbucket.org/pypy/pypy/changeset/443a81259c45/ Log: make more tests pass diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py --- a/rpython/jit/metainterp/optimizeopt/intbounds.py +++ b/rpython/jit/metainterp/optimizeopt/intbounds.py @@ -521,14 +521,14 @@ b1.make_gt(IntBound(0, 0)) self.propagate_bounds_backward(op.getarg(0)) elif r.getint() == valzero: - XXX - v1 = self.getvalue(op.getarg(0)) + b1 = self.getintbound(op.getarg(0)) + # XXX remove this hack maybe? # Clever hack, we can't use self.make_constant_int yet because # the args aren't in the values dictionary yet so it runs into # an assert, this is a clever way of expressing the same thing. - v1.getintbound().make_ge(IntBound(0, 0)) - v1.getintbound().make_lt(IntBound(1, 1)) - self.propagate_bounds_backward(op.getarg(0), v1) + b1.make_ge(IntBound(0, 0)) + b1.make_lt(IntBound(1, 1)) + self.propagate_bounds_backward(op.getarg(0)) def propagate_bounds_INT_IS_TRUE(self, op): self._propagate_int_is_true_or_zero(op, 1, 0) diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -437,8 +437,7 @@ def optimize_INT_IS_TRUE(self, op): if self.getintbound(op.getarg(0)).is_bool(): - xxx - self.make_equal_to(op, self.getvalue(op.getarg(0))) + self.make_equal_to(op, op.getarg(0)) return self._optimize_nullness(op, op.getarg(0), True) From noreply at buildbot.pypy.org Fri Mar 6 17:13:32 2015 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 6 Mar 2015 17:13:32 +0100 (CET) Subject: [pypy-commit] pypy optresult: pass the first virtual test Message-ID: <20150306161332.9EB911C0E95@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult Changeset: r76258:1f70e61d2780 Date: 2015-03-06 18:13 +0200 http://bitbucket.org/pypy/pypy/changeset/1f70e61d2780/ Log: pass the first virtual test diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py --- a/rpython/jit/metainterp/optimizeopt/info.py +++ b/rpython/jit/metainterp/optimizeopt/info.py @@ -41,6 +41,7 @@ class InstancePtrInfo(NonNullPtrInfo): _attrs_ = ('_known_class', '_is_virtual', '_fields') + _fields = None def __init__(self, known_class=None, is_virtual=False): self._known_class = known_class @@ -57,6 +58,17 @@ return newop return op + def setfield_virtual(self, descr, op): + if self._fields is None: + self._fields = {} + self._fields[descr] = op + + def getfield_virtual(self, descr): + return self._fields.get(descr, None) + + def is_virtual(self): + return self._is_virtual + def get_known_class(self, cpu): return self._known_class diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -665,22 +665,24 @@ return False def optimize_GETFIELD_GC_I(self, op): - value = self.getvalue(op.getarg(0)) + opinfo = self.getptrinfo(op.getarg(0)) # If this is an immutable field (as indicated by op.is_always_pure()) # then it's safe to reuse the virtual's field, even if it has been # forced, because it should never be written to again. - if value.is_forced_virtual() and op.is_always_pure(): - fieldvalue = value.getfield(op.getdescr(), None) - if fieldvalue is not None: - self.make_equal_to(op, fieldvalue) - return - if value.is_virtual(): - assert isinstance(value, AbstractVirtualValue) - fieldvalue = value.getfield(op.getdescr(), None) - if fieldvalue is None: + if op.is_always_pure(): + if value.is_forced_virtual() and op.is_always_pure(): + fieldvalue = value.getfield(op.getdescr(), None) + if fieldvalue is not None: + self.make_equal_to(op, fieldvalue) + return + if opinfo and opinfo.is_virtual(): + fieldop = opinfo.getfield_virtual(op.getdescr()) + if fieldop is None: + xxx fieldvalue = self.optimizer.new_const(op.getdescr()) - self.make_equal_to(op, fieldvalue) + self.make_equal_to(op, fieldop) else: + yyyy value.ensure_nonnull() self.emit_operation(op) optimize_GETFIELD_GC_R = optimize_GETFIELD_GC_I @@ -693,11 +695,11 @@ optimize_GETFIELD_GC_PURE_F = optimize_GETFIELD_GC_I def optimize_SETFIELD_GC(self, op): - value = self.getvalue(op.getarg(0)) - if value.is_virtual(): - fieldvalue = self.getvalue(op.getarg(1)) - value.setfield(op.getdescr(), fieldvalue) + opinfo = self.getptrinfo(op.getarg(0)) + if opinfo is not None and opinfo.is_virtual(): + opinfo.setfield_virtual(op.getdescr(), op.getarg(1)) else: + xxx value.ensure_nonnull() self.emit_operation(op) From noreply at buildbot.pypy.org Fri Mar 6 18:42:36 2015 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 6 Mar 2015 18:42:36 +0100 (CET) Subject: [pypy-commit] pypy optresult: one more test Message-ID: <20150306174236.E21741C04A7@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult Changeset: r76260:eb2bed1c7f35 Date: 2015-03-06 19:24 +0200 http://bitbucket.org/pypy/pypy/changeset/eb2bed1c7f35/ Log: one more test diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -507,6 +507,8 @@ optimize_GETFIELD_GC_PURE_F = optimize_GETFIELD_GC_PURE_I def optimize_SETFIELD_GC(self, op): + self.emit_operation(op) + return opnum = OpHelpers.getfield_pure_for_descr(op.getdescr()) if self.has_pure_result(opnum, [op.getarg(0)], op.getdescr()): From noreply at buildbot.pypy.org Fri Mar 6 18:42:35 2015 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 6 Mar 2015 18:42:35 +0100 (CET) Subject: [pypy-commit] pypy optresult: one more test Message-ID: <20150306174235.B1B211C04A7@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult Changeset: r76259:be5ec91a624a Date: 2015-03-06 19:24 +0200 http://bitbucket.org/pypy/pypy/changeset/be5ec91a624a/ Log: one more test diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py --- a/rpython/jit/metainterp/optimizeopt/info.py +++ b/rpython/jit/metainterp/optimizeopt/info.py @@ -15,6 +15,10 @@ MODE_STR = '\x01' MODE_UNICODE = '\x02' +INFO_NULL = 0 +INFO_NONNULL = 1 +INFO_UNKNOWN = 2 + class AbstractInfo(AbstractValue): is_info_class = True @@ -31,6 +35,13 @@ def is_null(self): return False + def getnullness(self): + if self.is_null(): + return INFO_NULL + elif self.is_nonnull(): + return INFO_NONNULL + return INFO_UNKNOWN + class NonNullPtrInfo(PtrInfo): _attrs_ = () @@ -96,6 +107,9 @@ def is_nonnull(self): return bool(self._const.getref_base()) + def is_virtual(self): + return False + def get_known_class(self, cpu): if not self._const.nonnull(): return None diff --git a/rpython/jit/metainterp/optimizeopt/intutils.py b/rpython/jit/metainterp/optimizeopt/intutils.py --- a/rpython/jit/metainterp/optimizeopt/intutils.py +++ b/rpython/jit/metainterp/optimizeopt/intutils.py @@ -1,7 +1,8 @@ from rpython.rlib.rarithmetic import ovfcheck, LONG_BIT, maxint, is_valid_int from rpython.rlib.objectmodel import we_are_translated from rpython.jit.metainterp.resoperation import rop, ResOperation -from rpython.jit.metainterp.optimizeopt.info import AbstractInfo +from rpython.jit.metainterp.optimizeopt.info import AbstractInfo, INFO_NONNULL,\ + INFO_UNKNOWN, INFO_NULL from rpython.jit.metainterp.history import ConstInt @@ -261,15 +262,13 @@ self.intersect(IntBound(0, 1)) def getnullness(self): - from rpython.jit.metainterp.optimizeopt import optimizer - if self.known_gt(IntBound(0, 0)) or \ self.known_lt(IntBound(0, 0)): - return optimizer.INFO_NONNULL + return INFO_NONNULL if self.known_ge(IntBound(0, 0)) and \ self.known_le(IntBound(0, 0)): - return optimizer.INFO_NULL - return optimizer.INFO_UNKNOWN + return INFO_NULL + return INFO_UNKNOWN def IntUpperBound(upper): b = IntBound(lower=0, upper=upper) diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -257,9 +257,6 @@ llhelper.CONST_NULLREF = llhelper.CONST_NULL REMOVED = AbstractResOp() -INFO_NULL = 0 -INFO_NONNULL = 1 -INFO_UNKNOWN = 2 class Optimization(object): next_optimization = None @@ -291,7 +288,12 @@ def getnullness(self, op): if op.type == 'i': return self.getintbound(op).getnullness() - xxxx + elif op.type == 'r': + ptrinfo = self.getptrinfo(op) + if ptrinfo is None: + return info.INFO_UNKNOWN + return ptrinfo.getnullness() + assert False def make_constant_class(self, op, class_const): op = self.get_box_replacement(op) diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -1,12 +1,13 @@ from rpython.jit.codewriter.effectinfo import EffectInfo from rpython.jit.codewriter import longlong from rpython.jit.metainterp import compile -from rpython.jit.metainterp.history import (Const, ConstInt, BoxInt, BoxFloat, - BoxPtr, make_hashable_int, ConstFloat) +from rpython.jit.metainterp.history import (Const, ConstInt, make_hashable_int, + ConstFloat) from rpython.jit.metainterp.optimize import InvalidLoop from rpython.jit.metainterp.optimizeopt.intutils import IntBound from rpython.jit.metainterp.optimizeopt.optimizer import (Optimization, REMOVED, - CONST_0, CONST_1, INFO_NONNULL, INFO_NULL) + CONST_0, CONST_1) +from rpython.jit.metainterp.optimizeopt.info import INFO_NONNULL, INFO_NULL from rpython.jit.metainterp.optimizeopt.util import _findall, make_dispatcher_method from rpython.jit.metainterp.resoperation import rop, ResOperation, opclasses,\ OpHelpers @@ -447,19 +448,20 @@ self._optimize_nullness(op, op.getarg(0), False) def _optimize_oois_ooisnot(self, op, expect_isnot, instance): - value0 = self.getvalue(op.getarg(0)) - value1 = self.getvalue(op.getarg(1)) - if value0.is_virtual(): + info0 = self.getptrinfo(op.getarg(0)) + info1 = self.getptrinfo(op.getarg(1)) + if info0 and info0.is_virtual(): + xxx if value1.is_virtual(): intres = (value0 is value1) ^ expect_isnot self.make_constant_int(op, intres) else: self.make_constant_int(op, expect_isnot) - elif value1.is_virtual(): + elif info1 and info1.is_virtual(): self.make_constant_int(op, expect_isnot) - elif value1.is_null(): + elif info1 and info1.is_null(): self._optimize_nullness(op, op.getarg(0), expect_isnot) - elif value0.is_null(): + elif info0 and info0.is_null(): self._optimize_nullness(op, op.getarg(1), expect_isnot) elif value0 is value1: self.make_constant_int(op, not expect_isnot) diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -699,8 +699,7 @@ if opinfo is not None and opinfo.is_virtual(): opinfo.setfield_virtual(op.getdescr(), op.getarg(1)) else: - xxx - value.ensure_nonnull() + self.make_nonnull(op) self.emit_operation(op) def optimize_NEW_WITH_VTABLE(self, op): From noreply at buildbot.pypy.org Fri Mar 6 18:42:38 2015 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 6 Mar 2015 18:42:38 +0100 (CET) Subject: [pypy-commit] pypy optresult: whack whack whack Message-ID: <20150306174238.0D0B21C04A7@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult Changeset: r76261:514545cbbcfb Date: 2015-03-06 19:42 +0200 http://bitbucket.org/pypy/pypy/changeset/514545cbbcfb/ Log: whack whack whack diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py --- a/rpython/jit/metainterp/optimizeopt/info.py +++ b/rpython/jit/metainterp/optimizeopt/info.py @@ -35,6 +35,9 @@ def is_null(self): return False + def is_virtual(self): + return False + def getnullness(self): if self.is_null(): return INFO_NULL diff --git a/rpython/jit/metainterp/optimizeopt/intutils.py b/rpython/jit/metainterp/optimizeopt/intutils.py --- a/rpython/jit/metainterp/optimizeopt/intutils.py +++ b/rpython/jit/metainterp/optimizeopt/intutils.py @@ -46,6 +46,14 @@ def make_gt(self, other): return self.make_ge(other.add(1)) + def is_constant(self): + return self.has_upper and self.has_lower and self.lower == self.upper + + def equal(self, value): + if not self.is_constant(): + return False + return self.lower == value + def make_constant(self, value): XXXX # don't call me self.has_lower = True diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -36,15 +36,13 @@ def try_boolinvers(self, op, targs): oldop = self.get_pure_result(targs) if oldop is not None: - value = self.getvalue(oldop.result) - if value.is_constant(): - if value.box.same_constant(CONST_1): - self.make_constant(op, CONST_0) - return True - elif value.box.same_constant(CONST_0): - self.make_constant(op, CONST_1) - return True - + b = self.getintbound(oldop) + if b.equal(1): + self.make_constant(op, CONST_0) + return True + elif b.equal(0): + self.make_constant(op, CONST_1) + return True return False @@ -62,8 +60,7 @@ top = ResOperation(oldopnum, [arg1, arg0], None) oldop = self.get_pure_result(top) if oldop is not None: - self.optimizer.make_equal_to(op.result, - self.getvalue(oldop.result), True) + self.optimizer.make_equal_to(op, oldop) return True if op.boolreflex == -1: diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -699,7 +699,7 @@ if opinfo is not None and opinfo.is_virtual(): opinfo.setfield_virtual(op.getdescr(), op.getarg(1)) else: - self.make_nonnull(op) + self.make_nonnull(op.getarg(0)) self.emit_operation(op) def optimize_NEW_WITH_VTABLE(self, op): From noreply at buildbot.pypy.org Fri Mar 6 19:04:09 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 6 Mar 2015 19:04:09 +0100 (CET) Subject: [pypy-commit] pypy recent-pure-ops: Add more precise assertions than just one comment Message-ID: <20150306180409.ED9171C04A7@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: recent-pure-ops Changeset: r76262:e1cfdc63467c Date: 2015-03-06 19:04 +0100 http://bitbucket.org/pypy/pypy/changeset/e1cfdc63467c/ Log: Add more precise assertions than just one comment diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -594,6 +594,13 @@ oparity.append(arity) opwithdescr.append(withdescr) assert len(opclasses) == len(oparity) == len(opwithdescr) == len(_oplist) + # for optimizeopt/pure.py's getrecentops() + assert (rop.INT_ADD_OVF - rop._OVF_FIRST == + rop.INT_ADD - rop._ALWAYS_PURE_FIRST) + assert (rop.INT_SUB_OVF - rop._OVF_FIRST == + rop.INT_SUB - rop._ALWAYS_PURE_FIRST) + assert (rop.INT_MUL_OVF - rop._OVF_FIRST == + rop.INT_MUL - rop._ALWAYS_PURE_FIRST) def get_base_class(mixin, base): try: From noreply at buildbot.pypy.org Fri Mar 6 23:35:11 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 6 Mar 2015 23:35:11 +0100 (CET) Subject: [pypy-commit] pypy.org extradoc: update the values Message-ID: <20150306223511.74E2C1C00FC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r576:bba799524d7c Date: 2015-03-06 23:35 +0100 http://bitbucket.org/pypy/pypy.org/changeset/bba799524d7c/ Log: update the values diff --git a/don4.html b/don4.html --- a/don4.html +++ b/don4.html @@ -17,7 +17,7 @@ 2nd call: - $22463 of $80000 (28.1%) + $22473 of $80000 (28.1%)
From noreply at buildbot.pypy.org Sat Mar 7 03:04:32 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Sat, 7 Mar 2015 03:04:32 +0100 (CET) Subject: [pypy-commit] pypy default: merge branch 'online-transforms-2' Message-ID: <20150307020432.B4D701C04A7@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r76264:4b7d6f8bc860 Date: 2015-03-07 01:57 +0000 http://bitbucket.org/pypy/pypy/changeset/4b7d6f8bc860/ Log: merge branch 'online-transforms-2' diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py --- a/rpython/annotator/annrpython.py +++ b/rpython/annotator/annrpython.py @@ -399,12 +399,22 @@ def flowin(self, graph, block): try: - for i, op in enumerate(block.operations): + i = 0 + while i < len(block.operations): + op = block.operations[i] self.bookkeeper.enter((graph, block, i)) try: + new_ops = op.transform(self) + if new_ops is not None: + block.operations[i:i+1] = new_ops + if not new_ops: + continue + new_ops[-1].result = op.result + op = new_ops[0] self.consider_op(op) finally: self.bookkeeper.leave() + i += 1 except BlockedInference as e: if e.op is block.raising_op: diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py --- a/rpython/annotator/binaryop.py +++ b/rpython/annotator/binaryop.py @@ -2,7 +2,6 @@ Binary operations between SomeValues. """ -import operator from rpython.tool.pairtype import pair, pairtype from rpython.annotator.model import ( SomeObject, SomeInteger, SomeBool, s_Bool, SomeString, SomeChar, SomeList, @@ -14,7 +13,7 @@ read_can_only_throw, add_knowntypedata, merge_knowntypedata,) from rpython.annotator.bookkeeper import immutablevalue -from rpython.flowspace.model import Variable, Constant +from rpython.flowspace.model import Variable, Constant, const from rpython.flowspace.operation import op from rpython.rlib import rarithmetic from rpython.annotator.model import AnnotatorError @@ -689,12 +688,16 @@ return super(thistype, pair(ins1, ins2)).improve() -class __extend__(pairtype(SomeInstance, SomeObject)): - def getitem((s_ins, s_idx)): - return s_ins._emulate_call("__getitem__", s_idx) + at op.getitem.register_transform(SomeInstance, SomeObject) +def getitem_SomeInstance(annotator, v_ins, v_idx): + get_getitem = op.getattr(v_ins, const('__getitem__')) + return [get_getitem, op.simple_call(get_getitem.result, v_idx)] - def setitem((s_ins, s_idx), s_value): - return s_ins._emulate_call("__setitem__", s_idx, s_value) + at op.setitem.register_transform(SomeInstance, SomeObject) +def setitem_SomeInstance(annotator, v_ins, v_idx, v_value): + get_setitem = op.getattr(v_ins, const('__setitem__')) + return [get_setitem, + op.simple_call(get_setitem.result, v_idx, v_value)] class __extend__(pairtype(SomeIterator, SomeIterator)): diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py --- a/rpython/annotator/unaryop.py +++ b/rpython/annotator/unaryop.py @@ -5,6 +5,7 @@ from __future__ import absolute_import from rpython.flowspace.operation import op +from rpython.flowspace.model import const from rpython.annotator.model import (SomeObject, SomeInteger, SomeBool, SomeString, SomeChar, SomeList, SomeDict, SomeTuple, SomeImpossibleValue, SomeUnicodeCodePoint, SomeInstance, SomeBuiltin, SomeBuiltinMethod, @@ -686,27 +687,33 @@ if not self.can_be_None: s.const = True - def _emulate_call(self, meth_name, *args_s): - bk = getbookkeeper() - s_attr = self._true_getattr(meth_name) - # record for calltables - bk.emulate_pbc_call(bk.position_key, s_attr, args_s) - return s_attr.call(simple_args(args_s)) + at op.len.register_transform(SomeInstance) +def len_SomeInstance(annotator, v_arg): + get_len = op.getattr(v_arg, const('__len__')) + return [get_len, op.simple_call(get_len.result)] - def iter(self): - return self._emulate_call('__iter__') + at op.iter.register_transform(SomeInstance) +def iter_SomeInstance(annotator, v_arg): + get_iter = op.getattr(v_arg, const('__iter__')) + return [get_iter, op.simple_call(get_iter.result)] - def next(self): - return self._emulate_call('next') + at op.next.register_transform(SomeInstance) +def next_SomeInstance(annotator, v_arg): + get_next = op.getattr(v_arg, const('next')) + return [get_next, op.simple_call(get_next.result)] - def len(self): - return self._emulate_call('__len__') + at op.getslice.register_transform(SomeInstance) +def getslice_SomeInstance(annotator, v_obj, v_start, v_stop): + get_getslice = op.getattr(v_obj, const('__getslice__')) + return [get_getslice, op.simple_call(get_getslice.result, v_start, v_stop)] - def getslice(self, s_start, s_stop): - return self._emulate_call('__getslice__', s_start, s_stop) - def setslice(self, s_start, s_stop, s_iterable): - return self._emulate_call('__setslice__', s_start, s_stop, s_iterable) + at op.setslice.register_transform(SomeInstance) +def setslice_SomeInstance(annotator, v_obj, v_start, v_stop, v_iterable): + get_setslice = op.getattr(v_obj, const('__setslice__')) + return [get_setslice, + op.simple_call(get_setslice.result, v_start, v_stop, v_iterable)] + class __extend__(SomeBuiltin): def call(self, args, implicit_init=False): diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py --- a/rpython/flowspace/operation.py +++ b/rpython/flowspace/operation.py @@ -57,8 +57,10 @@ setattr(op, cls.opname, cls) if cls.dispatch == 1: cls._registry = {} + cls._transform = {} elif cls.dispatch == 2: cls._registry = DoubleDispatchRegistry() + cls._transform = DoubleDispatchRegistry() class HLOperation(SpaceOperation): @@ -104,6 +106,14 @@ def get_can_only_throw(self, annotator): return None + def get_transformer(self, *args_s): + return lambda *args: None + + def transform(self, annotator): + args_s = [annotator.annotation(arg) for arg in self.args] + transformer = self.get_transformer(*args_s) + return transformer(annotator, *self.args) + class PureOperation(HLOperation): pure = True @@ -185,6 +195,37 @@ except AttributeError: return cls._dispatch(type(s_arg)) + @classmethod + def get_specialization(cls, s_arg, *_ignored): + try: + impl = getattr(s_arg, cls.opname) + + def specialized(annotator, arg, *other_args): + return impl(*[annotator.annotation(x) for x in other_args]) + try: + specialized.can_only_throw = impl.can_only_throw + except AttributeError: + pass + return specialized + except AttributeError: + return cls._dispatch(type(s_arg)) + + @classmethod + def register_transform(cls, Some_cls): + def decorator(func): + cls._transform[Some_cls] = func + return func + return decorator + + @classmethod + def get_transformer(cls, s_arg, *_ignored): + for c in type(s_arg).__mro__: + try: + return cls._transform[c] + except KeyError: + pass + return lambda *args: None + class DoubleDispatchMixin(object): dispatch = 2 @@ -216,6 +257,20 @@ spec = type(self).get_specialization(*args_s) return read_can_only_throw(spec, args_s[0], args_s[1]) + @classmethod + def register_transform(cls, Some1, Some2): + def decorator(func): + cls._transform[Some1, Some2] = func + return func + return decorator + + @classmethod + def get_transformer(cls, s_arg1, s_arg2, *_ignored): + try: + return cls._transform[type(s_arg1), type(s_arg2)] + except KeyError: + return lambda *args: None + def add_operator(name, arity, dispatch=None, pyfunc=None, pure=False, ovf=False): operator_func = getattr(operator, name, None) diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py --- a/rpython/rtyper/rclass.py +++ b/rpython/rtyper/rclass.py @@ -820,40 +820,6 @@ vinst, = hop.inputargs(self) return hop.genop('ptr_nonzero', [vinst], resulttype=Bool) - def _emulate_call(self, hop, meth_name): - vinst = hop.args_v[0] - clsdef = hop.args_s[0].classdef - s_unbound_attr = clsdef.find_attribute(meth_name).getvalue() - s_attr = clsdef.lookup_filter(s_unbound_attr, meth_name, - hop.args_s[0].flags) - # does that even happen? - assert not s_attr.is_constant() - if '__iter__' in self.allinstancefields: - raise Exception("__iter__ on instance disallowed") - r_method = self.rtyper.getrepr(s_attr) - r_method.get_method_from_instance(self, vinst, hop.llops) - hop2 = hop.copy() - hop2.spaceop = op.simple_call(*hop.spaceop.args) - hop2.spaceop.result = hop.spaceop.result - hop2.args_r[0] = r_method - hop2.args_s[0] = s_attr - return hop2.dispatch() - - def rtype_iter(self, hop): - return self._emulate_call(hop, '__iter__') - - def rtype_next(self, hop): - return self._emulate_call(hop, 'next') - - def rtype_getslice(self, hop): - return self._emulate_call(hop, "__getslice__") - - def rtype_setslice(self, hop): - return self._emulate_call(hop, "__setslice__") - - def rtype_len(self, hop): - return self._emulate_call(hop, "__len__") - def ll_str(self, i): # doesn't work for non-gc classes! from rpython.rtyper.lltypesystem.ll_str import ll_int2hex from rpython.rlib.rarithmetic import r_uint @@ -1023,14 +989,6 @@ return hop.gendirectcall(ll_isinstance, v_obj, v_cls) -class __extend__(pairtype(InstanceRepr, Repr)): - def rtype_getitem((r_ins, r_obj), hop): - return r_ins._emulate_call(hop, "__getitem__") - - def rtype_setitem((r_ins, r_obj), hop): - return r_ins._emulate_call(hop, "__setitem__") - - class __extend__(pairtype(InstanceRepr, InstanceRepr)): def convert_from_to((r_ins1, r_ins2), v, llops): # which is a subclass of which? From noreply at buildbot.pypy.org Sat Mar 7 03:04:33 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Sat, 7 Mar 2015 03:04:33 +0100 (CET) Subject: [pypy-commit] pypy default: document branch Message-ID: <20150307020433.E25631C04A7@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r76265:2b553dc58a66 Date: 2015-03-07 02:05 +0000 http://bitbucket.org/pypy/pypy/changeset/2b553dc58a66/ Log: document branch diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -34,3 +34,6 @@ .. branch: nonquadratic-heapcache Speed up the warmup times of the JIT by removing a quadratic algorithm in the heapcache. + +.. branch: online-transforms-2 +Simplify flow graphs on the fly during annotation phase. From noreply at buildbot.pypy.org Sat Mar 7 03:04:31 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Sat, 7 Mar 2015 03:04:31 +0100 (CET) Subject: [pypy-commit] pypy online-transforms-2: close branch before merging Message-ID: <20150307020431.358981C04A7@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: online-transforms-2 Changeset: r76263:fe220b4c7cbb Date: 2015-03-07 01:43 +0000 http://bitbucket.org/pypy/pypy/changeset/fe220b4c7cbb/ Log: close branch before merging From noreply at buildbot.pypy.org Sat Mar 7 09:31:03 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 7 Mar 2015 09:31:03 +0100 (CET) Subject: [pypy-commit] pypy default: More missing stack alignment Message-ID: <20150307083103.12C9A1C1357@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76266:df8de9ee47b5 Date: 2015-03-07 09:31 +0100 http://bitbucket.org/pypy/pypy/changeset/df8de9ee47b5/ Log: More missing stack alignment diff --git a/rpython/translator/c/src/stacklet/switch_x86_64_gcc.h b/rpython/translator/c/src/stacklet/switch_x86_64_gcc.h --- a/rpython/translator/c/src/stacklet/switch_x86_64_gcc.h +++ b/rpython/translator/c/src/stacklet/switch_x86_64_gcc.h @@ -10,7 +10,10 @@ "pushq %%r12\n" "pushq %%r13\n" "pushq %%r14\n" + "movq %%rsp, %%rbp\n" + "andq $-16, %%rsp\n" /* <= align the stack here... */ "pushq %%r15\n" + "pushq %%rbp\n" /* ...so that rsp is now a multiple of 16 */ "movq %%rax, %%r12\n" /* save 'restore_state' for later */ "movq %%rsi, %%r13\n" /* save 'extra' for later */ @@ -34,7 +37,9 @@ /* The stack's content is now restored. */ "0:\n" + "popq %%rbp\n" "popq %%r15\n" + "movq %%rbp, %%rsp\n" "popq %%r14\n" "popq %%r13\n" "popq %%r12\n" From noreply at buildbot.pypy.org Sat Mar 7 09:45:19 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 7 Mar 2015 09:45:19 +0100 (CET) Subject: [pypy-commit] pypy default: Copy the stack alignment logic here too Message-ID: <20150307084519.D8B861C157F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76267:1fdade0f6b61 Date: 2015-03-07 09:37 +0100 http://bitbucket.org/pypy/pypy/changeset/1fdade0f6b61/ Log: Copy the stack alignment logic here too diff --git a/rpython/translator/c/src/stacklet/switch_x86_gcc.h b/rpython/translator/c/src/stacklet/switch_x86_gcc.h --- a/rpython/translator/c/src/stacklet/switch_x86_gcc.h +++ b/rpython/translator/c/src/stacklet/switch_x86_gcc.h @@ -8,7 +8,10 @@ "pushl %%ebp\n" "pushl %%ebx\n" /* push some registers that may contain */ "pushl %%esi\n" /* some value that is meant to be saved */ + "movl %%esp, %%ebp\n" + "andl $-16, %%esp\n" /* <= align the stack here, for the calls */ "pushl %%edi\n" + "pushl %%ebp\n" "movl %%eax, %%esi\n" /* save 'restore_state' for later */ "movl %%edx, %%edi\n" /* save 'extra' for later */ @@ -35,7 +38,9 @@ "0:\n" "addl $8, %%esp\n" + "popl %%ebp\n" "popl %%edi\n" + "movl %%ebp, %%esp\n" "popl %%esi\n" "popl %%ebx\n" "popl %%ebp\n" From noreply at buildbot.pypy.org Sat Mar 7 13:17:01 2015 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 7 Mar 2015 13:17:01 +0100 (CET) Subject: [pypy-commit] pypy optresult: implement guard_value -> true/false Message-ID: <20150307121701.E50B61C052B@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult Changeset: r76268:852623badb92 Date: 2015-03-06 20:58 +0200 http://bitbucket.org/pypy/pypy/changeset/852623badb92/ Log: implement guard_value -> true/false diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -692,26 +692,25 @@ raise compile.giveup() descr.store_final_boxes(op, newboxes, self.metainterp_sd) # - if op.getopnum() == rop.GUARD_VALUE and 0: # XXX - val = self.getvalue(op.getarg(0)) - if val in self.bool_boxes: - # Hack: turn guard_value(bool) into guard_true/guard_false. - # This is done after the operation is emitted to let - # store_final_boxes_in_guard set the guard_opnum field of the - # descr to the original rop.GUARD_VALUE. - constvalue = op.getarg(1).getint() - if constvalue == 0: - opnum = rop.GUARD_FALSE - elif constvalue == 1: - opnum = rop.GUARD_TRUE - else: - raise AssertionError("uh?") - newop = self.replace_op_with(op, opnum, [op.getarg(0)], descr) - newop.setfailargs(op.getfailargs()) - return newop - else: - # a real GUARD_VALUE. Make it use one counter per value. - descr.make_a_counter_per_value(op) + if op.getopnum() == rop.GUARD_VALUE: + if op.getarg(0).type == 'i': + b = self.getintbound(op.getarg(0)) + if b.is_bool(): + # Hack: turn guard_value(bool) into guard_true/guard_false. + # This is done after the operation is emitted to let + # store_final_boxes_in_guard set the guard_opnum field of + # the descr to the original rop.GUARD_VALUE. + constvalue = op.getarg(1).getint() + if constvalue == 0: + opnum = rop.GUARD_FALSE + elif constvalue == 1: + opnum = rop.GUARD_TRUE + else: + raise AssertionError("uh?") + newop = self.replace_op_with(op, opnum, [op.getarg(0)], descr) + return newop + # a real GUARD_VALUE. Make it use one counter per value. + descr.make_a_counter_per_value(op) return op def optimize_default(self, op): From noreply at buildbot.pypy.org Sat Mar 7 13:17:03 2015 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 7 Mar 2015 13:17:03 +0100 (CET) Subject: [pypy-commit] pypy optresult: basic virtuals Message-ID: <20150307121703.2C1461C052B@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult Changeset: r76269:917cc4c8ef85 Date: 2015-03-07 14:05 +0200 http://bitbucket.org/pypy/pypy/changeset/917cc4c8ef85/ Log: basic virtuals diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py --- a/rpython/jit/backend/llgraph/runner.py +++ b/rpython/jit/backend/llgraph/runner.py @@ -89,8 +89,10 @@ return getkind(self.RESULT)[0] class SizeDescr(AbstractDescr): - def __init__(self, S): + def __init__(self, S, runner): self.S = S + self.all_fielddescrs = heaptracker.all_fielddescrs(runner, S, + get_field_descr=LLGraphCPU.fielddescrof) def as_vtable_size_descr(self): return self @@ -106,6 +108,7 @@ self.S = S self.fieldname = fieldname self.FIELD = getattr(S, fieldname) + self.index = heaptracker.get_fielddescr_index_in(S, fieldname) def get_vinfo(self): return self.vinfo @@ -374,7 +377,7 @@ try: return self.descrs[key] except KeyError: - descr = SizeDescr(S) + descr = SizeDescr(S, self) self.descrs[key] = descr return descr diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py --- a/rpython/jit/backend/llsupport/descr.py +++ b/rpython/jit/backend/llsupport/descr.py @@ -36,10 +36,11 @@ tid = llop.combine_ushort(lltype.Signed, 0, 0) def __init__(self, size, count_fields_if_immut=-1, - gc_fielddescrs=None): + gc_fielddescrs=None, all_fielddescrs=None): self.size = size self.count_fields_if_immut = count_fields_if_immut self.gc_fielddescrs = gc_fielddescrs + self.all_fielddescrs = all_fielddescrs def count_fields_if_immutable(self): return self.count_fields_if_immut @@ -61,12 +62,13 @@ size = symbolic.get_size(STRUCT, gccache.translate_support_code) count_fields_if_immut = heaptracker.count_fields_if_immutable(STRUCT) gc_fielddescrs = heaptracker.gc_fielddescrs(gccache, STRUCT) + all_fielddescrs = heaptracker.all_fielddescrs(gccache, STRUCT) if heaptracker.has_gcstruct_a_vtable(STRUCT): sizedescr = SizeDescrWithVTable(size, count_fields_if_immut, - gc_fielddescrs) + gc_fielddescrs, all_fielddescrs) else: sizedescr = SizeDescr(size, count_fields_if_immut, - gc_fielddescrs) + gc_fielddescrs, all_fielddescrs) gccache.init_size_descr(STRUCT, sizedescr) cache[STRUCT] = sizedescr return sizedescr diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py --- a/rpython/jit/codewriter/heaptracker.py +++ b/rpython/jit/codewriter/heaptracker.py @@ -136,9 +136,12 @@ vtable = llmemory.cast_ptr_to_adr(vtable) return adr2int(vtable) -def fielddescrs_from_struct(gccache, STRUCT, only_gc=False, res=None): +def all_fielddescrs(gccache, STRUCT, only_gc=False, res=None, + get_field_descr=None): from rpython.jit.backend.llsupport import descr + if get_field_descr is None: + get_field_descr = descr.get_field_descr if res is None: res = [] # order is not relevant, except for tests @@ -146,11 +149,31 @@ FIELD = getattr(STRUCT, name) if FIELD is lltype.Void: continue + if name == 'typeptr': + continue # dealt otherwise elif isinstance(FIELD, lltype.Struct): - fielddescrs_from_struct(gccache, FIELD, only_gc, res) + all_fielddescrs(gccache, FIELD, only_gc, res, get_field_descr) elif (not only_gc) or (isinstance(FIELD, lltype.Ptr) and FIELD._needsgc()): - res.append(descr.get_field_descr(gccache, STRUCT, name)) + res.append(get_field_descr(gccache, STRUCT, name)) return res def gc_fielddescrs(gccache, STRUCT): - return fielddescrs_from_struct(gccache, STRUCT, True) + return all_fielddescrs(gccache, STRUCT, True) + +def get_fielddescr_index_in(STRUCT, fieldname, cur_index=0): + for name in STRUCT._names: + FIELD = getattr(STRUCT, name) + if FIELD is lltype.Void: + continue + if name == 'typeptr': + continue # dealt otherwise + elif isinstance(FIELD, lltype.Struct): + r = get_fielddescr_index_in(FIELD, fieldname, cur_index) + if r != -1: + return r + continue + elif name == fieldname: + return cur_index + cur_index += 1 + return -1 # not found + diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -478,6 +478,8 @@ return pendingfields def optimize_GETFIELD_GC_I(self, op): + self.emit_operation(op) + return structvalue = self.getvalue(op.getarg(0)) cf = self.field_cache(op.getdescr()) fieldvalue = cf.getfield_from_cache(self, structvalue) diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py --- a/rpython/jit/metainterp/optimizeopt/info.py +++ b/rpython/jit/metainterp/optimizeopt/info.py @@ -1,5 +1,6 @@ -from rpython.jit.metainterp.resoperation import AbstractValue +from rpython.jit.metainterp.resoperation import AbstractValue, ResOperation,\ + rop """ The tag field on PtrOptInfo has a following meaning: @@ -52,14 +53,9 @@ def is_nonnull(self): return True - -class InstancePtrInfo(NonNullPtrInfo): - _attrs_ = ('_known_class', '_is_virtual', '_fields') - _fields = None - def __init__(self, known_class=None, is_virtual=False): - self._known_class = known_class - self._is_virtual = is_virtual +class AbstractStructPtrInfo(NonNullPtrInfo): + _attrs_ = ('_is_virtual', '_fields') def force_box(self, op, optforce): if self._is_virtual: @@ -69,20 +65,38 @@ op.set_forwarded(newop) newop.set_forwarded(self) self._is_virtual = False + if self._fields is not None: + descr = op.getdescr() + for i, flddescr in enumerate(descr.all_fielddescrs): + fld = self._fields[i] + if fld is not None: + subbox = optforce.force_box(fld) + op = ResOperation(rop.SETFIELD_GC, [op, subbox], + descr=flddescr) + optforce.emit_operation(op) return newop return op + def init_fields(self, descr): + self._fields = [None] * len(descr.all_fielddescrs) + def setfield_virtual(self, descr, op): - if self._fields is None: - self._fields = {} - self._fields[descr] = op + self._fields[descr.index] = op def getfield_virtual(self, descr): - return self._fields.get(descr, None) + return self._fields[descr.index] def is_virtual(self): return self._is_virtual +class InstancePtrInfo(AbstractStructPtrInfo): + _attrs_ = ('_known_class') + _fields = None + + def __init__(self, known_class=None, is_virtual=False): + self._known_class = known_class + self._is_virtual = is_virtual + def get_known_class(self, cpu): return self._known_class diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py --- a/rpython/jit/metainterp/optimizeopt/intbounds.py +++ b/rpython/jit/metainterp/optimizeopt/intbounds.py @@ -441,12 +441,12 @@ self.propagate_bounds_backward(box2) def make_int_le(self, box1, box2): - v1 = self.getvalue(box1) - v2 = self.getvalue(box2) - if v1.getintbound().make_le(v2.getintbound()): - self.propagate_bounds_backward(box1, v1) - if v2.getintbound().make_ge(v1.getintbound()): - self.propagate_bounds_backward(box2, v2) + b1 = self.getintbound(box1) + b2 = self.getintbound(box2) + if b1.make_le(b2): + self.propagate_bounds_backward(box1) + if b2.make_ge(b1): + self.propagate_bounds_backward(box2) def make_int_gt(self, box1, box2): self.make_int_lt(box2, box1) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -199,14 +199,14 @@ def test_remove_guard_class_2(self): ops = """ [i0] - p0 = new_with_vtable(ConstClass(node_vtable)) + p0 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) escape_n(p0) guard_class(p0, ConstClass(node_vtable)) [] jump(i0) """ expected = """ [i0] - p0 = new_with_vtable(ConstClass(node_vtable)) + p0 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) escape_n(p0) jump(i0) """ @@ -426,7 +426,7 @@ def test_ooisnull_oononnull_via_virtual(self): ops = """ [p0] - pv = new_with_vtable(ConstClass(node_vtable)) + pv = new_with_vtable(ConstClass(node_vtable), descr=nodesize) setfield_gc(pv, p0, descr=valuedescr) guard_nonnull(p0) [] p1 = getfield_gc_r(pv, descr=valuedescr) @@ -575,7 +575,7 @@ [i1, p2, p3] i3 = getfield_gc_i(p3, descr=valuedescr) escape_n(i3) - p1 = new_with_vtable(ConstClass(node_vtable)) + p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) setfield_gc(p1, i1, descr=valuedescr) jump(i1, p1, p2) """ diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -524,6 +524,7 @@ def make_virtual(self, known_class, source_op, descr): opinfo = info.InstancePtrInfo(known_class, is_virtual=True) + opinfo.init_fields(descr) source_op.set_forwarded(opinfo) return opinfo @@ -682,8 +683,7 @@ fieldvalue = self.optimizer.new_const(op.getdescr()) self.make_equal_to(op, fieldop) else: - yyyy - value.ensure_nonnull() + self.make_nonnull(op.getarg(0)) self.emit_operation(op) optimize_GETFIELD_GC_R = optimize_GETFIELD_GC_I optimize_GETFIELD_GC_F = optimize_GETFIELD_GC_I diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -719,7 +719,7 @@ 'GETFIELD_RAW/1d/fi', '_MALLOC_FIRST', 'NEW/0d/r', #-> GcStruct, gcptrs inside are zeroed (not the rest) - 'NEW_WITH_VTABLE/1/r',#-> GcStruct with vtable, gcptrs inside are zeroed + 'NEW_WITH_VTABLE/1d/r',#-> GcStruct with vtable, gcptrs inside are zeroed 'NEW_ARRAY/1d/r', #-> GcArray, not zeroed. only for arrays of primitives 'NEW_ARRAY_CLEAR/1d/r',#-> GcArray, fully zeroed 'NEWSTR/1/r', #-> STR, the hash field is zeroed From noreply at buildbot.pypy.org Sat Mar 7 13:17:04 2015 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 7 Mar 2015 13:17:04 +0100 (CET) Subject: [pypy-commit] pypy optresult: due to lack of heapcache for now, shuffle the operation order Message-ID: <20150307121704.59A7F1C052B@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult Changeset: r76270:d0dc3f49b00b Date: 2015-03-07 14:14 +0200 http://bitbucket.org/pypy/pypy/changeset/d0dc3f49b00b/ Log: due to lack of heapcache for now, shuffle the operation order diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py --- a/rpython/jit/metainterp/optimizeopt/info.py +++ b/rpython/jit/metainterp/optimizeopt/info.py @@ -71,9 +71,9 @@ fld = self._fields[i] if fld is not None: subbox = optforce.force_box(fld) - op = ResOperation(rop.SETFIELD_GC, [op, subbox], - descr=flddescr) - optforce.emit_operation(op) + setfieldop = ResOperation(rop.SETFIELD_GC, [op, subbox], + descr=flddescr) + optforce.emit_operation(setfieldop) return newop return op diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -587,10 +587,10 @@ [i1, p2, p3] i3 = getfield_gc_i(p3, descr=valuedescr) escape_n(i3) - p1 = new_with_vtable(ConstClass(node_vtable)) - p1sub = new_with_vtable(ConstClass(node_vtable2)) + p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + setfield_gc(p1, i1, descr=valuedescr) + p1sub = new_with_vtable(ConstClass(node_vtable2), descr=nodesize2) setfield_gc(p1sub, i1, descr=valuedescr) - setfield_gc(p1, i1, descr=valuedescr) setfield_gc(p1, p1sub, descr=nextdescr) jump(i1, p1, p2) """ @@ -604,10 +604,10 @@ p3sub = getfield_gc_r(p3, descr=nextdescr) i3 = getfield_gc_i(p3sub, descr=valuedescr) escape_n(i3) - p1 = new_with_vtable(ConstClass(node_vtable)) - p2sub = new_with_vtable(ConstClass(node_vtable2)) + p2sub = new_with_vtable(ConstClass(node_vtable2), descr=nodesize2) setfield_gc(p2sub, i1, descr=valuedescr) setfield_gc(p2, p2sub, descr=nextdescr) + p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) jump(i1, p1, p2) """ # The same as test_p123_simple, but in the end the "old" p2 contains From noreply at buildbot.pypy.org Sat Mar 7 13:17:05 2015 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 7 Mar 2015 13:17:05 +0100 (CET) Subject: [pypy-commit] pypy optresult: fix tests until varray appears Message-ID: <20150307121705.780C61C052B@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult Changeset: r76271:1029118d258f Date: 2015-03-07 14:16 +0200 http://bitbucket.org/pypy/pypy/changeset/1029118d258f/ Log: fix tests until varray appears diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py --- a/rpython/jit/metainterp/optimizeopt/pure.py +++ b/rpython/jit/metainterp/optimizeopt/pure.py @@ -1,5 +1,5 @@ from rpython.jit.metainterp.optimizeopt.optimizer import Optimization, REMOVED -from rpython.jit.metainterp.resoperation import rop, ResOperation +from rpython.jit.metainterp.resoperation import rop, OpHelpers from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method @@ -158,8 +158,7 @@ # all identical # this removes a CALL_PURE that has the same (non-constant) # arguments as a previous CALL_PURE. - oldvalue = self.getvalue(old_op.result) - self.make_equal_to(op.result, oldvalue) + self.make_equal_to(op, old_op) self.last_emitted_operation = REMOVED return True return False diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -408,9 +408,8 @@ self.loop_invariant_producer[key] = op newop = self.replace_op_with(op, OpHelpers.call_for_descr(op.getdescr())) - resvalue = self.optimizer.getvalue(op) self.emit_operation(newop) - self.loop_invariant_results[key] = resvalue + self.loop_invariant_results[key] = op optimize_CALL_LOOPINVARIANT_R = optimize_CALL_LOOPINVARIANT_I optimize_CALL_LOOPINVARIANT_F = optimize_CALL_LOOPINVARIANT_I optimize_CALL_LOOPINVARIANT_N = optimize_CALL_LOOPINVARIANT_I diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -829,7 +829,7 @@ def test_virtual_3(self): ops = """ [i] - p1 = new_with_vtable(ConstClass(node_vtable)) + p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) setfield_gc(p1, i, descr=valuedescr) i0 = getfield_gc_i(p1, descr=valuedescr) i1 = int_add(i0, 1) @@ -894,7 +894,7 @@ def test_virtual_constant_isnull(self): ops = """ [i0] - p0 = new_with_vtable(ConstClass(node_vtable)) + p0 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) setfield_gc(p0, NULL, descr=nextdescr) p2 = getfield_gc_r(p0, descr=nextdescr) i1 = ptr_eq(p2, NULL) @@ -909,7 +909,7 @@ def test_virtual_constant_isnonnull(self): ops = """ [i0] - p0 = new_with_vtable(ConstClass(node_vtable)) + p0 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) setfield_gc(p0, ConstPtr(myptr), descr=nextdescr) p2 = getfield_gc_r(p0, descr=nextdescr) i1 = ptr_eq(p2, NULL) From noreply at buildbot.pypy.org Sat Mar 7 14:52:34 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 7 Mar 2015 14:52:34 +0100 (CET) Subject: [pypy-commit] pypy default: Add a test checking we get the exact same result as CPython Message-ID: <20150307135234.7D1F41C13DA@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76272:ba1ed3805ad7 Date: 2015-03-07 14:52 +0100 http://bitbucket.org/pypy/pypy/changeset/ba1ed3805ad7/ Log: Add a test checking we get the exact same result as CPython diff --git a/pypy/module/_random/test/test_random.py b/pypy/module/_random/test/test_random.py --- a/pypy/module/_random/test/test_random.py +++ b/pypy/module/_random/test/test_random.py @@ -102,3 +102,10 @@ self.x = x r = R(x=15) assert r.x == 15 + + def test_exact_result(self): + # this passes on CPython 2.7.9 on Linux 32 and Linux 64 + import _random + rnd = _random.Random(-3**31) + x = rnd.random() + assert x == 0.8181851342382107 From noreply at buildbot.pypy.org Sat Mar 7 15:02:43 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 7 Mar 2015 15:02:43 +0100 (CET) Subject: [pypy-commit] pypy optresult: hg merge recent-pure-ops (again) Message-ID: <20150307140243.E1A0B1C14C1@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: optresult Changeset: r76273:1188016be42b Date: 2015-03-07 15:02 +0100 http://bitbucket.org/pypy/pypy/changeset/1188016be42b/ Log: hg merge recent-pure-ops (again) diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -837,6 +837,13 @@ if debug_print: print '%30s = %d' % (name, i) i += 1 + # for optimizeopt/pure.py's getrecentops() + assert (rop.INT_ADD_OVF - rop._OVF_FIRST == + rop.INT_ADD - rop._ALWAYS_PURE_FIRST) + assert (rop.INT_SUB_OVF - rop._OVF_FIRST == + rop.INT_SUB - rop._ALWAYS_PURE_FIRST) + assert (rop.INT_MUL_OVF - rop._OVF_FIRST == + rop.INT_MUL - rop._ALWAYS_PURE_FIRST) def get_base_class(mixins, base): try: From noreply at buildbot.pypy.org Sat Mar 7 15:02:45 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 7 Mar 2015 15:02:45 +0100 (CET) Subject: [pypy-commit] pypy optresult: merge heads Message-ID: <20150307140245.1EA591C14C1@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: optresult Changeset: r76274:e25c349f8ae6 Date: 2015-03-07 15:02 +0100 http://bitbucket.org/pypy/pypy/changeset/e25c349f8ae6/ Log: merge heads diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py --- a/rpython/jit/backend/llgraph/runner.py +++ b/rpython/jit/backend/llgraph/runner.py @@ -89,8 +89,10 @@ return getkind(self.RESULT)[0] class SizeDescr(AbstractDescr): - def __init__(self, S): + def __init__(self, S, runner): self.S = S + self.all_fielddescrs = heaptracker.all_fielddescrs(runner, S, + get_field_descr=LLGraphCPU.fielddescrof) def as_vtable_size_descr(self): return self @@ -106,6 +108,7 @@ self.S = S self.fieldname = fieldname self.FIELD = getattr(S, fieldname) + self.index = heaptracker.get_fielddescr_index_in(S, fieldname) def get_vinfo(self): return self.vinfo @@ -374,7 +377,7 @@ try: return self.descrs[key] except KeyError: - descr = SizeDescr(S) + descr = SizeDescr(S, self) self.descrs[key] = descr return descr diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py --- a/rpython/jit/backend/llsupport/descr.py +++ b/rpython/jit/backend/llsupport/descr.py @@ -36,10 +36,11 @@ tid = llop.combine_ushort(lltype.Signed, 0, 0) def __init__(self, size, count_fields_if_immut=-1, - gc_fielddescrs=None): + gc_fielddescrs=None, all_fielddescrs=None): self.size = size self.count_fields_if_immut = count_fields_if_immut self.gc_fielddescrs = gc_fielddescrs + self.all_fielddescrs = all_fielddescrs def count_fields_if_immutable(self): return self.count_fields_if_immut @@ -61,12 +62,13 @@ size = symbolic.get_size(STRUCT, gccache.translate_support_code) count_fields_if_immut = heaptracker.count_fields_if_immutable(STRUCT) gc_fielddescrs = heaptracker.gc_fielddescrs(gccache, STRUCT) + all_fielddescrs = heaptracker.all_fielddescrs(gccache, STRUCT) if heaptracker.has_gcstruct_a_vtable(STRUCT): sizedescr = SizeDescrWithVTable(size, count_fields_if_immut, - gc_fielddescrs) + gc_fielddescrs, all_fielddescrs) else: sizedescr = SizeDescr(size, count_fields_if_immut, - gc_fielddescrs) + gc_fielddescrs, all_fielddescrs) gccache.init_size_descr(STRUCT, sizedescr) cache[STRUCT] = sizedescr return sizedescr diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py --- a/rpython/jit/codewriter/heaptracker.py +++ b/rpython/jit/codewriter/heaptracker.py @@ -136,9 +136,12 @@ vtable = llmemory.cast_ptr_to_adr(vtable) return adr2int(vtable) -def fielddescrs_from_struct(gccache, STRUCT, only_gc=False, res=None): +def all_fielddescrs(gccache, STRUCT, only_gc=False, res=None, + get_field_descr=None): from rpython.jit.backend.llsupport import descr + if get_field_descr is None: + get_field_descr = descr.get_field_descr if res is None: res = [] # order is not relevant, except for tests @@ -146,11 +149,31 @@ FIELD = getattr(STRUCT, name) if FIELD is lltype.Void: continue + if name == 'typeptr': + continue # dealt otherwise elif isinstance(FIELD, lltype.Struct): - fielddescrs_from_struct(gccache, FIELD, only_gc, res) + all_fielddescrs(gccache, FIELD, only_gc, res, get_field_descr) elif (not only_gc) or (isinstance(FIELD, lltype.Ptr) and FIELD._needsgc()): - res.append(descr.get_field_descr(gccache, STRUCT, name)) + res.append(get_field_descr(gccache, STRUCT, name)) return res def gc_fielddescrs(gccache, STRUCT): - return fielddescrs_from_struct(gccache, STRUCT, True) + return all_fielddescrs(gccache, STRUCT, True) + +def get_fielddescr_index_in(STRUCT, fieldname, cur_index=0): + for name in STRUCT._names: + FIELD = getattr(STRUCT, name) + if FIELD is lltype.Void: + continue + if name == 'typeptr': + continue # dealt otherwise + elif isinstance(FIELD, lltype.Struct): + r = get_fielddescr_index_in(FIELD, fieldname, cur_index) + if r != -1: + return r + continue + elif name == fieldname: + return cur_index + cur_index += 1 + return -1 # not found + diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -478,6 +478,8 @@ return pendingfields def optimize_GETFIELD_GC_I(self, op): + self.emit_operation(op) + return structvalue = self.getvalue(op.getarg(0)) cf = self.field_cache(op.getdescr()) fieldvalue = cf.getfield_from_cache(self, structvalue) diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py --- a/rpython/jit/metainterp/optimizeopt/info.py +++ b/rpython/jit/metainterp/optimizeopt/info.py @@ -1,5 +1,6 @@ -from rpython.jit.metainterp.resoperation import AbstractValue +from rpython.jit.metainterp.resoperation import AbstractValue, ResOperation,\ + rop """ The tag field on PtrOptInfo has a following meaning: @@ -52,14 +53,9 @@ def is_nonnull(self): return True - -class InstancePtrInfo(NonNullPtrInfo): - _attrs_ = ('_known_class', '_is_virtual', '_fields') - _fields = None - def __init__(self, known_class=None, is_virtual=False): - self._known_class = known_class - self._is_virtual = is_virtual +class AbstractStructPtrInfo(NonNullPtrInfo): + _attrs_ = ('_is_virtual', '_fields') def force_box(self, op, optforce): if self._is_virtual: @@ -69,20 +65,38 @@ op.set_forwarded(newop) newop.set_forwarded(self) self._is_virtual = False + if self._fields is not None: + descr = op.getdescr() + for i, flddescr in enumerate(descr.all_fielddescrs): + fld = self._fields[i] + if fld is not None: + subbox = optforce.force_box(fld) + setfieldop = ResOperation(rop.SETFIELD_GC, [op, subbox], + descr=flddescr) + optforce.emit_operation(setfieldop) return newop return op + def init_fields(self, descr): + self._fields = [None] * len(descr.all_fielddescrs) + def setfield_virtual(self, descr, op): - if self._fields is None: - self._fields = {} - self._fields[descr] = op + self._fields[descr.index] = op def getfield_virtual(self, descr): - return self._fields.get(descr, None) + return self._fields[descr.index] def is_virtual(self): return self._is_virtual +class InstancePtrInfo(AbstractStructPtrInfo): + _attrs_ = ('_known_class') + _fields = None + + def __init__(self, known_class=None, is_virtual=False): + self._known_class = known_class + self._is_virtual = is_virtual + def get_known_class(self, cpu): return self._known_class diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py --- a/rpython/jit/metainterp/optimizeopt/intbounds.py +++ b/rpython/jit/metainterp/optimizeopt/intbounds.py @@ -441,12 +441,12 @@ self.propagate_bounds_backward(box2) def make_int_le(self, box1, box2): - v1 = self.getvalue(box1) - v2 = self.getvalue(box2) - if v1.getintbound().make_le(v2.getintbound()): - self.propagate_bounds_backward(box1, v1) - if v2.getintbound().make_ge(v1.getintbound()): - self.propagate_bounds_backward(box2, v2) + b1 = self.getintbound(box1) + b2 = self.getintbound(box2) + if b1.make_le(b2): + self.propagate_bounds_backward(box1) + if b2.make_ge(b1): + self.propagate_bounds_backward(box2) def make_int_gt(self, box1, box2): self.make_int_lt(box2, box1) diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -692,26 +692,25 @@ raise compile.giveup() descr.store_final_boxes(op, newboxes, self.metainterp_sd) # - if op.getopnum() == rop.GUARD_VALUE and 0: # XXX - val = self.getvalue(op.getarg(0)) - if val in self.bool_boxes: - # Hack: turn guard_value(bool) into guard_true/guard_false. - # This is done after the operation is emitted to let - # store_final_boxes_in_guard set the guard_opnum field of the - # descr to the original rop.GUARD_VALUE. - constvalue = op.getarg(1).getint() - if constvalue == 0: - opnum = rop.GUARD_FALSE - elif constvalue == 1: - opnum = rop.GUARD_TRUE - else: - raise AssertionError("uh?") - newop = self.replace_op_with(op, opnum, [op.getarg(0)], descr) - newop.setfailargs(op.getfailargs()) - return newop - else: - # a real GUARD_VALUE. Make it use one counter per value. - descr.make_a_counter_per_value(op) + if op.getopnum() == rop.GUARD_VALUE: + if op.getarg(0).type == 'i': + b = self.getintbound(op.getarg(0)) + if b.is_bool(): + # Hack: turn guard_value(bool) into guard_true/guard_false. + # This is done after the operation is emitted to let + # store_final_boxes_in_guard set the guard_opnum field of + # the descr to the original rop.GUARD_VALUE. + constvalue = op.getarg(1).getint() + if constvalue == 0: + opnum = rop.GUARD_FALSE + elif constvalue == 1: + opnum = rop.GUARD_TRUE + else: + raise AssertionError("uh?") + newop = self.replace_op_with(op, opnum, [op.getarg(0)], descr) + return newop + # a real GUARD_VALUE. Make it use one counter per value. + descr.make_a_counter_per_value(op) return op def optimize_default(self, op): diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py --- a/rpython/jit/metainterp/optimizeopt/pure.py +++ b/rpython/jit/metainterp/optimizeopt/pure.py @@ -1,5 +1,5 @@ from rpython.jit.metainterp.optimizeopt.optimizer import Optimization, REMOVED -from rpython.jit.metainterp.resoperation import rop, ResOperation +from rpython.jit.metainterp.resoperation import rop, OpHelpers from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method @@ -158,8 +158,7 @@ # all identical # this removes a CALL_PURE that has the same (non-constant) # arguments as a previous CALL_PURE. - oldvalue = self.getvalue(old_op.result) - self.make_equal_to(op.result, oldvalue) + self.make_equal_to(op, old_op) self.last_emitted_operation = REMOVED return True return False diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -408,9 +408,8 @@ self.loop_invariant_producer[key] = op newop = self.replace_op_with(op, OpHelpers.call_for_descr(op.getdescr())) - resvalue = self.optimizer.getvalue(op) self.emit_operation(newop) - self.loop_invariant_results[key] = resvalue + self.loop_invariant_results[key] = op optimize_CALL_LOOPINVARIANT_R = optimize_CALL_LOOPINVARIANT_I optimize_CALL_LOOPINVARIANT_F = optimize_CALL_LOOPINVARIANT_I optimize_CALL_LOOPINVARIANT_N = optimize_CALL_LOOPINVARIANT_I diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -199,14 +199,14 @@ def test_remove_guard_class_2(self): ops = """ [i0] - p0 = new_with_vtable(ConstClass(node_vtable)) + p0 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) escape_n(p0) guard_class(p0, ConstClass(node_vtable)) [] jump(i0) """ expected = """ [i0] - p0 = new_with_vtable(ConstClass(node_vtable)) + p0 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) escape_n(p0) jump(i0) """ @@ -426,7 +426,7 @@ def test_ooisnull_oononnull_via_virtual(self): ops = """ [p0] - pv = new_with_vtable(ConstClass(node_vtable)) + pv = new_with_vtable(ConstClass(node_vtable), descr=nodesize) setfield_gc(pv, p0, descr=valuedescr) guard_nonnull(p0) [] p1 = getfield_gc_r(pv, descr=valuedescr) @@ -575,7 +575,7 @@ [i1, p2, p3] i3 = getfield_gc_i(p3, descr=valuedescr) escape_n(i3) - p1 = new_with_vtable(ConstClass(node_vtable)) + p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) setfield_gc(p1, i1, descr=valuedescr) jump(i1, p1, p2) """ @@ -587,10 +587,10 @@ [i1, p2, p3] i3 = getfield_gc_i(p3, descr=valuedescr) escape_n(i3) - p1 = new_with_vtable(ConstClass(node_vtable)) - p1sub = new_with_vtable(ConstClass(node_vtable2)) + p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + setfield_gc(p1, i1, descr=valuedescr) + p1sub = new_with_vtable(ConstClass(node_vtable2), descr=nodesize2) setfield_gc(p1sub, i1, descr=valuedescr) - setfield_gc(p1, i1, descr=valuedescr) setfield_gc(p1, p1sub, descr=nextdescr) jump(i1, p1, p2) """ @@ -604,10 +604,10 @@ p3sub = getfield_gc_r(p3, descr=nextdescr) i3 = getfield_gc_i(p3sub, descr=valuedescr) escape_n(i3) - p1 = new_with_vtable(ConstClass(node_vtable)) - p2sub = new_with_vtable(ConstClass(node_vtable2)) + p2sub = new_with_vtable(ConstClass(node_vtable2), descr=nodesize2) setfield_gc(p2sub, i1, descr=valuedescr) setfield_gc(p2, p2sub, descr=nextdescr) + p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) jump(i1, p1, p2) """ # The same as test_p123_simple, but in the end the "old" p2 contains @@ -829,7 +829,7 @@ def test_virtual_3(self): ops = """ [i] - p1 = new_with_vtable(ConstClass(node_vtable)) + p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) setfield_gc(p1, i, descr=valuedescr) i0 = getfield_gc_i(p1, descr=valuedescr) i1 = int_add(i0, 1) @@ -894,7 +894,7 @@ def test_virtual_constant_isnull(self): ops = """ [i0] - p0 = new_with_vtable(ConstClass(node_vtable)) + p0 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) setfield_gc(p0, NULL, descr=nextdescr) p2 = getfield_gc_r(p0, descr=nextdescr) i1 = ptr_eq(p2, NULL) @@ -909,7 +909,7 @@ def test_virtual_constant_isnonnull(self): ops = """ [i0] - p0 = new_with_vtable(ConstClass(node_vtable)) + p0 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) setfield_gc(p0, ConstPtr(myptr), descr=nextdescr) p2 = getfield_gc_r(p0, descr=nextdescr) i1 = ptr_eq(p2, NULL) diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -524,6 +524,7 @@ def make_virtual(self, known_class, source_op, descr): opinfo = info.InstancePtrInfo(known_class, is_virtual=True) + opinfo.init_fields(descr) source_op.set_forwarded(opinfo) return opinfo @@ -682,8 +683,7 @@ fieldvalue = self.optimizer.new_const(op.getdescr()) self.make_equal_to(op, fieldop) else: - yyyy - value.ensure_nonnull() + self.make_nonnull(op.getarg(0)) self.emit_operation(op) optimize_GETFIELD_GC_R = optimize_GETFIELD_GC_I optimize_GETFIELD_GC_F = optimize_GETFIELD_GC_I diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -719,7 +719,7 @@ 'GETFIELD_RAW/1d/fi', '_MALLOC_FIRST', 'NEW/0d/r', #-> GcStruct, gcptrs inside are zeroed (not the rest) - 'NEW_WITH_VTABLE/1/r',#-> GcStruct with vtable, gcptrs inside are zeroed + 'NEW_WITH_VTABLE/1d/r',#-> GcStruct with vtable, gcptrs inside are zeroed 'NEW_ARRAY/1d/r', #-> GcArray, not zeroed. only for arrays of primitives 'NEW_ARRAY_CLEAR/1d/r',#-> GcArray, fully zeroed 'NEWSTR/1/r', #-> STR, the hash field is zeroed From noreply at buildbot.pypy.org Sat Mar 7 15:03:53 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 7 Mar 2015 15:03:53 +0100 (CET) Subject: [pypy-commit] pypy.org extradoc: update the values Message-ID: <20150307140353.035151C14C1@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r577:17524cdd9f57 Date: 2015-03-07 15:04 +0100 http://bitbucket.org/pypy/pypy.org/changeset/17524cdd9f57/ Log: update the values diff --git a/don3.html b/don3.html --- a/don3.html +++ b/don3.html @@ -15,7 +15,7 @@ - $51321 of $60000 (85.5%) + $51326 of $60000 (85.5%)
From noreply at buildbot.pypy.org Sat Mar 7 15:07:58 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 7 Mar 2015 15:07:58 +0100 (CET) Subject: [pypy-commit] stmgc c8-card-marking: Duh, abort() can be called from a signal handler. Message-ID: <20150307140758.5DAC61C14C0@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-card-marking Changeset: r1690:f52d5033cce6 Date: 2015-03-07 15:08 +0100 http://bitbucket.org/pypy/stmgc/changeset/f52d5033cce6/ Log: Duh, abort() can be called from a signal handler. diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -228,8 +228,7 @@ addr >= stm_object_pages+TOTAL_MEMORY) { /* actual segfault, unrelated to stmgc */ fprintf(stderr, "Segmentation fault: accessing %p\n", addr); - raise(SIGABRT); - raise(SIGKILL); + abort(); } int segnum = get_segment_of_linear_address(addr); @@ -237,8 +236,7 @@ if (segnum != STM_SEGMENT->segment_num) { fprintf(stderr, "Segmentation fault: accessing %p (seg %d) from" " seg %d\n", addr, segnum, STM_SEGMENT->segment_num); - raise(SIGABRT); - raise(SIGKILL); + abort(); } dprintf(("-> segment: %d\n", segnum)); @@ -247,8 +245,7 @@ if (pagenum < END_NURSERY_PAGE) { fprintf(stderr, "Segmentation fault: accessing %p (seg %d " "page %lu)\n", addr, segnum, pagenum); - raise(SIGABRT); - raise(SIGKILL); + abort(); } DEBUG_EXPECT_SEGFAULT(false); From noreply at buildbot.pypy.org Sat Mar 7 15:27:37 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 7 Mar 2015 15:27:37 +0100 (CET) Subject: [pypy-commit] stmgc c8-card-marking: Close branch, which seems to be ready to merge Message-ID: <20150307142737.C7DC81C023F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-card-marking Changeset: r1691:3da2ffce635d Date: 2015-03-07 15:09 +0100 http://bitbucket.org/pypy/stmgc/changeset/3da2ffce635d/ Log: Close branch, which seems to be ready to merge From noreply at buildbot.pypy.org Sat Mar 7 15:27:39 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 7 Mar 2015 15:27:39 +0100 (CET) Subject: [pypy-commit] stmgc default: hg merge c8-card-marking Message-ID: <20150307142739.464801C023F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1692:21554aee8ae6 Date: 2015-03-07 15:09 +0100 http://bitbucket.org/pypy/stmgc/changeset/21554aee8ae6/ Log: hg merge c8-card-marking diff too long, truncating to 2000 out of 2709 lines diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -228,7 +228,7 @@ addr >= stm_object_pages+TOTAL_MEMORY) { /* actual segfault, unrelated to stmgc */ fprintf(stderr, "Segmentation fault: accessing %p\n", addr); - raise(SIGINT); + abort(); } int segnum = get_segment_of_linear_address(addr); @@ -236,7 +236,7 @@ if (segnum != STM_SEGMENT->segment_num) { fprintf(stderr, "Segmentation fault: accessing %p (seg %d) from" " seg %d\n", addr, segnum, STM_SEGMENT->segment_num); - raise(SIGINT); + abort(); } dprintf(("-> segment: %d\n", segnum)); @@ -245,7 +245,7 @@ if (pagenum < END_NURSERY_PAGE) { fprintf(stderr, "Segmentation fault: accessing %p (seg %d " "page %lu)\n", addr, segnum, pagenum); - raise(SIGINT); + abort(); } DEBUG_EXPECT_SEGFAULT(false); @@ -576,118 +576,422 @@ } -void _stm_write_slowpath(object_t *obj) +bool obj_should_use_cards(char *seg_base, object_t *obj) { - assert(_seems_to_be_running_transaction()); - assert(!_is_in_nursery(obj)); - assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); + if (is_small_uniform(obj)) + return false; - int my_segnum = STM_SEGMENT->segment_num; - uintptr_t end_page, first_page = ((uintptr_t)obj) / 4096UL; - char *realobj; - size_t obj_size; + struct object_s *realobj = (struct object_s *) + REAL_ADDRESS(seg_base, obj); + long supports = stmcb_obj_supports_cards(realobj); + if (!supports) + return false; - realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); - obj_size = stmcb_size_rounded_up((struct object_s *)realobj); - /* get the last page containing data from the object */ - if (LIKELY(is_small_uniform(obj))) { - end_page = first_page; - } else { - end_page = (((uintptr_t)obj) + obj_size - 1) / 4096UL; - } + /* check also if it makes sense: */ + size_t size = stmcb_size_rounded_up(realobj); + return (size >= _STM_MIN_CARD_OBJ_SIZE); +} - /* add to read set: */ - stm_read(obj); - if (obj->stm_flags & GCFLAG_WB_EXECUTED) { - /* already executed WB once in this transaction. do GC - part again: */ - dprintf(("write_slowpath-fast(%p)\n", obj)); - obj->stm_flags &= ~GCFLAG_WRITE_BARRIER; - LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj); - return; - } - - assert(!(obj->stm_flags & GCFLAG_WB_EXECUTED)); - dprintf(("write_slowpath(%p): sz=%lu\n", obj, obj_size)); - - retry: - /* privatize pages: */ - /* XXX don't always acquire all locks... */ - acquire_all_privatization_locks(); +static void make_bk_slices_for_range( + object_t *obj, + stm_char *start, stm_char *end) /* [start, end[ */ +{ + dprintf(("make_bk_slices_for_range(%p, %lu, %lu)\n", + obj, start - (stm_char*)obj, end - start)); + char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); + uintptr_t first_page = ((uintptr_t)start) / 4096UL; + uintptr_t end_page = ((uintptr_t)end) / 4096UL; uintptr_t page; - for (page = first_page; page <= end_page; page++) { - if (get_page_status_in(my_segnum, page) == PAGE_NO_ACCESS) { - /* XXX: slow? */ - release_all_privatization_locks(); - - volatile char *dummy = REAL_ADDRESS(STM_SEGMENT->segment_base, page * 4096UL); - *dummy; /* force segfault */ - - goto retry; - } - } - /* all pages are private to us and we hold the privatization_locks so - we are allowed to modify them */ - - /* phew, now add the obj to the write-set and register the - backup copy. */ - /* XXX: we should not be here at all fiddling with page status - if 'obj' is merely an overflow object. FIX ME, likely by copying - the overflow number logic from c7. */ - - DEBUG_EXPECT_SEGFAULT(false); - - acquire_modification_lock(STM_SEGMENT->segment_num); uintptr_t slice_sz; - uintptr_t in_page_offset = (uintptr_t)obj % 4096UL; - uintptr_t remaining_obj_sz = obj_size; - for (page = first_page; page <= end_page; page++) { - /* XXX Maybe also use mprotect() again to mark pages of the object as read-only, and - only stick it into modified_old_objects page-by-page? Maybe it's - possible to do card-marking that way, too. */ - OPT_ASSERT(remaining_obj_sz); - + uintptr_t slice_off = start - (stm_char*)obj; + uintptr_t in_page_offset = (uintptr_t)start % 4096UL; + uintptr_t remaining_obj_sz = end - start; + for (page = first_page; page <= end_page && remaining_obj_sz; page++) { slice_sz = remaining_obj_sz; if (in_page_offset + slice_sz > 4096UL) { /* not over page boundaries */ slice_sz = 4096UL - in_page_offset; } - size_t slice_off = obj_size - remaining_obj_sz; + remaining_obj_sz -= slice_sz; + in_page_offset = (in_page_offset + slice_sz) % 4096UL; /* mostly 0 */ /* make backup slice: */ char *bk_slice = malloc(slice_sz); increment_total_allocated(slice_sz); memcpy(bk_slice, realobj + slice_off, slice_sz); + acquire_modification_lock(STM_SEGMENT->segment_num); /* !! follows layout of "struct stm_undo_s" !! */ STM_PSEGMENT->modified_old_objects = list_append3( STM_PSEGMENT->modified_old_objects, (uintptr_t)obj, /* obj */ (uintptr_t)bk_slice, /* bk_addr */ NEW_SLICE(slice_off, slice_sz)); + dprintf(("> append slice %p, off=%lu, sz=%lu\n", bk_slice, slice_off, slice_sz)); + release_modification_lock(STM_SEGMENT->segment_num); - remaining_obj_sz -= slice_sz; - in_page_offset = (in_page_offset + slice_sz) % 4096UL; /* mostly 0 */ + slice_off += slice_sz; } - OPT_ASSERT(remaining_obj_sz == 0); - /* remove the WRITE_BARRIER flag and add WB_EXECUTED */ - obj->stm_flags &= ~GCFLAG_WRITE_BARRIER; - obj->stm_flags |= GCFLAG_WB_EXECUTED; +} + +static void make_bk_slices(object_t *obj, + bool first_call, /* tells us if we also need to make a bk + of the non-array part of the object */ + uintptr_t index, /* index == -1: all cards, index == -2: no cards */ + bool do_missing_cards /* only bk the cards that don't have a bk */ + ) +{ + dprintf(("make_bk_slices(%p, %d, %ld, %d)\n", obj, first_call, index, do_missing_cards)); + /* do_missing_cards also implies that all cards are cleared at the end */ + /* index == -1 but not do_missing_cards: bk whole obj */ + assert(IMPLY(index == -2, first_call && !do_missing_cards)); + assert(IMPLY(index == -1 && !do_missing_cards, first_call)); + assert(IMPLY(do_missing_cards, index == -1)); + assert(IMPLY(is_small_uniform(obj), index == -1 && !do_missing_cards && first_call)); + assert(IMPLY(first_call, !do_missing_cards)); + assert(IMPLY(index != -1, obj_should_use_cards(STM_SEGMENT->segment_base, obj))); + + /* get whole card range */ + struct object_s *realobj = (struct object_s*)REAL_ADDRESS(STM_SEGMENT->segment_base, obj); + size_t obj_size = stmcb_size_rounded_up(realobj); + uintptr_t offset_itemsize[2] = {-1, -1}; + + /* decide where to start copying: */ + size_t start_offset; + if (first_call) { + start_offset = 0; + } else { + start_offset = -1; + } + + /* decide if we don't want to look at cards at all: */ + if ((index == -1 || index == -2) && !do_missing_cards) { + assert(first_call); + if (index == -1) { + /* whole obj */ + make_bk_slices_for_range(obj, (stm_char*)obj + start_offset, + (stm_char*)obj + obj_size); + if (obj_should_use_cards(STM_SEGMENT->segment_base, obj)) { + /* mark whole obj as MARKED_OLD so we don't do bk slices anymore */ + _reset_object_cards(get_priv_segment(STM_SEGMENT->segment_num), + obj, STM_SEGMENT->transaction_read_version, + true, false); + } + } else { + /* only fixed part */ + stmcb_get_card_base_itemsize(realobj, offset_itemsize); + make_bk_slices_for_range(obj, (stm_char*)obj + start_offset, + (stm_char*)obj + offset_itemsize[0]); + } + return; + } + + stmcb_get_card_base_itemsize(realobj, offset_itemsize); + + size_t real_idx_count = (obj_size - offset_itemsize[0]) / offset_itemsize[1]; + assert(IMPLY(index != -1 && index != -2, index >= 0 && index < real_idx_count)); + struct stm_read_marker_s *cards = get_read_marker(STM_SEGMENT->segment_base, (uintptr_t)obj); + uintptr_t last_card_index = get_index_to_card_index(real_idx_count - 1); /* max valid index */ + uintptr_t card_index; + + /* decide if we want only a specific card: */ + if (index != -1) { + if (start_offset != -1) { + /* bk fixed part separately: */ + make_bk_slices_for_range(obj, (stm_char*)obj + start_offset, + (stm_char*)obj + offset_itemsize[0]); + } + + card_index = get_index_to_card_index(index); + + size_t card_offset = offset_itemsize[0] + + get_card_index_to_index(card_index) * offset_itemsize[1]; + size_t after_card_offset = offset_itemsize[0] + + get_card_index_to_index(card_index + 1) * offset_itemsize[1]; + + if (after_card_offset > obj_size) + after_card_offset = obj_size; + + make_bk_slices_for_range( + obj, (stm_char*)obj + card_offset, (stm_char*)obj + after_card_offset); + + return; + } + + /* look for CARD_CLEAR or some non-transaction_read_version cards + and make bk slices for them */ + assert(do_missing_cards && index == -1 && start_offset == -1); + card_index = 1; + uintptr_t start_card_index = -1; + while (card_index <= last_card_index) { + uint8_t card_value = cards[card_index].rm; + + if (card_value == CARD_CLEAR + || (card_value != CARD_MARKED + && card_value < STM_SEGMENT->transaction_read_version)) { + /* we need a backup of this card */ + if (start_card_index == -1) { /* first unmarked card */ + start_card_index = card_index; + } + } else { + /* "CARD_MARKED_OLD" or CARD_MARKED */ + OPT_ASSERT(card_value == STM_SEGMENT->transaction_read_version + || card_value == CARD_MARKED); + } + /* in any case, remember that we already made a bk slice for this + card, so set to "MARKED_OLD": */ + cards[card_index].rm = STM_SEGMENT->transaction_read_version; + + + if (start_card_index != -1 /* something to copy */ + && (card_value == CARD_MARKED /* found marked card */ + || card_value == STM_SEGMENT->transaction_read_version/* old marked */ + || card_index == last_card_index)) { /* this is the last card */ + + /* do the bk slice: */ + uintptr_t copy_size; + uintptr_t next_card_offset; + uintptr_t start_card_offset; + uintptr_t next_card_index = card_index; + + if (card_value == CARD_CLEAR + || (card_value != CARD_MARKED + && card_value < STM_SEGMENT->transaction_read_version)) { + /* this was actually the last card which wasn't set, but we + need to go one further to get the right offset */ + next_card_index++; + } + + start_card_offset = offset_itemsize[0] + + get_card_index_to_index(start_card_index) * offset_itemsize[1]; + + next_card_offset = offset_itemsize[0] + + get_card_index_to_index(next_card_index) * offset_itemsize[1]; + + if (next_card_offset > obj_size) + next_card_offset = obj_size; + + copy_size = next_card_offset - start_card_offset; + OPT_ASSERT(copy_size > 0); + + /* add the slices: */ + make_bk_slices_for_range( + obj, (stm_char*)obj + start_card_offset, + (stm_char*)obj + next_card_offset); + + start_card_index = -1; + } + + card_index++; + } + + obj->stm_flags &= ~GCFLAG_CARDS_SET; + _cards_cleared_in_object(get_priv_segment(STM_SEGMENT->segment_num), obj, false); +} + +__attribute__((always_inline)) +static void write_slowpath_overflow_obj(object_t *obj, bool mark_card) +{ + assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); + assert(!(obj->stm_flags & GCFLAG_WB_EXECUTED)); + dprintf(("write_slowpath_overflow_obj(%p)\n", obj)); + + if (!mark_card) { + /* The basic case, with no card marking. We append the object + into 'objects_pointing_to_nursery', and remove the flag so + that the write_slowpath will not be called again until the + next minor collection. */ + if (obj->stm_flags & GCFLAG_CARDS_SET) { + /* if we clear this flag, we also need to clear the cards. + bk_slices are not needed as this is an overflow object */ + _reset_object_cards(get_priv_segment(STM_SEGMENT->segment_num), + obj, CARD_CLEAR, false, false); + } + obj->stm_flags &= ~(GCFLAG_WRITE_BARRIER | GCFLAG_CARDS_SET); + LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj); + } else { + /* Card marking. Don't remove GCFLAG_WRITE_BARRIER because we + need to come back to _stm_write_slowpath_card() for every + card to mark. Add GCFLAG_CARDS_SET. + again, we don't need bk_slices as this is an overflow obj */ + assert(!(obj->stm_flags & GCFLAG_CARDS_SET)); + obj->stm_flags |= GCFLAG_CARDS_SET; + LIST_APPEND(STM_PSEGMENT->old_objects_with_cards_set, obj); + } +} + + +__attribute__((always_inline)) +static void write_slowpath_common(object_t *obj, bool mark_card) +{ + assert(_seems_to_be_running_transaction()); + assert(!_is_in_nursery(obj)); + assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); + + if (IS_OVERFLOW_OBJ(STM_PSEGMENT, obj)) { + /* already executed WB once in this transaction. do GC + part again: */ + assert(!(obj->stm_flags & GCFLAG_WB_EXECUTED)); + write_slowpath_overflow_obj(obj, mark_card); + return; + } + + dprintf(("write_slowpath(%p)\n", obj)); + + /* add to read set: */ + stm_read(obj); + + if (!(obj->stm_flags & GCFLAG_WB_EXECUTED)) { + /* the first time we write this obj, make sure it is fully + accessible, as major gc may depend on being able to trace + the full obj in this segment (XXX) */ + char *realobj; + size_t obj_size; + int my_segnum = STM_SEGMENT->segment_num; + uintptr_t end_page, first_page = ((uintptr_t)obj) / 4096UL; + + realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); + obj_size = stmcb_size_rounded_up((struct object_s *)realobj); + /* get the last page containing data from the object */ + if (LIKELY(is_small_uniform(obj))) { + end_page = first_page; + } else { + end_page = (((uintptr_t)obj) + obj_size - 1) / 4096UL; + } + + acquire_privatization_lock(STM_SEGMENT->segment_num); + uintptr_t page; + for (page = first_page; page <= end_page; page++) { + if (get_page_status_in(my_segnum, page) == PAGE_NO_ACCESS) { + release_privatization_lock(STM_SEGMENT->segment_num); + volatile char *dummy = REAL_ADDRESS(STM_SEGMENT->segment_base, page * 4096UL); + *dummy; /* force segfault */ + acquire_privatization_lock(STM_SEGMENT->segment_num); + } + } + release_privatization_lock(STM_SEGMENT->segment_num); + } + + if (mark_card) { + if (!(obj->stm_flags & GCFLAG_WB_EXECUTED)) { + make_bk_slices(obj, + true, /* first_call */ + -2, /* index: backup only fixed part */ + false); /* do_missing_cards */ + } + + DEBUG_EXPECT_SEGFAULT(false); + + /* don't remove WRITE_BARRIER, but add CARDS_SET */ + obj->stm_flags |= (GCFLAG_CARDS_SET | GCFLAG_WB_EXECUTED); + LIST_APPEND(STM_PSEGMENT->old_objects_with_cards_set, obj); + } else { + /* called if WB_EXECUTED is set or this is the first time + for this obj: */ + + /* add it to the GC list for minor collections */ + LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj); + + if (obj->stm_flags & GCFLAG_CARDS_SET) { + assert(obj->stm_flags & GCFLAG_WB_EXECUTED); + + /* this is not the first_call to the WB for this obj, + we executed the above then-part before. + if we clear this flag, we have to add all the other + bk slices we didn't add yet */ + make_bk_slices(obj, + false, /* first_call */ + -1, /* index: whole obj */ + true); /* do_missing_cards */ + + } else if (!(obj->stm_flags & GCFLAG_WB_EXECUTED)) { + /* first and only time we enter here: */ + make_bk_slices(obj, + true, /* first_call */ + -1, /* index: whole obj */ + false); /* do_missing_cards */ + } + + DEBUG_EXPECT_SEGFAULT(false); + /* remove the WRITE_BARRIER flag and add WB_EXECUTED */ + obj->stm_flags &= ~(GCFLAG_WRITE_BARRIER | GCFLAG_CARDS_SET); + obj->stm_flags |= GCFLAG_WB_EXECUTED; + } DEBUG_EXPECT_SEGFAULT(true); +} - release_modification_lock(STM_SEGMENT->segment_num); - /* done fiddling with protection and privatization */ - release_all_privatization_locks(); - /* also add it to the GC list for minor collections */ - LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj); +void _stm_write_slowpath_card(object_t *obj, uintptr_t index) +{ + dprintf_test(("write_slowpath_card(%p, %lu)\n", + obj, index)); + + /* If CARDS_SET is not set so far, issue a normal write barrier. + If the object is large enough, ask it to set up the object for + card marking instead. */ + if (!(obj->stm_flags & GCFLAG_CARDS_SET)) { + bool mark_card = obj_should_use_cards(STM_SEGMENT->segment_base, obj); + write_slowpath_common(obj, mark_card); + if (!mark_card) + return; + } + + assert(obj_should_use_cards(STM_SEGMENT->segment_base, obj)); + dprintf_test(("write_slowpath_card %p -> index:%lu\n", + obj, index)); + + /* We reach this point if we have to mark the card. */ + assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); + assert(obj->stm_flags & GCFLAG_CARDS_SET); + assert(!is_small_uniform(obj)); /* not supported/tested */ + +#ifndef NDEBUG + struct object_s *realobj = (struct object_s *) + REAL_ADDRESS(STM_SEGMENT->segment_base, obj); + size_t size = stmcb_size_rounded_up(realobj); + /* we need at least one read marker in addition to the STM-reserved object + write-lock */ + assert(size >= 32); + /* the 'index' must be in range(length-of-obj), but we don't have + a direct way to know the length. We know that it is smaller + than the size in bytes. */ + assert(index < size); + /* this object was allocated with allocate_outside_nursery_large(), + which returns addresses aligned to 16 bytes */ + assert((((uintptr_t)obj) & 15) == 0); +#endif + + /* Write into the card's lock. This is used by the next minor + collection to know what parts of the big object may have changed. + We already own the object here or it is an overflow obj. */ + stm_read_marker_t *card = (stm_read_marker_t *)(((uintptr_t)obj) >> 4); + card += get_index_to_card_index(index); + + if (!IS_OVERFLOW_OBJ(STM_PSEGMENT, obj) + && !(card->rm == CARD_MARKED + || card->rm == STM_SEGMENT->transaction_read_version)) { + /* need to do the backup slice of the card */ + make_bk_slices(obj, + false, /* first_call */ + index, /* index: only 1 card */ + false); /* do_missing_cards */ + } + card->rm = CARD_MARKED; + + dprintf(("mark %p index %lu, card:%lu with %d\n", + obj, index, get_index_to_card_index(index), CARD_MARKED)); } +void _stm_write_slowpath(object_t *obj) { + write_slowpath_common(obj, /* mark_card */ false); +} + + static void reset_transaction_read_version(void) { /* force-reset all read markers to 0 */ @@ -705,7 +1009,8 @@ #endif memset(readmarkers, 0, NB_READMARKER_PAGES * 4096UL); } - STM_SEGMENT->transaction_read_version = 1; + STM_SEGMENT->transaction_read_version = 2; + assert(STM_SEGMENT->transaction_read_version > _STM_CARD_MARKED); } static void reset_wb_executed_flags(void) @@ -766,7 +1071,7 @@ } assert(list_is_empty(STM_PSEGMENT->modified_old_objects)); - assert(list_is_empty(STM_PSEGMENT->new_objects)); + assert(list_is_empty(STM_PSEGMENT->large_overflow_objects)); assert(list_is_empty(STM_PSEGMENT->objects_pointing_to_nursery)); assert(list_is_empty(STM_PSEGMENT->young_weakrefs)); assert(tree_is_cleared(STM_PSEGMENT->young_outside_nursery)); @@ -826,8 +1131,11 @@ STM_PSEGMENT->safe_point = SP_NO_TRANSACTION; STM_PSEGMENT->transaction_state = TS_NONE; + + _verify_cards_cleared_in_all_lists(get_priv_segment(STM_SEGMENT->segment_num)); list_clear(STM_PSEGMENT->objects_pointing_to_nursery); - list_clear(STM_PSEGMENT->new_objects); + list_clear(STM_PSEGMENT->old_objects_with_cards_set); + list_clear(STM_PSEGMENT->large_overflow_objects); release_thread_segment(tl); /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ @@ -847,13 +1155,16 @@ #endif } -static void push_new_objects_to_other_segments(void) +static void push_large_overflow_objects_to_other_segments(void) { + if (list_is_empty(STM_PSEGMENT->large_overflow_objects)) + return; + + /* XXX: also pushes small ones right now */ acquire_privatization_lock(STM_SEGMENT->segment_num); - LIST_FOREACH_R(STM_PSEGMENT->new_objects, object_t *, + LIST_FOREACH_R(STM_PSEGMENT->large_overflow_objects, object_t *, ({ - assert(item->stm_flags & GCFLAG_WB_EXECUTED); - item->stm_flags &= ~GCFLAG_WB_EXECUTED; + assert(!(item->stm_flags & GCFLAG_WB_EXECUTED)); synchronize_object_enqueue(item); })); synchronize_objects_flush(); @@ -867,7 +1178,7 @@ in handle_segfault_in_page() that also copies unknown-to-the-segment/uncommitted things. */ - list_clear(STM_PSEGMENT->new_objects); + list_clear(STM_PSEGMENT->large_overflow_objects); } @@ -882,20 +1193,32 @@ dprintf(("> stm_commit_transaction()\n")); minor_collection(1); - push_new_objects_to_other_segments(); + push_large_overflow_objects_to_other_segments(); /* push before validate. otherwise they are reachable too early */ bool was_inev = STM_PSEGMENT->transaction_state == TS_INEVITABLE; _validate_and_add_to_commit_log(); + stm_rewind_jmp_forget(STM_SEGMENT->running_thread); /* XXX do we still need a s_mutex_lock() section here? */ s_mutex_lock(); + commit_finalizers(); + /* update 'overflow_number' if needed */ + if (STM_PSEGMENT->overflow_number_has_been_used) { + highest_overflow_number += GCFLAG_OVERFLOW_NUMBER_bit0; + assert(highest_overflow_number != /* XXX else, overflow! */ + (uint32_t)-GCFLAG_OVERFLOW_NUMBER_bit0); + STM_PSEGMENT->overflow_number = highest_overflow_number; + STM_PSEGMENT->overflow_number_has_been_used = false; + } + + invoke_and_clear_user_callbacks(0); /* for commit */ + + /* >>>>> there may be a FORK() happening in the safepoint below <<<<<*/ enter_safe_point_if_requested(); assert(STM_SEGMENT->nursery_end == NURSERY_END); - stm_rewind_jmp_forget(STM_SEGMENT->running_thread); - /* if a major collection is required, do it here */ if (is_major_collection_requested()) { synchronize_all_threads(STOP_OTHERS_UNTIL_MUTEX_UNLOCK); @@ -905,9 +1228,7 @@ } } - commit_finalizers(); - - invoke_and_clear_user_callbacks(0); /* for commit */ + _verify_cards_cleared_in_all_lists(get_priv_segment(STM_SEGMENT->segment_num)); if (globally_unique_transaction && was_inev) { committed_globally_unique_transaction(); @@ -946,8 +1267,9 @@ undo->backup, SLICE_SIZE(undo->slice)); - dprintf(("reset_modified_from_backup_copies(%d): obj=%p off=%lu bk=%p\n", - segment_num, obj, SLICE_OFFSET(undo->slice), undo->backup)); + dprintf(("reset_modified_from_backup_copies(%d): obj=%p off=%lu sz=%d bk=%p\n", + segment_num, obj, SLICE_OFFSET(undo->slice), + SLICE_SIZE(undo->slice), undo->backup)); free_bk(undo); } @@ -982,9 +1304,18 @@ long bytes_in_nursery = throw_away_nursery(pseg); + /* clear CARD_MARKED on objs (don't care about CARD_MARKED_OLD) */ + LIST_FOREACH_R(pseg->old_objects_with_cards_set, object_t * /*item*/, + { + /* CARDS_SET may have already been lost because stm_validate() + may call reset_modified_from_backup_copies() */ + _reset_object_cards(pseg, item, CARD_CLEAR, false, false); + }); + acquire_modification_lock(segment_num); reset_modified_from_backup_copies(segment_num); release_modification_lock(segment_num); + _verify_cards_cleared_in_all_lists(pseg); stm_thread_local_t *tl = pseg->pub.running_thread; #ifdef STM_NO_AUTOMATIC_SETJMP @@ -1008,7 +1339,8 @@ tl->last_abort__bytes_in_nursery = bytes_in_nursery; list_clear(pseg->objects_pointing_to_nursery); - list_clear(pseg->new_objects); + list_clear(pseg->old_objects_with_cards_set); + list_clear(pseg->large_overflow_objects); list_clear(pseg->young_weakrefs); #pragma pop_macro("STM_SEGMENT") #pragma pop_macro("STM_PSEGMENT") @@ -1138,6 +1470,8 @@ ++STM_PSEGMENT->sq_len; } + + static void synchronize_object_enqueue(object_t *obj) { assert(!_is_young(obj)); @@ -1150,14 +1484,14 @@ OPT_ASSERT(obj_size >= 16); if (LIKELY(is_small_uniform(obj))) { + assert(!(obj->stm_flags & GCFLAG_CARDS_SET)); OPT_ASSERT(obj_size <= GC_LAST_SMALL_SIZE); _synchronize_fragment((stm_char *)obj, obj_size); return; } /* else, a more complicated case for large objects, to copy - around data only within the needed pages - */ + around data only within the needed pages */ uintptr_t start = (uintptr_t)obj; uintptr_t end = start + obj_size; @@ -1168,6 +1502,10 @@ } uintptr_t copy_size = copy_up_to - start; + /* double-check that the result fits in one page */ + assert(copy_size > 0); + assert(copy_size + (start & 4095) <= 4096); + _synchronize_fragment((stm_char *)start, copy_size); start = copy_up_to; diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -34,17 +34,34 @@ #define FIRST_OLD_RM_PAGE (OLD_RM_START / 4096UL) #define NB_READMARKER_PAGES (FIRST_OBJECT_PAGE - FIRST_READMARKER_PAGE) +#define CARD_SIZE _STM_CARD_SIZE + enum /* stm_flags */ { GCFLAG_WRITE_BARRIER = _STM_GCFLAG_WRITE_BARRIER, GCFLAG_HAS_SHADOW = 0x02, GCFLAG_WB_EXECUTED = 0x04, - GCFLAG_VISITED = 0x08, - GCFLAG_FINALIZATION_ORDERING = 0x10, + GCFLAG_CARDS_SET = _STM_GCFLAG_CARDS_SET, + GCFLAG_VISITED = 0x10, + GCFLAG_FINALIZATION_ORDERING = 0x20, + /* All remaining bits of the 32-bit 'stm_flags' field are taken by + the "overflow number". This is a number that identifies the + "overflow objects" from the current transaction among all old + objects. More precisely, overflow objects are objects from the + current transaction that have been flushed out of the nursery, + which occurs if the same transaction allocates too many objects. + */ + GCFLAG_OVERFLOW_NUMBER_bit0 = 0x40 /* must be last */ }; +#define SYNC_QUEUE_SIZE 31 +enum /* card values in read markers */ { + CARD_CLEAR = 0, /* card not used at all */ + CARD_MARKED = _STM_CARD_MARKED, /* card marked for tracing in the next gc */ + /* CARD_MARKED_OLD = STM_PSEGMENT->transaction_read_version, */ + /* card was marked before, but cleared in a GC */ +}; -#define SYNC_QUEUE_SIZE 31 /************************************************************/ @@ -72,6 +89,7 @@ struct list_s *modified_old_objects; struct list_s *objects_pointing_to_nursery; + struct list_s *old_objects_with_cards_set; struct tree_s *young_outside_nursery; struct tree_s *nursery_objects_shadows; @@ -88,8 +106,9 @@ /* list of objects created in the current transaction and that survived at least one minor collection. They need to be synchronized to other segments on commit, but they - do not need to be in the commit log entry. */ - struct list_s *new_objects; + do not need to be in the commit log entry. + XXX: for now it also contains small overflow objs */ + struct list_s *large_overflow_objects; uint8_t privatization_lock; // XXX KILL @@ -101,6 +120,14 @@ struct tree_s *callbacks_on_commit_and_abort[2]; + /* This is the number stored in the overflowed objects (a multiple of + GCFLAG_OVERFLOW_NUMBER_bit0). It is incremented when the + transaction is done, but only if we actually overflowed any + object; otherwise, no object has got this number. */ + uint32_t overflow_number; + bool overflow_number_has_been_used; + + struct stm_commit_log_entry_s *last_commit_log_entry; struct stm_shadowentry_s *shadowstack_at_start_of_transaction; @@ -193,6 +220,21 @@ #define REAL_ADDRESS(segment_base, src) ((segment_base) + (uintptr_t)(src)) +#define IS_OVERFLOW_OBJ(pseg, obj) (((obj)->stm_flags & -GCFLAG_OVERFLOW_NUMBER_bit0) \ + == (pseg)->overflow_number) + +static inline uintptr_t get_index_to_card_index(uintptr_t index) { + return (index / CARD_SIZE) + 1; +} + +static inline uintptr_t get_card_index_to_index(uintptr_t card_index) { + return (card_index - 1) * CARD_SIZE; +} + +static inline struct stm_read_marker_s *get_read_marker(char *segment_base, uintptr_t obj) +{ + return (struct stm_read_marker_s *)(segment_base + (obj >> 4)); +} static inline char *get_segment_base(long segment_num) { return stm_object_pages + segment_num * (NB_PAGES * 4096UL); @@ -215,6 +257,7 @@ return (addr - stm_object_pages) / (NB_PAGES * 4096UL); } +bool obj_should_use_cards(char *seg_base, object_t *obj); static bool _is_tl_registered(stm_thread_local_t *tl); static bool _seems_to_be_running_transaction(void); diff --git a/c8/stm/finalizer.c b/c8/stm/finalizer.c --- a/c8/stm/finalizer.c +++ b/c8/stm/finalizer.c @@ -98,14 +98,14 @@ list_clear(lst); } - /* also deals with newly created objects: they are at the tail of + /* also deals with overflow objects: they are at the tail of old_objects_with_light_finalizers (this list is kept in order and we cannot add any already-committed object) */ lst = pseg->old_objects_with_light_finalizers; count = list_count(lst); while (count > 0) { object_t *obj = (object_t *)list_item(lst, --count); - if (!(obj->stm_flags & GCFLAG_WB_EXECUTED)) + if (!IS_OVERFLOW_OBJ(pseg, obj)) break; lst->count = count; if (must_fix_gs) { @@ -264,11 +264,14 @@ LIST_APPEND(_finalizer_tmpstack, obj); } -static inline struct list_s *finalizer_trace(char *base, object_t *obj, - struct list_s *lst) +static inline struct list_s *finalizer_trace( + struct stm_priv_segment_info_s *pseg, object_t *obj, struct list_s *lst) { - if (!is_new_object(obj)) + char *base; + if (!is_overflow_obj_safe(pseg, obj)) base = stm_object_pages; + else + base = pseg->pub.segment_base; struct object_s *realobj = (struct object_s *)REAL_ADDRESS(base, obj); _finalizer_tmpstack = lst; @@ -277,7 +280,8 @@ } -static void _recursively_bump_finalization_state_from_2_to_3(char *base, object_t *obj) +static void _recursively_bump_finalization_state_from_2_to_3( + struct stm_priv_segment_info_s *pseg, object_t *obj) { assert(_finalization_state(obj) == 2); struct list_s *tmpstack = _finalizer_emptystack; @@ -289,7 +293,7 @@ realobj->stm_flags &= ~GCFLAG_FINALIZATION_ORDERING; /* trace */ - tmpstack = finalizer_trace(base, obj, tmpstack); + tmpstack = finalizer_trace(pseg, obj, tmpstack); } if (list_is_empty(tmpstack)) @@ -300,14 +304,16 @@ _finalizer_emptystack = tmpstack; } -static void _recursively_bump_finalization_state_from_1_to_2(char *base, object_t *obj) +static void _recursively_bump_finalization_state_from_1_to_2( + struct stm_priv_segment_info_s *pseg, object_t *obj) { assert(_finalization_state(obj) == 1); /* The call will add GCFLAG_VISITED recursively, thus bump state 1->2 */ - mark_visit_possibly_new_object(base, obj); + mark_visit_possibly_new_object(obj, pseg); } -static struct list_s *mark_finalize_step1(char *base, struct finalizers_s *f) +static struct list_s *mark_finalize_step1( + struct stm_priv_segment_info_s *pseg, struct finalizers_s *f) { if (f == NULL) return NULL; @@ -336,21 +342,22 @@ int state = _finalization_state(y); if (state <= 0) { _bump_finalization_state_from_0_to_1(y); - pending = finalizer_trace(base, y, pending); + pending = finalizer_trace(pseg, y, pending); } else if (state == 2) { - _recursively_bump_finalization_state_from_2_to_3(base, y); + _recursively_bump_finalization_state_from_2_to_3(pseg, y); } } _finalizer_pending = pending; assert(_finalization_state(x) == 1); - _recursively_bump_finalization_state_from_1_to_2(base, x); + _recursively_bump_finalization_state_from_1_to_2(pseg, x); } return marked; } -static void mark_finalize_step2(char *base, struct finalizers_s *f, - struct list_s *marked) +static void mark_finalize_step2( + struct stm_priv_segment_info_s *pseg, struct finalizers_s *f, + struct list_s *marked) { if (f == NULL) return; @@ -367,7 +374,7 @@ if (run_finalizers == NULL) run_finalizers = list_create(); LIST_APPEND(run_finalizers, x); - _recursively_bump_finalization_state_from_2_to_3(base, x); + _recursively_bump_finalization_state_from_2_to_3(pseg, x); } else { struct list_s *lst = f->objects_with_finalizers; @@ -403,29 +410,28 @@ long j; for (j = 1; j < NB_SEGMENTS; j++) { struct stm_priv_segment_info_s *pseg = get_priv_segment(j); - marked_seg[j] = mark_finalize_step1(pseg->pub.segment_base, - pseg->finalizers); + marked_seg[j] = mark_finalize_step1(pseg, pseg->finalizers); } - marked_seg[0] = mark_finalize_step1(stm_object_pages, &g_finalizers); + marked_seg[0] = mark_finalize_step1(get_priv_segment(0), &g_finalizers); LIST_FREE(_finalizer_pending); for (j = 1; j < NB_SEGMENTS; j++) { struct stm_priv_segment_info_s *pseg = get_priv_segment(j); - mark_finalize_step2(pseg->pub.segment_base, pseg->finalizers, - marked_seg[j]); + mark_finalize_step2(pseg, pseg->finalizers, marked_seg[j]); } - mark_finalize_step2(stm_object_pages, &g_finalizers, marked_seg[0]); + mark_finalize_step2(get_priv_segment(0), &g_finalizers, marked_seg[0]); LIST_FREE(_finalizer_emptystack); } -static void mark_visit_from_finalizer1(char *base, struct finalizers_s *f) +static void mark_visit_from_finalizer1( + struct stm_priv_segment_info_s *pseg, struct finalizers_s *f) { if (f != NULL && f->run_finalizers != NULL) { LIST_FOREACH_R(f->run_finalizers, object_t * /*item*/, ({ - mark_visit_possibly_new_object(base, item); + mark_visit_possibly_new_object(item, pseg); })); } } @@ -435,9 +441,9 @@ long j; for (j = 1; j < NB_SEGMENTS; j++) { struct stm_priv_segment_info_s *pseg = get_priv_segment(j); - mark_visit_from_finalizer1(pseg->pub.segment_base, pseg->finalizers); + mark_visit_from_finalizer1(pseg, pseg->finalizers); } - mark_visit_from_finalizer1(stm_object_pages, &g_finalizers); + mark_visit_from_finalizer1(get_priv_segment(0), &g_finalizers); } static void _execute_finalizers(struct finalizers_s *f) diff --git a/c8/stm/forksupport.c b/c8/stm/forksupport.c --- a/c8/stm/forksupport.c +++ b/c8/stm/forksupport.c @@ -84,11 +84,23 @@ stm_thread_local_t *tl = pr->pub.running_thread; dprintf(("forksupport_child: abort in seg%ld\n", i)); assert(tl->associated_segment_num == i); - assert(pr->transaction_state == TS_REGULAR); + assert(pr->transaction_state != TS_INEVITABLE); set_gs_register(get_segment_base(i)); assert(STM_SEGMENT->segment_num == i); s_mutex_lock(); + if (pr->transaction_state == TS_NONE) { + /* just committed, TS_NONE but still has running_thread */ + + /* do _finish_transaction() */ + STM_PSEGMENT->safe_point = SP_NO_TRANSACTION; + list_clear(STM_PSEGMENT->objects_pointing_to_nursery); + list_clear(STM_PSEGMENT->large_overflow_objects); + + s_mutex_unlock(); + return; + } + #ifndef NDEBUG pr->running_pthread = pthread_self(); #endif diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -52,10 +52,14 @@ static stm_char *allocate_outside_nursery_large(uint64_t size) { - /* Allocate the object with largemalloc.c from the lower addresses. */ - char *addr = _stm_large_malloc(size); + /* Allocate the object with largemalloc.c from the lower + addresses. Round up the size to a multiple of 16, rather than + 8, as a quick way to simplify the code in stm_write_card(). + */ + char *addr = _stm_large_malloc((size + 15) & ~15); if (addr == NULL) stm_fatalerror("not enough memory!"); + assert((((uintptr_t)addr) & 15) == 0); /* alignment check */ if (LIKELY(addr + size <= uninitialized_page_start)) { dprintf(("allocate_outside_nursery_large(%lu): %p, page=%lu\n", @@ -200,11 +204,16 @@ /************************************************************/ +static bool is_overflow_obj_safe(struct stm_priv_segment_info_s *pseg, object_t *obj) +{ + /* this function first also checks if the page is accessible in order + to not cause segfaults during major gc (it does exactly the same + as IS_OVERFLOW_OBJ otherwise) */ + if (get_page_status_in(pseg->pub.segment_num, (uintptr_t)obj / 4096UL) == PAGE_NO_ACCESS) + return false; -static bool is_new_object(object_t *obj) -{ - struct object_s *realobj = (struct object_s*)REAL_ADDRESS(stm_object_pages, obj); /* seg0 */ - return realobj->stm_flags & GCFLAG_WB_EXECUTED; + struct object_s *realobj = (struct object_s*)REAL_ADDRESS(pseg->pub.segment_base, obj); + return IS_OVERFLOW_OBJ(pseg, realobj); } @@ -230,7 +239,10 @@ } -static void mark_and_trace(object_t *obj, char *segment_base) +static void mark_and_trace( + object_t *obj, + char *segment_base, /* to trace obj in */ + struct stm_priv_segment_info_s *pseg) /* to trace children in */ { /* mark the obj and trace all reachable objs from it */ @@ -242,36 +254,40 @@ stmcb_trace(realobj, &mark_record_trace); /* trace all references found in sharing seg0 (should always be - up-to-date and not cause segfaults, except for new objs) */ + up-to-date and not cause segfaults, except for overflow objs) */ + segment_base = pseg->pub.segment_base; while (!list_is_empty(marked_objects_to_trace)) { obj = (object_t *)list_pop_item(marked_objects_to_trace); - char *base = is_new_object(obj) ? segment_base : stm_object_pages; + char *base = is_overflow_obj_safe(pseg, obj) ? segment_base : stm_object_pages; realobj = (struct object_s *)REAL_ADDRESS(base, obj); stmcb_trace(realobj, &mark_record_trace); } } -static inline void mark_visit_object(object_t *obj, char *segment_base) +static inline void mark_visit_object( + object_t *obj, + char *segment_base, /* to trace ojb in */ + struct stm_priv_segment_info_s *pseg) /* to trace children in */ { /* if already visited, don't trace */ if (obj == NULL || mark_visited_test_and_set(obj)) return; - mark_and_trace(obj, segment_base); + mark_and_trace(obj, segment_base, pseg); } -static void mark_visit_possibly_new_object(char *segment_base, object_t *obj) +static void mark_visit_possibly_new_object(object_t *obj, struct stm_priv_segment_info_s *pseg) { /* if newly allocated object, we trace in segment_base, otherwise in the sharing seg0 */ if (obj == NULL) return; - if (is_new_object(obj)) { - mark_visit_object(obj, segment_base); + if (is_overflow_obj_safe(pseg, obj)) { + mark_visit_object(obj, pseg->pub.segment_base, pseg); } else { - mark_visit_object(obj, stm_object_pages); + mark_visit_object(obj, stm_object_pages, pseg); } } @@ -282,8 +298,10 @@ end = (const struct stm_shadowentry_s *)(slice + size); for (; p < end; p++) if ((((uintptr_t)p->ss) & 3) == 0) { - assert(!is_new_object(p->ss)); - mark_visit_object(p->ss, stm_object_pages); // seg0 + mark_visit_object(p->ss, stm_object_pages, // seg0 + /* there should be no overflow objs not already + visited, so any pseg is fine really: */ + get_priv_segment(STM_SEGMENT->segment_num)); } return NULL; } @@ -350,7 +368,7 @@ and thus make all pages accessible. */ assert_obj_accessible_in(i, item); - assert(!is_new_object(item)); /* should never be in that list */ + assert(!is_overflow_obj_safe(get_priv_segment(i), item)); /* should never be in that list */ if (!mark_visited_test_and_set(item)) { /* trace shared, committed version: only do this if we didn't @@ -358,9 +376,9 @@ objs before mark_visit_from_modified_objects AND if we do mark_and_trace on an obj that is modified in >1 segment, the tracing always happens in seg0 (see mark_and_trace). */ - mark_and_trace(item, stm_object_pages); + mark_and_trace(item, stm_object_pages, get_priv_segment(i)); } - mark_and_trace(item, base); /* private, modified version */ + mark_and_trace(item, base, get_priv_segment(i)); /* private, modified version */ })); list_clear(uniques); @@ -372,7 +390,11 @@ { if (testing_prebuilt_objs != NULL) { LIST_FOREACH_R(testing_prebuilt_objs, object_t * /*item*/, - mark_visit_object(item, stm_object_pages)); // seg0 + mark_visit_object(item, stm_object_pages, // seg0 + /* any pseg is fine, as we already traced modified + objs and thus covered all overflow objs reachable + from here */ + get_priv_segment(STM_SEGMENT->segment_num))); } stm_thread_local_t *tl = stm_all_thread_locals; @@ -380,7 +402,7 @@ /* look at all objs on the shadow stack (they are old but may be uncommitted so far, so only exist in the associated_segment_num). - IF they are uncommitted new objs, trace in the actual segment, + IF they are uncommitted overflow objs, trace in the actual segment, otherwise, since we just executed a minor collection, they were all synced to the sharing seg0. Thus we can trace them there. @@ -392,17 +414,17 @@ If 'tl' is currently running, its 'last_associated_segment_num' field is the segment number that contains the correct version of its overflowed objects. */ - char *segment_base = get_segment_base(tl->last_associated_segment_num); + struct stm_priv_segment_info_s *pseg = get_priv_segment(tl->last_associated_segment_num); struct stm_shadowentry_s *current = tl->shadowstack; struct stm_shadowentry_s *base = tl->shadowstack_base; while (current-- != base) { if ((((uintptr_t)current->ss) & 3) == 0) { - mark_visit_possibly_new_object(segment_base, current->ss); + mark_visit_possibly_new_object(current->ss, pseg); } } - mark_visit_possibly_new_object(segment_base, tl->thread_local_obj); + mark_visit_possibly_new_object(tl->thread_local_obj, pseg); tl = tl->next; } while (tl != stm_all_thread_locals); @@ -413,8 +435,8 @@ for (i = 1; i < NB_SEGMENTS; i++) { if (get_priv_segment(i)->transaction_state != TS_NONE) { mark_visit_possibly_new_object( - get_segment_base(i), - get_priv_segment(i)->threadlocal_at_start_of_transaction); + get_priv_segment(i)->threadlocal_at_start_of_transaction, + get_priv_segment(i)); stm_rewind_jmp_enum_shadowstack( get_segment(i)->running_thread, @@ -423,49 +445,6 @@ } } -static void ready_new_objects(void) -{ -#pragma push_macro("STM_PSEGMENT") -#pragma push_macro("STM_SEGMENT") -#undef STM_PSEGMENT -#undef STM_SEGMENT - /* objs in new_objects only have garbage in the sharing seg0, - since it is used to mark objs as visited, we must make - sure the flag is cleared at the start of a major collection. - (XXX: ^^^ may be optional if we have the part below) - - Also, we need to be able to recognize these objects in order - to only trace them in the segment they are valid in. So we - also make sure to set WB_EXECUTED in the sharing seg0. No - other objs than new_objects have WB_EXECUTED in seg0 (since - there can only be committed versions there). - */ - - long i; - for (i = 1; i < NB_SEGMENTS; i++) { - struct stm_priv_segment_info_s *pseg = get_priv_segment(i); - struct list_s *lst = pseg->new_objects; - - LIST_FOREACH_R(lst, object_t* /*item*/, - ({ - struct object_s *realobj; - /* WB_EXECUTED always set in this segment */ - assert(realobj = (struct object_s*)REAL_ADDRESS(pseg->pub.segment_base, item)); - assert(realobj->stm_flags & GCFLAG_WB_EXECUTED); - - /* clear VISITED (garbage) and ensure WB_EXECUTED in seg0 */ - mark_visited_test_and_clear(item); - realobj = (struct object_s*)REAL_ADDRESS(stm_object_pages, item); - realobj->stm_flags |= GCFLAG_WB_EXECUTED; - - /* make sure this flag is cleared as well */ - realobj->stm_flags &= ~GCFLAG_FINALIZATION_ORDERING; - })); - } -#pragma pop_macro("STM_SEGMENT") -#pragma pop_macro("STM_PSEGMENT") -} - static void clean_up_segment_lists(void) { @@ -494,11 +473,10 @@ ({ struct object_s *realobj = (struct object_s *) REAL_ADDRESS(pseg->pub.segment_base, (uintptr_t)item); + assert(!(realobj->stm_flags & GCFLAG_WRITE_BARRIER)); + realobj->stm_flags |= GCFLAG_WRITE_BARRIER; - assert(realobj->stm_flags & GCFLAG_WB_EXECUTED); - assert(!(realobj->stm_flags & GCFLAG_WRITE_BARRIER)); - - realobj->stm_flags |= GCFLAG_WRITE_BARRIER; + OPT_ASSERT(!(realobj->stm_flags & GCFLAG_CARDS_SET)); })); list_clear(lst); } else { @@ -507,12 +485,31 @@ modified_old_objs. */ } - /* remove from new_objects all objects that die */ - lst = pseg->new_objects; + lst = pseg->old_objects_with_cards_set; + LIST_FOREACH_R(lst, object_t* /*item*/, + ({ + struct object_s *realobj = (struct object_s *) + REAL_ADDRESS(pseg->pub.segment_base, item); + OPT_ASSERT(realobj->stm_flags & GCFLAG_WRITE_BARRIER); + + /* mark marked cards as old if it survives, otherwise + CLEAR, as their spot could get reused */ + uint8_t mark_value = mark_visited_test(item) ? + pseg->pub.transaction_read_version : CARD_CLEAR; + _reset_object_cards(pseg, item, mark_value, false, + mark_value == CARD_CLEAR); + })); + list_clear(lst); + + + /* remove from large_overflow_objects all objects that die */ + lst = pseg->large_overflow_objects; uintptr_t n = list_count(lst); while (n-- > 0) { object_t *obj = (object_t *)list_item(lst, n); if (!mark_visited_test(obj)) { + if (obj_should_use_cards(pseg->pub.segment_base, obj)) + _reset_object_cards(pseg, obj, CARD_CLEAR, false, true); list_set_item(lst, n, list_pop_item(lst)); } } @@ -683,8 +680,6 @@ DEBUG_EXPECT_SEGFAULT(false); - ready_new_objects(); - /* marking */ LIST_CREATE(marked_objects_to_trace); mark_visit_from_modified_objects(); diff --git a/c8/stm/gcpage.h b/c8/stm/gcpage.h --- a/c8/stm/gcpage.h +++ b/c8/stm/gcpage.h @@ -7,7 +7,6 @@ #define GC_MIN (NB_NURSERY_PAGES * 4096 * 8) #define GC_MAJOR_COLLECT 1.82 - static struct list_s *testing_prebuilt_objs; static char *uninitialized_page_start; /* within segment 0 */ static char *uninitialized_page_stop; diff --git a/c8/stm/misc.c b/c8/stm/misc.c --- a/c8/stm/misc.c +++ b/c8/stm/misc.c @@ -42,6 +42,11 @@ return (obj->stm_flags & _STM_GCFLAG_WRITE_BARRIER) == 0; } +bool _stm_was_written_card(object_t *obj) +{ + return obj->stm_flags & _STM_GCFLAG_CARDS_SET; +} + long _stm_count_cl_entries() { struct stm_commit_log_entry_s *cl = &commit_log_root; @@ -80,6 +85,13 @@ return list_count(STM_PSEGMENT->objects_pointing_to_nursery); } +long _stm_count_old_objects_with_cards_set(void) +{ + if (STM_PSEGMENT->old_objects_with_cards_set == NULL) + return -1; + return list_count(STM_PSEGMENT->old_objects_with_cards_set); +} + object_t *_stm_enum_modified_old_objects(long index) { return (object_t *)list_item( @@ -92,6 +104,27 @@ STM_PSEGMENT->objects_pointing_to_nursery, index); } +object_t *_stm_enum_old_objects_with_cards_set(long index) +{ + return (object_t *)list_item( + STM_PSEGMENT->old_objects_with_cards_set, index); +} + + +uint8_t _stm_get_card_value(object_t *obj, long idx) +{ + struct stm_read_marker_s *cards = get_read_marker(STM_SEGMENT->segment_base, + (uintptr_t)obj); + return cards[get_index_to_card_index(idx)].rm; +} + +uint8_t _stm_get_transaction_read_version() +{ + return STM_SEGMENT->transaction_read_version; +} + + + static struct stm_commit_log_entry_s *_last_cl_entry; static long _last_cl_entry_index; void _stm_start_enum_last_cl_entry() diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c --- a/c8/stm/nursery.c +++ b/c8/stm/nursery.c @@ -38,7 +38,7 @@ } static inline bool _is_from_same_transaction(object_t *obj) { - return _is_young(obj) || (obj->stm_flags & GCFLAG_WB_EXECUTED); + return _is_young(obj) || IS_OVERFLOW_OBJ(STM_PSEGMENT, obj); } long stm_can_move(object_t *obj) @@ -132,18 +132,166 @@ nobj_sync_now = ((uintptr_t)nobj) | FLAG_SYNC_LARGE; } - /* if this is not during commit, we will add them to the new_objects - list and push them to other segments on commit. Thus we can add - the WB_EXECUTED flag so that they don't end up in modified_old_objects */ + /* if this is not during commit, we make them overflow objects + and push them to other segments on commit. */ assert(!(nobj->stm_flags & GCFLAG_WB_EXECUTED)); + assert((nobj->stm_flags & -GCFLAG_OVERFLOW_NUMBER_bit0) == 0); if (!STM_PSEGMENT->minor_collect_will_commit_now) { - nobj->stm_flags |= GCFLAG_WB_EXECUTED; + nobj->stm_flags |= STM_PSEGMENT->overflow_number; } /* Must trace the object later */ LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, nobj_sync_now); + _cards_cleared_in_object(get_priv_segment(STM_SEGMENT->segment_num), nobj, true); } +static void _cards_cleared_in_object(struct stm_priv_segment_info_s *pseg, object_t *obj, + bool strict) /* strict = MARKED_OLD not allowed */ +{ +#ifndef NDEBUG + struct object_s *realobj = (struct object_s *)REAL_ADDRESS(pseg->pub.segment_base, obj); + size_t size = stmcb_size_rounded_up(realobj); + + if (size < _STM_MIN_CARD_OBJ_SIZE) + return; /* too small for cards */ + + assert(!(realobj->stm_flags & GCFLAG_CARDS_SET)); + + if (!stmcb_obj_supports_cards(realobj)) + return; + + uintptr_t offset_itemsize[2] = {0, 0}; + stmcb_get_card_base_itemsize(realobj, offset_itemsize); + struct stm_read_marker_s *cards = get_read_marker(pseg->pub.segment_base, (uintptr_t)obj); + uintptr_t card_index = 1; + size_t real_idx_count = (size - offset_itemsize[0]) / offset_itemsize[1]; + uintptr_t last_card_index = get_index_to_card_index(real_idx_count - 1); /* max valid index */ + + while (card_index <= last_card_index) { + assert(cards[card_index].rm == CARD_CLEAR + || (cards[card_index].rm != CARD_MARKED + && cards[card_index].rm < pseg->pub.transaction_read_version) + || (!strict && cards[card_index].rm != CARD_MARKED)); + card_index++; + } +#endif +} + +static void _verify_cards_cleared_in_all_lists(struct stm_priv_segment_info_s *pseg) +{ +#ifndef NDEBUG + struct list_s *list = pseg->modified_old_objects; + struct stm_undo_s *undo = (struct stm_undo_s *)list->items; + struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count); + + for (; undo < end; undo++) { + _cards_cleared_in_object(pseg, undo->object, false); + } + LIST_FOREACH_R( + pseg->large_overflow_objects, object_t * /*item*/, + _cards_cleared_in_object(pseg, item, false)); + LIST_FOREACH_R( + pseg->objects_pointing_to_nursery, object_t * /*item*/, + _cards_cleared_in_object(pseg, item, false)); + LIST_FOREACH_R( + pseg->old_objects_with_cards_set, object_t * /*item*/, + _cards_cleared_in_object(pseg, item, false)); +#endif +} + +static void _reset_object_cards(struct stm_priv_segment_info_s *pseg, + object_t *obj, uint8_t mark_value, + bool mark_all, bool really_clear) +{ +#pragma push_macro("STM_PSEGMENT") +#pragma push_macro("STM_SEGMENT") +#undef STM_PSEGMENT +#undef STM_SEGMENT + dprintf(("_reset_object_cards(%p, mark=%d, mark_all=%d, really_clear=%d)\n", + obj, mark_value, mark_all, really_clear)); + struct object_s *realobj = (struct object_s *)REAL_ADDRESS(pseg->pub.segment_base, obj); + size_t size = stmcb_size_rounded_up(realobj); + OPT_ASSERT(size >= _STM_MIN_CARD_OBJ_SIZE); + + uintptr_t offset_itemsize[2]; + stmcb_get_card_base_itemsize(realobj, offset_itemsize); + size = (size - offset_itemsize[0]) / offset_itemsize[1]; + + /* really_clear only used for freed new objs in minor collections, as + they need to clear cards even if they are set to transaction_read_version */ + assert(IMPLY(really_clear, mark_value == CARD_CLEAR && !mark_all)); + assert(IMPLY(mark_value == CARD_CLEAR, !mark_all)); /* not necessary */ + assert(IMPLY(mark_all, + mark_value == pseg->pub.transaction_read_version)); /* set *all* to OLD */ + + struct stm_read_marker_s *cards = get_read_marker(pseg->pub.segment_base, (uintptr_t)obj); + uintptr_t card_index = 1; + uintptr_t last_card_index = get_index_to_card_index(size - 1); /* max valid index */ + + /* dprintf(("mark cards of %p, size %lu with %d, all: %d\n", + obj, size, mark_value, mark_all)); + dprintf(("obj has %lu cards\n", last_card_index));*/ + while (card_index <= last_card_index) { + if (mark_all || cards[card_index].rm == CARD_MARKED + || (really_clear && cards[card_index].rm != CARD_CLEAR)) { + /* dprintf(("mark card %lu,wl:%lu of %p with %d\n", */ + /* card_index, card_lock_idx, obj, mark_value)); */ + cards[card_index].rm = mark_value; + } + card_index++; + } + + realobj->stm_flags &= ~GCFLAG_CARDS_SET; + +#pragma pop_macro("STM_SEGMENT") +#pragma pop_macro("STM_PSEGMENT") +} + + +static void _trace_card_object(object_t *obj) +{ + assert(!_is_in_nursery(obj)); + assert(obj->stm_flags & GCFLAG_CARDS_SET); + assert(obj->stm_flags & GCFLAG_WRITE_BARRIER); + + dprintf(("_trace_card_object(%p)\n", obj)); + + struct object_s *realobj = (struct object_s *)REAL_ADDRESS(STM_SEGMENT->segment_base, obj); + size_t size = stmcb_size_rounded_up(realobj); + uintptr_t offset_itemsize[2]; + stmcb_get_card_base_itemsize(realobj, offset_itemsize); + size = (size - offset_itemsize[0]) / offset_itemsize[1]; + + struct stm_read_marker_s *cards = get_read_marker(STM_SEGMENT->segment_base, (uintptr_t)obj); + uintptr_t card_index = 1; + uintptr_t last_card_index = get_index_to_card_index(size - 1); /* max valid index */ + + /* XXX: merge ranges */ + while (card_index <= last_card_index) { + if (cards[card_index].rm == CARD_MARKED) { + /* clear or set to old: */ + cards[card_index].rm = STM_SEGMENT->transaction_read_version; + + uintptr_t start = get_card_index_to_index(card_index); + uintptr_t stop = get_card_index_to_index(card_index + 1); + if (card_index == last_card_index) { + assert(stop >= size); + stop = size; + } + else { + assert(stop < size); + } + + dprintf(("trace_cards on %p with start:%lu stop:%lu\n", + obj, start, stop)); + stmcb_trace_cards(realobj, &minor_trace_if_young, + start, stop); + } + + card_index++; + } + obj->stm_flags &= ~GCFLAG_CARDS_SET; +} static void collect_roots_in_nursery(void) { @@ -177,15 +325,20 @@ static inline void _collect_now(object_t *obj) { assert(!_is_young(obj)); + assert(!(obj->stm_flags & GCFLAG_CARDS_SET)); //dprintf(("_collect_now: %p\n", obj)); - assert(!(obj->stm_flags & GCFLAG_WRITE_BARRIER)); + if (!(obj->stm_flags & GCFLAG_WRITE_BARRIER)) { + /* Trace the 'obj' to replace pointers to nursery with pointers + outside the nursery, possibly forcing nursery objects out and + adding them to 'objects_pointing_to_nursery' as well. */ + char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); + stmcb_trace((struct object_s *)realobj, &minor_trace_if_young); - char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); - stmcb_trace((struct object_s *)realobj, &minor_trace_if_young); - - obj->stm_flags |= GCFLAG_WRITE_BARRIER; + obj->stm_flags |= GCFLAG_WRITE_BARRIER; + } + /* else traced in collect_cardrefs_to_nursery if necessary */ } @@ -201,25 +354,29 @@ assert(!_is_in_nursery(obj)); _collect_now(obj); + assert(!(obj->stm_flags & GCFLAG_CARDS_SET)); if (obj_sync_now & FLAG_SYNC_LARGE) { + /* XXX: SYNC_LARGE even set for small objs right now */ /* this is a newly allocated obj in this transaction. We must either synchronize the object to other segments now, or - add the object to new_objects list */ - if (STM_PSEGMENT->minor_collect_will_commit_now) { - acquire_privatization_lock(STM_SEGMENT->segment_num); + add the object to large_overflow_objects list */ + struct stm_priv_segment_info_s *pseg = get_priv_segment(STM_SEGMENT->segment_num); + if (pseg->minor_collect_will_commit_now) { + acquire_privatization_lock(pseg->pub.segment_num); synchronize_object_enqueue(obj); - release_privatization_lock(STM_SEGMENT->segment_num); + release_privatization_lock(pseg->pub.segment_num); } else { - LIST_APPEND(STM_PSEGMENT->new_objects, obj); + LIST_APPEND(STM_PSEGMENT->large_overflow_objects, obj); } + _cards_cleared_in_object(pseg, obj, false); } /* the list could have moved while appending */ lst = STM_PSEGMENT->objects_pointing_to_nursery; } - /* flush all new objects to other segments now */ + /* flush all overflow objects to other segments now */ if (STM_PSEGMENT->minor_collect_will_commit_now) { acquire_privatization_lock(STM_SEGMENT->segment_num); synchronize_objects_flush(); @@ -230,6 +387,30 @@ } } + +static void collect_cardrefs_to_nursery(void) +{ + dprintf(("collect_cardrefs_to_nursery\n")); + struct list_s *lst = STM_PSEGMENT->old_objects_with_cards_set; + + while (!list_is_empty(lst)) { + object_t *obj = (object_t*)list_pop_item(lst); + + assert(!_is_young(obj)); + + if (!(obj->stm_flags & GCFLAG_CARDS_SET)) { + /* sometimes we remove the CARDS_SET in the WB slowpath, see core.c */ + continue; + } + + /* traces cards, clears marked cards or marks them old if necessary */ + _trace_card_object(obj); + + assert(!(obj->stm_flags & GCFLAG_CARDS_SET)); + } +} + + static void collect_objs_still_young_but_with_finalizers(void) { struct list_s *lst = STM_PSEGMENT->finalizers->objects_with_finalizers; @@ -313,6 +494,13 @@ dprintf(("minor_collection commit=%d\n", (int)commit)); STM_PSEGMENT->minor_collect_will_commit_now = commit; + if (!commit) { + /* 'STM_PSEGMENT->overflow_number' is used now by this collection, + in the sense that it's copied to the overflow objects */ + STM_PSEGMENT->overflow_number_has_been_used = true; + } + + collect_cardrefs_to_nursery(); collect_roots_in_nursery(); @@ -320,6 +508,7 @@ collect_objs_still_young_but_with_finalizers(); collect_oldrefs_to_nursery(); + assert(list_is_empty(STM_PSEGMENT->old_objects_with_cards_set)); /* now all surviving nursery objects have been moved out */ acquire_privatization_lock(STM_SEGMENT->segment_num); diff --git a/c8/stm/nursery.h b/c8/stm/nursery.h --- a/c8/stm/nursery.h +++ b/c8/stm/nursery.h @@ -2,6 +2,14 @@ #define NSE_SIGPAUSE _STM_NSE_SIGNAL_MAX #define NSE_SIGABORT _STM_NSE_SIGNAL_ABORT +static uint32_t highest_overflow_number; + +static void _cards_cleared_in_object(struct stm_priv_segment_info_s *pseg, object_t *obj, + bool strict); +static void _reset_object_cards(struct stm_priv_segment_info_s *pseg, + object_t *obj, uint8_t mark_value, + bool mark_all, bool really_clear); + static void minor_collection(bool commit); static void check_nursery_at_transaction_start(void); static size_t throw_away_nursery(struct stm_priv_segment_info_s *pseg); diff --git a/c8/stm/setup.c b/c8/stm/setup.c --- a/c8/stm/setup.c +++ b/c8/stm/setup.c @@ -100,10 +100,11 @@ pr->pub.segment_num = i; pr->pub.segment_base = segment_base; pr->modified_old_objects = list_create(); - pr->new_objects = list_create(); + pr->large_overflow_objects = list_create(); pr->young_weakrefs = list_create(); pr->old_weakrefs = list_create(); pr->objects_pointing_to_nursery = list_create(); + pr->old_objects_with_cards_set = list_create(); pr->young_outside_nursery = tree_create(); pr->nursery_objects_shadows = tree_create(); pr->callbacks_on_commit_and_abort[0] = tree_create(); @@ -112,6 +113,8 @@ pr->old_objects_with_light_finalizers = list_create(); pr->last_commit_log_entry = &commit_log_root; + pr->overflow_number = GCFLAG_OVERFLOW_NUMBER_bit0 * i; + highest_overflow_number = pr->overflow_number; pr->pub.transaction_read_version = 0xff; } @@ -147,9 +150,10 @@ struct stm_priv_segment_info_s *pr = get_priv_segment(i); assert(list_is_empty(pr->objects_pointing_to_nursery)); list_free(pr->objects_pointing_to_nursery); + list_free(pr->old_objects_with_cards_set); list_free(pr->modified_old_objects); - assert(list_is_empty(pr->new_objects)); - list_free(pr->new_objects); + assert(list_is_empty(pr->large_overflow_objects)); + list_free(pr->large_overflow_objects); list_free(pr->young_weakrefs); list_free(pr->old_weakrefs); tree_free(pr->young_outside_nursery); diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -77,7 +77,15 @@ #define _STM_NSE_SIGNAL_ABORT 1 #define _STM_NSE_SIGNAL_MAX 2 +#define _STM_CARD_MARKED 1 /* should always be 1... */ +#define _STM_GCFLAG_CARDS_SET 0x8 +#define _STM_CARD_BITS 5 /* must be 5/6/7 for the pypy jit */ +#define _STM_CARD_SIZE (1 << _STM_CARD_BITS) +#define _STM_MIN_CARD_COUNT 17 +#define _STM_MIN_CARD_OBJ_SIZE (_STM_CARD_SIZE * _STM_MIN_CARD_COUNT) + void _stm_write_slowpath(object_t *); +void _stm_write_slowpath_card(object_t *, uintptr_t); object_t *_stm_allocate_slowpath(ssize_t); object_t *_stm_allocate_external(ssize_t); void _stm_become_inevitable(const char*); @@ -87,9 +95,11 @@ char *_stm_real_address(object_t *o); #ifdef STM_TESTS #include +uint8_t _stm_get_transaction_read_version(); +uint8_t _stm_get_card_value(object_t *obj, long idx); bool _stm_was_read(object_t *obj); bool _stm_was_written(object_t *obj); - +bool _stm_was_written_card(object_t *obj); bool _stm_is_accessible_page(uintptr_t pagenum); void _stm_test_switch(stm_thread_local_t *tl); @@ -125,7 +135,8 @@ object_t *_stm_next_last_cl_entry(); void _stm_start_enum_last_cl_entry(); long _stm_count_cl_entries(); - +long _stm_count_old_objects_with_cards_set(void); +object_t *_stm_enum_old_objects_with_cards_set(long index); uint64_t _stm_total_allocated(void); #endif @@ -156,6 +167,22 @@ extern ssize_t stmcb_size_rounded_up(struct object_s *); void stmcb_trace(struct object_s *obj, void visit(object_t **)); +/* a special trace-callback that is only called for the marked + ranges of indices (using stm_write_card(o, index)) */ +extern void stmcb_trace_cards(struct object_s *, void (object_t **), + uintptr_t start, uintptr_t stop); +/* this function will be called on objects that support cards. + It returns the base_offset (in bytes) inside the object from + where the indices start, and item_size (in bytes) for the size of + one item */ +extern void stmcb_get_card_base_itemsize(struct object_s *, + uintptr_t offset_itemsize[2]); +/* returns whether this object supports cards. we will only call + stmcb_get_card_base_itemsize on objs that do so. */ +extern long stmcb_obj_supports_cards(struct object_s *); + + + __attribute__((always_inline)) static inline void stm_read(object_t *obj) @@ -173,6 +200,45 @@ __attribute__((always_inline)) +static inline void stm_write_card(object_t *obj, uintptr_t index) +{ + /* if GCFLAG_WRITE_BARRIER is set, then don't do anything more. */ + if (UNLIKELY((obj->stm_flags & _STM_GCFLAG_WRITE_BARRIER) != 0)) { + + /* GCFLAG_WRITE_BARRIER is not set. This might be because + it's the first time we see a given small array; or it might + be because it's a big array with card marking. In the + latter case we will always reach this point, even if we + already marked the correct card. Based on the idea that it + is actually the most common case, check it here. If the + array doesn't actually use card marking, the following read + is a bit nonsensical, but in a way that should never return + CARD_MARKED by mistake. + + The computation of the card marker is further optimized by + assuming that large objects are allocated to multiples of + 16 (rather than just 8, as all objects are). Under this + assumption the following code is equivalent to: + + (obj >> 4) + (index / _STM_CARD_SIZE) + 1 + + The code below however takes only a couple of assembler + instructions. It also assumes that the intermediate value + fits in a 64-bit value, which it clearly does (all values + are much smaller than 2 ** 60). + */ + uintptr_t v = (((uintptr_t)obj) << (_STM_CARD_BITS - 4)) + index; + stm_read_marker_t *card1 = (stm_read_marker_t *)(v >> _STM_CARD_BITS); + if (card1[1].rm != _STM_CARD_MARKED) { + + /* slow path. */ + _stm_write_slowpath_card(obj, index); + } + } +} + + +__attribute__((always_inline)) static inline object_t *stm_allocate(ssize_t size_rounded_up) { OPT_ASSERT(size_rounded_up >= 16); @@ -327,14 +393,8 @@ /* dummies for now: */ -__attribute__((always_inline)) -static inline void stm_write_card(object_t *obj, uintptr_t index) -{ - stm_write(obj); -} +static inline void stm_flush_timing(stm_thread_local_t *tl, int verbose) {} - -static inline void stm_flush_timing(stm_thread_local_t *tl, int verbose) {} /* ==================== END ==================== */ static void (*stmcb_expand_marker)(char *segment_base, uintptr_t odd_number, diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -10,6 +10,8 @@ #define STM_NB_SEGMENTS ... #define _STM_GCFLAG_WRITE_BARRIER ... #define _STM_FAST_ALLOC ... +#define _STM_CARD_SIZE ... +#define _STM_CARD_MARKED ... typedef struct { ...; @@ -43,6 +45,11 @@ object_t *stm_allocate_weakref(ssize_t size_rounded_up); object_t *stm_allocate_with_finalizer(ssize_t size_rounded_up); +/*void stm_write_card(); use _checked_stm_write_card() instead */ + +uint8_t _stm_get_card_value(object_t *obj, long idx); +uint8_t _stm_get_transaction_read_version(); + void stm_setup(void); void stm_teardown(void); void stm_register_thread_local(stm_thread_local_t *tl); @@ -59,8 +66,10 @@ ssize_t stmcb_size_rounded_up(struct object_s *obj); bool _checked_stm_write(object_t *obj); +bool _checked_stm_write_card(object_t *obj, uintptr_t index); bool _stm_was_read(object_t *obj); bool _stm_was_written(object_t *obj); +bool _stm_was_written_card(object_t *obj); char *_stm_get_segment_base(long index); bool _stm_in_transaction(stm_thread_local_t *tl); int _stm_get_flags(object_t *obj); @@ -118,8 +127,10 @@ long _stm_count_modified_old_objects(void); long _stm_count_objects_pointing_to_nursery(void); +long _stm_count_old_objects_with_cards_set(void); object_t *_stm_enum_modified_old_objects(long index); object_t *_stm_enum_objects_pointing_to_nursery(long index); +object_t *_stm_enum_old_objects_with_cards_set(long index); object_t *_stm_next_last_cl_entry(); void _stm_start_enum_last_cl_entry(); long _stm_count_cl_entries(); @@ -191,6 +202,10 @@ CHECKED(stm_write(object)); } +bool _checked_stm_write_card(object_t *object, uintptr_t index) { + CHECKED(stm_write_card(object, index)); +} + bool _check_commit_transaction(void) { CHECKED(stm_commit_transaction()); } @@ -322,6 +337,43 @@ } } +long stmcb_obj_supports_cards(struct object_s *obj) +{ + return 1; +} + +void stmcb_trace_cards(struct object_s *obj, void visit(object_t **), + uintptr_t start, uintptr_t stop) +{ + int i; + struct myobj_s *myobj = (struct myobj_s*)obj; + assert(myobj->type_id != 421419); + assert(myobj->type_id != 421418); + if (myobj->type_id < 421420) { + /* basic case: no references */ + return; + } + + for (i=start; (i < myobj->type_id - 421420) && (i < stop); i++) { + object_t **ref = ((object_t **)(myobj + 1)) + i; + visit(ref); + } +} + +void stmcb_get_card_base_itemsize(struct object_s *obj, + uintptr_t offset_itemsize[2]) +{ + struct myobj_s *myobj = (struct myobj_s*)obj; + if (myobj->type_id < 421420) { + offset_itemsize[0] = SIZEOF_MYOBJ; + offset_itemsize[1] = 1; + } + else { + offset_itemsize[0] = sizeof(struct myobj_s); + offset_itemsize[1] = sizeof(object_t *); + } +} + long current_segment_num(void) { return STM_SEGMENT->segment_num; @@ -347,6 +399,11 @@ GCFLAG_WRITE_BARRIER = lib._STM_GCFLAG_WRITE_BARRIER NB_SEGMENTS = lib.STM_NB_SEGMENTS FAST_ALLOC = lib._STM_FAST_ALLOC +CARD_SIZE = lib._STM_CARD_SIZE # 16b at least +CARD_CLEAR = 0 +CARD_MARKED = lib._STM_CARD_MARKED +CARD_MARKED_OLD = lib._stm_get_transaction_read_version + class Conflict(Exception): pass @@ -506,11 +563,11 @@ return None return map(lib._stm_enum_objects_pointing_to_nursery, range(count)) -def old_objects_with_cards(): - count = lib._stm_count_old_objects_with_cards() +def old_objects_with_cards_set(): + count = lib._stm_count_old_objects_with_cards_set() if count < 0: return None - return map(lib._stm_enum_old_objects_with_cards, range(count)) + return map(lib._stm_enum_old_objects_with_cards_set, range(count)) def last_commit_log_entry_objs(): lib._stm_start_enum_last_cl_entry() diff --git a/c8/test/test_card_marking.py b/c8/test/test_card_marking.py new file mode 100644 --- /dev/null +++ b/c8/test/test_card_marking.py @@ -0,0 +1,376 @@ +from support import * +import py + + +get_card_value = lib._stm_get_card_value + From noreply at buildbot.pypy.org Sat Mar 7 15:27:40 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 7 Mar 2015 15:27:40 +0100 (CET) Subject: [pypy-commit] stmgc default: Add a comment and an XXXXXX Message-ID: <20150307142740.44C821C023F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1693:c447910735da Date: 2015-03-07 15:28 +0100 http://bitbucket.org/pypy/stmgc/changeset/c447910735da/ Log: Add a comment and an XXXXXX diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -498,6 +498,10 @@ break; /* success! */ } else if (old->next == INEV_RUNNING) { /* we failed because there is an INEV transaction running */ + /* XXXXXX for now just sleep (XXX with the lock acquired? + isn't it a bad idea?). We should really ask to inev + transaction to do the commit for us, and then we can + continue running. */ usleep(10); } From noreply at buildbot.pypy.org Sat Mar 7 16:33:44 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 7 Mar 2015 16:33:44 +0100 (CET) Subject: [pypy-commit] stmgc c8-marker: A branch to re-implement the marker logic to debug contentions Message-ID: <20150307153344.B02341C052B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-marker Changeset: r1694:9b75f20316ae Date: 2015-03-07 15:16 +0100 http://bitbucket.org/pypy/stmgc/changeset/9b75f20316ae/ Log: A branch to re-implement the marker logic to debug contentions From noreply at buildbot.pypy.org Sat Mar 7 16:33:45 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 7 Mar 2015 16:33:45 +0100 (CET) Subject: [pypy-commit] stmgc c8-marker: in-progress: import some code from c7 Message-ID: <20150307153345.D60FE1C052B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-marker Changeset: r1695:a3f57a07666c Date: 2015-03-07 16:34 +0100 http://bitbucket.org/pypy/stmgc/changeset/a3f57a07666c/ Log: in-progress: import some code from c7 diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -528,6 +528,7 @@ STM_PSEGMENT->safe_point = SP_NO_TRANSACTION; list_clear(STM_PSEGMENT->modified_old_objects); + list_clear(STM_PSEGMENT->modified_old_objects_markers); STM_PSEGMENT->last_commit_log_entry = new; release_modification_lock(STM_SEGMENT->segment_num); } @@ -557,6 +558,7 @@ STM_PSEGMENT->transaction_state = TS_NONE; STM_PSEGMENT->safe_point = SP_NO_TRANSACTION; list_clear(STM_PSEGMENT->modified_old_objects); + list_clear(STM_PSEGMENT->modified_old_objects_markers); STM_PSEGMENT->last_commit_log_entry = new; /* do it: */ @@ -630,6 +632,7 @@ (uintptr_t)obj, /* obj */ (uintptr_t)bk_slice, /* bk_addr */ NEW_SLICE(slice_off, slice_sz)); + timing_record_write(); dprintf(("> append slice %p, off=%lu, sz=%lu\n", bk_slice, slice_off, slice_sz)); release_modification_lock(STM_SEGMENT->segment_num); @@ -1051,6 +1054,7 @@ assert(STM_PSEGMENT->safe_point == SP_NO_TRANSACTION); assert(STM_PSEGMENT->transaction_state == TS_NONE); + timing_event(tl, STM_TRANSACTION_START); STM_PSEGMENT->transaction_state = TS_REGULAR; STM_PSEGMENT->safe_point = SP_RUNNING; #ifndef NDEBUG @@ -1071,6 +1075,7 @@ } assert(list_is_empty(STM_PSEGMENT->modified_old_objects)); + assert(list_is_empty(STM_PSEGMENT->modified_old_objects_markers)); assert(list_is_empty(STM_PSEGMENT->large_overflow_objects)); assert(list_is_empty(STM_PSEGMENT->objects_pointing_to_nursery)); assert(list_is_empty(STM_PSEGMENT->young_weakrefs)); @@ -1080,6 +1085,10 @@ assert(tree_is_cleared(STM_PSEGMENT->callbacks_on_commit_and_abort[1])); assert(list_is_empty(STM_PSEGMENT->young_objects_with_light_finalizers)); assert(STM_PSEGMENT->finalizers == NULL); +#ifndef NDEBUG + /* this should not be used when objects_pointing_to_nursery == NULL */ + STM_PSEGMENT->modified_old_objects_markers_num_old = 99999999999999999L; +#endif check_nursery_at_transaction_start(); @@ -1125,7 +1134,7 @@ /************************************************************/ -static void _finish_transaction() +static void _finish_transaction(enum stm_event_e event) { stm_thread_local_t *tl = STM_SEGMENT->running_thread; @@ -1136,6 +1145,7 @@ list_clear(STM_PSEGMENT->objects_pointing_to_nursery); list_clear(STM_PSEGMENT->old_objects_with_cards_set); list_clear(STM_PSEGMENT->large_overflow_objects); + timing_event(tl, event); release_thread_segment(tl); /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ @@ -1236,7 +1246,7 @@ /* done */ stm_thread_local_t *tl = STM_SEGMENT->running_thread; - _finish_transaction(); + _finish_transaction(STM_TRANSACTION_COMMIT); /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ s_mutex_unlock(); @@ -1369,7 +1379,7 @@ : NURSERY_END; } - _finish_transaction(); + _finish_transaction(STM_TRANSACTION_ABORT); /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ return tl; diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -93,6 +93,12 @@ struct tree_s *young_outside_nursery; struct tree_s *nursery_objects_shadows; + /* For each entry in 'modified_old_objects', we have two entries + in the following list, which give the marker at the time we added + the entry to modified_old_objects. */ + struct list_s *modified_old_objects_markers; + uintptr_t modified_old_objects_markers_num_old; + /* List of all young weakrefs to check in minor collections. These are the only weakrefs that may point to young objects and never contain NULL. */ diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -386,6 +386,20 @@ LIST_FREE(uniques); } +static void mark_visit_from_markers(void) +{ + long j; + for (j = 1; j < NB_SEGMENTS; j++) { + struct stm_priv_segment_info_s *pseg = get_priv_segment(j); + struct list_s *lst = pseg->modified_old_objects_markers; + uintptr_t i; + for (i = list_count(lst); i > 0; i -= 2) { + mark_visit_possibly_new_object((object_t *)list_item(lst, i - 1), + pseg); + } + } +} + static void mark_visit_from_roots(void) { if (testing_prebuilt_objs != NULL) { @@ -683,6 +697,7 @@ /* marking */ LIST_CREATE(marked_objects_to_trace); mark_visit_from_modified_objects(); + mark_visit_from_markers(); mark_visit_from_roots(); mark_visit_from_finalizer_pending(); diff --git a/c7/stm/marker.c b/c8/stm/marker.c copy from c7/stm/marker.c copy to c8/stm/marker.c --- a/c7/stm/marker.c +++ b/c8/stm/marker.c @@ -31,15 +31,6 @@ } } -static void _timing_fetch_inev(void) -{ - stm_loc_marker_t marker; - marker.tl = STM_SEGMENT->running_thread; - marker_fetch(&marker); - STM_PSEGMENT->marker_inev.odd_number = marker.odd_number; - STM_PSEGMENT->marker_inev.object = marker.object; -} - static void marker_fetch_obj_write(object_t *obj, stm_loc_marker_t *out_marker) { /* From 'out_marker->tl', fill in 'out_marker->segment_base' and @@ -48,23 +39,16 @@ */ assert(_has_mutex()); - /* here, we acquired the other thread's marker_lock, which means that: - - (1) it has finished filling 'modified_old_objects' after it sets - up the write_locks[] value that we're conflicting with - - (2) it is not mutating 'modified_old_objects' right now (we have - the global mutex_lock at this point too). - */ - long i; + long i, num; int in_segment_num = out_marker->tl->associated_segment_num; assert(in_segment_num >= 1); struct stm_priv_segment_info_s *pseg = get_priv_segment(in_segment_num); struct list_s *mlst = pseg->modified_old_objects; struct list_s *mlstm = pseg->modified_old_objects_markers; - assert(list_count(mlstm) <= 2 * list_count(mlst)); - for (i = list_count(mlstm) / 2; --i >= 0; ) { - if (list_item(mlst, i) == (uintptr_t)obj) { + num = list_count(mlstm) / 2; + assert(num * 3 <= list_count(mlst)); + for (i = 0; i < num; i++) { + if (list_item(mlst, i * 3) == (uintptr_t)obj) { out_marker->odd_number = list_item(mlstm, i * 2 + 0); out_marker->object = (object_t *)list_item(mlstm, i * 2 + 1); return; @@ -80,7 +64,7 @@ marker.tl = STM_SEGMENT->running_thread; marker_fetch(&marker); - long base_count = list_count(STM_PSEGMENT->modified_old_objects); + long base_count = list_count(STM_PSEGMENT->modified_old_objects) / 3; struct list_s *mlstm = STM_PSEGMENT->modified_old_objects_markers; while (list_count(mlstm) < 2 * base_count) { mlstm = list_append2(mlstm, 0, 0); @@ -89,55 +73,21 @@ STM_PSEGMENT->modified_old_objects_markers = mlstm; } -static void _timing_contention(enum stm_event_e kind, - uint8_t other_segment_num, object_t *obj) +static void timing_write_read_contention(object_t *obj) { - struct stm_priv_segment_info_s *other_pseg; - other_pseg = get_priv_segment(other_segment_num); + if (stmcb_timing_event == NULL) + return; - char *other_segment_base = other_pseg->pub.segment_base; - acquire_marker_lock(other_segment_base); + /* Collect the older location of the write from the current thread. */ + stm_loc_marker_t marker; + marker.tl = STM_SEGMENT->running_thread; + marker.segment_base = STM_SEGMENT->segment_base; + marker_fetch_obj_write(obj, &marker); - stm_loc_marker_t markers[2]; - - /* Collect the location for myself. It's usually the current - location, except in a write-read abort, in which case it's the - older location of the write. */ - markers[0].tl = STM_SEGMENT->running_thread; - markers[0].segment_base = STM_SEGMENT->segment_base; - - if (kind == STM_CONTENTION_WRITE_READ) - marker_fetch_obj_write(obj, &markers[0]); - else - marker_fetch(&markers[0]); - - /* For some categories, we can also collect the relevant information - for the other segment. */ - markers[1].tl = other_pseg->pub.running_thread; - markers[1].segment_base = other_pseg->pub.segment_base; - - switch (kind) { - case STM_CONTENTION_WRITE_WRITE: - marker_fetch_obj_write(obj, &markers[1]); - break; - case STM_CONTENTION_INEVITABLE: - markers[1].odd_number = other_pseg->marker_inev.odd_number; - markers[1].object = other_pseg->marker_inev.object; - break; - default: - markers[1].odd_number = 0; - markers[1].object = NULL; - break; - } - - stmcb_timing_event(markers[0].tl, kind, markers); - - /* only release the lock after stmcb_timing_event(), otherwise it could - run into race conditions trying to interpret 'markers[1].object' */ - release_marker_lock(other_segment_base); + stmcb_timing_event(marker.tl, STM_CONTENTION_WRITE_READ, &marker); } void (*stmcb_timing_event)(stm_thread_local_t *tl, /* the local thread */ enum stm_event_e event, - stm_loc_marker_t *markers); + stm_loc_marker_t *marker); diff --git a/c7/stm/marker.h b/c8/stm/marker.h copy from c7/stm/marker.h copy to c8/stm/marker.h --- a/c7/stm/marker.h +++ b/c8/stm/marker.h @@ -1,8 +1,6 @@ static void _timing_record_write(void); -static void _timing_fetch_inev(void); -static void _timing_contention(enum stm_event_e kind, - uint8_t other_segment_num, object_t *obj); +static void timing_write_read_contention(object_t *obj); #define timing_event(tl, event) \ @@ -10,10 +8,3 @@ #define timing_record_write() \ (stmcb_timing_event != NULL ? _timing_record_write() : (void)0) - -#define timing_fetch_inev() \ - (stmcb_timing_event != NULL ? _timing_fetch_inev() : (void)0) - -#define timing_contention(kind, other_segnum, obj) \ - (stmcb_timing_event != NULL ? \ - _timing_contention(kind, other_segnum, obj) : (void)0) diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c --- a/c8/stm/nursery.c +++ b/c8/stm/nursery.c @@ -410,6 +410,18 @@ } } +static void collect_roots_from_markers(uintptr_t num_old) +{ + dprintf(("collect_roots_from_markers\n")); + /* visit the marker objects */ + struct list_s *mlst = STM_PSEGMENT->modified_old_objects_markers; + STM_PSEGMENT->modified_old_objects_markers_num_old = list_count(mlst); + uintptr_t i, total = list_count(mlst); + assert((total & 1) == 0); + for (i = num_old + 1; i < total; i += 2) { + minor_trace_if_young((object_t **)list_ptr_to_item(mlst, i)); + } +} static void collect_objs_still_young_but_with_finalizers(void) { @@ -494,6 +506,13 @@ dprintf(("minor_collection commit=%d\n", (int)commit)); STM_PSEGMENT->minor_collect_will_commit_now = commit; + + uintptr_t num_old; + if (STM_PSEGMENT->overflow_number_has_been_used) + num_old = STM_PSEGMENT->modified_old_objects_markers_num_old; + else + num_old = 0; + if (!commit) { /* 'STM_PSEGMENT->overflow_number' is used now by this collection, in the sense that it's copied to the overflow objects */ @@ -502,6 +521,8 @@ collect_cardrefs_to_nursery(); + collect_roots_from_markers(num_old); + collect_roots_in_nursery(); if (STM_PSEGMENT->finalizers != NULL) diff --git a/c8/stm/setup.c b/c8/stm/setup.c --- a/c8/stm/setup.c +++ b/c8/stm/setup.c @@ -100,6 +100,7 @@ pr->pub.segment_num = i; pr->pub.segment_base = segment_base; pr->modified_old_objects = list_create(); + pr->modified_old_objects_markers = list_create(); pr->large_overflow_objects = list_create(); pr->young_weakrefs = list_create(); pr->old_weakrefs = list_create(); @@ -152,6 +153,7 @@ list_free(pr->objects_pointing_to_nursery); list_free(pr->old_objects_with_cards_set); list_free(pr->modified_old_objects); + list_free(pr->modified_old_objects_markers); assert(list_is_empty(pr->large_overflow_objects)); list_free(pr->large_overflow_objects); list_free(pr->young_weakrefs); diff --git a/c8/stmgc.c b/c8/stmgc.c --- a/c8/stmgc.c +++ b/c8/stmgc.c @@ -14,6 +14,7 @@ #include "stm/gcpage.h" #include "stm/extra.h" #include "stm/fprintcolor.h" +#include "stm/marker.h" #include "stm/rewind_setjmp.h" #include "stm/finalizer.h" @@ -34,5 +35,6 @@ #include "stm/core.c" #include "stm/extra.c" #include "stm/fprintcolor.c" +#include "stm/marker.c" #include "stm/rewind_setjmp.c" #include "stm/finalizer.c" diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -342,6 +342,69 @@ void stm_resume_all_other_threads(void); +/* Profiling events. In the comments: content of the markers, if any */ +enum stm_event_e { + /* always STM_TRANSACTION_START followed later by one of COMMIT or ABORT */ + STM_TRANSACTION_START, + STM_TRANSACTION_COMMIT, + STM_TRANSACTION_ABORT, + + /* write-read contention: a "marker" is included in the PYPYSTM file + saying where the write was done. Followed by STM_TRANSACTION_ABORT. */ + STM_CONTENTION_WRITE_READ, + + /* always one STM_WAIT_xxx followed later by STM_WAIT_DONE */ + STM_WAIT_FREE_SEGMENT, + STM_WAIT_OTHER_INEVITABLE, + STM_WAIT_DONE, + + /* start and end of GC cycles */ + STM_GC_MINOR_START, + STM_GC_MINOR_DONE, + STM_GC_MAJOR_START, + STM_GC_MAJOR_DONE, + + _STM_EVENT_N +}; + +#define STM_EVENT_NAMES \ + "transaction start", \ + "transaction commit", \ + "transaction abort", \ + "contention write read", \ + "wait free segment", \ + "wait other inevitable", \ + "wait done", \ + "gc minor start", \ + "gc minor done", \ + "gc major start", \ + "gc major done" + +/* The markers pushed in the shadowstack are an odd number followed by a + regular pointer. */ +typedef struct { + stm_thread_local_t *tl; + char *segment_base; /* base to interpret the 'object' below */ + uintptr_t odd_number; /* marker odd number, or 0 if marker is missing */ + object_t *object; /* marker object, or NULL if marker is missing */ +} stm_loc_marker_t; +extern void (*stmcb_timing_event)(stm_thread_local_t *tl, /* the local thread */ + enum stm_event_e event, + stm_loc_marker_t *marker); + +/* Calling this sets up a stmcb_timing_event callback that will produce + a binary file called 'profiling_file_name'. Call it with + 'fork_mode == 0' for only the main process, and with + 'fork_mode == 1' to also write files called + 'profiling_file_name.fork' after a fork(). Call it with NULL to + stop profiling. Returns -1 in case of error (see errno then). + The optional 'expand_marker' function pointer is called to expand + the marker's odd_number and object into data, starting at the given + position and with the given maximum length. */ +int stm_set_timing_log(const char *profiling_file_name, int fork_mode, + int expand_marker(stm_loc_marker_t *, char *, int)); + + /* Convenience macros to push the markers into the shadowstack */ #define STM_PUSH_MARKER(tl, odd_num, p) do { \ uintptr_t _odd_num = (odd_num); \ diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -124,6 +124,46 @@ long stm_call_on_abort(stm_thread_local_t *, void *key, void callback(void *)); long stm_call_on_commit(stm_thread_local_t *, void *key, void callback(void *)); +/* Profiling events. In the comments: content of the markers, if any */ +enum stm_event_e { + /* always STM_TRANSACTION_START followed later by one of COMMIT or ABORT */ + STM_TRANSACTION_START, + STM_TRANSACTION_COMMIT, + STM_TRANSACTION_ABORT, + + /* write-read contention: a "marker" is included in the PYPYSTM file + saying where the write was done. Followed by STM_TRANSACTION_ABORT. */ + STM_CONTENTION_WRITE_READ, + + /* always one STM_WAIT_xxx followed later by STM_WAIT_DONE */ + STM_WAIT_FREE_SEGMENT, + STM_WAIT_OTHER_INEVITABLE, + STM_WAIT_DONE, + + /* start and end of GC cycles */ + STM_GC_MINOR_START, + STM_GC_MINOR_DONE, + STM_GC_MAJOR_START, + STM_GC_MAJOR_DONE, + ... +}; + +typedef struct { + stm_thread_local_t *tl; + /* If segment_base==NULL, the remaining fields are undefined. If non-NULL, + the rest is a marker to interpret from this segment_base addr. */ + char *segment_base; + uintptr_t odd_number; + object_t *object; +} stm_loc_marker_t; + +typedef void (*stmcb_timing_event_fn)(stm_thread_local_t *tl, + enum stm_event_e event, + stm_loc_marker_t *markers); +stmcb_timing_event_fn stmcb_timing_event; + +int stm_set_timing_log(const char *profiling_file_name, int prof_mode, + int expand_marker(stm_loc_marker_t *, char *, int)); long _stm_count_modified_old_objects(void); long _stm_count_objects_pointing_to_nursery(void); diff --git a/c7/test/test_marker.py b/c8/test/test_marker.py copy from c7/test/test_marker.py copy to c8/test/test_marker.py From noreply at buildbot.pypy.org Sat Mar 7 21:50:42 2015 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 7 Mar 2015 21:50:42 +0100 (CET) Subject: [pypy-commit] pypy default: test, fix random.getstate() to return longs so that repr(state) is identical w/cpython Message-ID: <20150307205042.11E451C0107@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r76275:a2a352a3b535 Date: 2015-03-07 22:51 +0200 http://bitbucket.org/pypy/pypy/changeset/a2a352a3b535/ Log: test, fix random.getstate() to return longs so that repr(state) is identical w/cpython diff --git a/pypy/module/_random/interp_random.py b/pypy/module/_random/interp_random.py --- a/pypy/module/_random/interp_random.py +++ b/pypy/module/_random/interp_random.py @@ -4,7 +4,7 @@ from pypy.interpreter.typedef import TypeDef from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.baseobjspace import W_Root -from rpython.rlib.rarithmetic import r_uint, intmask +from rpython.rlib.rarithmetic import r_uint, intmask, widen from rpython.rlib import rbigint, rrandom, rstring @@ -54,8 +54,8 @@ def getstate(self, space): state = [None] * (rrandom.N + 1) for i in range(rrandom.N): - state[i] = space.newint(intmask(self._rnd.state[i])) - state[rrandom.N] = space.newint(self._rnd.index) + state[i] = space.wrap(widen(self._rnd.state[i])) + state[rrandom.N] = space.newlong(self._rnd.index) return space.newtuple(state) def setstate(self, space, w_state): diff --git a/pypy/module/_random/test/test_random.py b/pypy/module/_random/test/test_random.py --- a/pypy/module/_random/test/test_random.py +++ b/pypy/module/_random/test/test_random.py @@ -41,6 +41,17 @@ # does not crash rnd1.setstate((-1, ) * 624 + (0, )) + def test_state_repr(self): + # since app-level jumpahead salts with repr(state), + # it is important the repr is consistent with cpython + import _random + rnd = _random.Random() + rnd.seed(1234) + state = rnd.getstate() + s = repr(state) + assert len(s) == 7956 + assert s.count('L') == 625 + def test_seed(self): import _random, sys rnd = _random.Random() From noreply at buildbot.pypy.org Sat Mar 7 22:59:31 2015 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 7 Mar 2015 22:59:31 +0100 (CET) Subject: [pypy-commit] pypy default: test, export ndarray.conjugate Message-ID: <20150307215931.863401C14CF@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r76276:0720512cd72d Date: 2015-03-07 23:54 +0200 http://bitbucket.org/pypy/pypy/changeset/0720512cd72d/ Log: test, export ndarray.conjugate diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -1462,6 +1462,7 @@ imag = GetSetProperty(W_NDimArray.descr_get_imag, W_NDimArray.descr_set_imag), conj = interp2app(W_NDimArray.descr_conj), + conjugate = interp2app(W_NDimArray.descr_conj), argsort = interp2app(W_NDimArray.descr_argsort), sort = interp2app(W_NDimArray.descr_sort), diff --git a/pypy/module/micronumpy/test/test_complex.py b/pypy/module/micronumpy/test/test_complex.py --- a/pypy/module/micronumpy/test/test_complex.py +++ b/pypy/module/micronumpy/test/test_complex.py @@ -382,6 +382,7 @@ assert np.conjugate(1+2j) == 1-2j eye2 = np.array([[1, 0], [0, 1]]) + assert (eye2.conjugate() == eye2).all() x = eye2 + 1j * eye2 for a, b in zip(np.conjugate(x), np.array([[ 1.-1.j, 0.-0.j], [ 0.-0.j, 1.-1.j]])): assert a[0] == b[0] From noreply at buildbot.pypy.org Sun Mar 8 09:08:13 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 8 Mar 2015 09:08:13 +0100 (CET) Subject: [pypy-commit] cffi release-0.9: hg merge default Message-ID: <20150308080813.DE3AB1C019D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: release-0.9 Changeset: r1657:cf39296d0088 Date: 2015-03-08 09:02 +0100 http://bitbucket.org/cffi/cffi/changeset/cf39296d0088/ Log: hg merge default diff --git a/c/malloc_closure.h b/c/malloc_closure.h --- a/c/malloc_closure.h +++ b/c/malloc_closure.h @@ -125,6 +125,7 @@ if (item == NULL) return; #else + { int prot = PROT_READ | PROT_WRITE | PROT_EXEC; if (is_emutramp_enabled ()) prot &= ~PROT_EXEC; @@ -136,6 +137,7 @@ 0); if (item == (void *)MAP_FAILED) return; + } #endif #ifdef MALLOC_CLOSURE_DEBUG From noreply at buildbot.pypy.org Sun Mar 8 09:08:15 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 8 Mar 2015 09:08:15 +0100 (CET) Subject: [pypy-commit] cffi release-0.9: Prepare release 0.9.1 Message-ID: <20150308080815.019561C019D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: release-0.9 Changeset: r1658:0e3fbf96164b Date: 2015-03-08 09:06 +0100 http://bitbucket.org/cffi/cffi/changeset/0e3fbf96164b/ Log: Prepare release 0.9.1 diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -5689,7 +5689,7 @@ if (v == NULL || PyModule_AddObject(m, "_C_API", v) < 0) INITERROR; - v = PyText_FromString("0.9.0"); + v = PyText_FromString("0.9.1"); if (v == NULL || PyModule_AddObject(m, "__version__", v) < 0) INITERROR; diff --git a/c/test_c.py b/c/test_c.py --- a/c/test_c.py +++ b/c/test_c.py @@ -3260,4 +3260,4 @@ def test_version(): # this test is here mostly for PyPy - assert __version__ == "0.9.0" + assert __version__ == "0.9.1" diff --git a/cffi/__init__.py b/cffi/__init__.py --- a/cffi/__init__.py +++ b/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "0.9.0" -__version_info__ = (0, 9, 0) +__version__ = "0.9.1" +__version_info__ = (0, 9, 1) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/doc/source/conf.py b/doc/source/conf.py --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -47,7 +47,7 @@ # The short X.Y version. version = '0.9' # The full version, including alpha/beta/rc tags. -release = '0.9.0' +release = '0.9.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/source/index.rst b/doc/source/index.rst --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -88,13 +88,13 @@ Download and Installation: -* http://pypi.python.org/packages/source/c/cffi/cffi-0.9.0.tar.gz +* http://pypi.python.org/packages/source/c/cffi/cffi-0.9.1.tar.gz - Or grab the most current version by following the instructions below. - - MD5: 3a2f6b9f16e8082271aed6dcac51a71a + - MD5: ... - - SHA: 02e44ecada40cb859e18e0b628cc52deba064a39 + - SHA: ... * Or get it from the `Bitbucket page`_: ``hg clone https://bitbucket.org/cffi/cffi`` diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -139,7 +139,7 @@ `Mailing list `_ """, - version='0.9.0', + version='0.9.1', packages=['cffi'], zip_safe=False, From noreply at buildbot.pypy.org Sun Mar 8 09:08:16 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 8 Mar 2015 09:08:16 +0100 (CET) Subject: [pypy-commit] cffi release-0.9: Add the MD5/SHA Message-ID: <20150308080816.108D81C019D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: release-0.9 Changeset: r1659:ec7781f6d8bc Date: 2015-03-08 09:08 +0100 http://bitbucket.org/cffi/cffi/changeset/ec7781f6d8bc/ Log: Add the MD5/SHA diff --git a/doc/source/index.rst b/doc/source/index.rst --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -92,9 +92,9 @@ - Or grab the most current version by following the instructions below. - - MD5: ... + - MD5: 8dbdf23c600845b75654024e434601ce - - SHA: ... + - SHA: 6a1b297ee7108803f54bb6333cf1e5630a193756 * Or get it from the `Bitbucket page`_: ``hg clone https://bitbucket.org/cffi/cffi`` From noreply at buildbot.pypy.org Sun Mar 8 09:08:17 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 8 Mar 2015 09:08:17 +0100 (CET) Subject: [pypy-commit] cffi default: hg merge release-0.9 (this is 0.9.1) Message-ID: <20150308080817.8DF771C019D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1660:bc4e4949b047 Date: 2015-03-08 09:08 +0100 http://bitbucket.org/cffi/cffi/changeset/bc4e4949b047/ Log: hg merge release-0.9 (this is 0.9.1) diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -5689,7 +5689,7 @@ if (v == NULL || PyModule_AddObject(m, "_C_API", v) < 0) INITERROR; - v = PyText_FromString("0.9.0"); + v = PyText_FromString("0.9.1"); if (v == NULL || PyModule_AddObject(m, "__version__", v) < 0) INITERROR; diff --git a/c/test_c.py b/c/test_c.py --- a/c/test_c.py +++ b/c/test_c.py @@ -3260,4 +3260,4 @@ def test_version(): # this test is here mostly for PyPy - assert __version__ == "0.9.0" + assert __version__ == "0.9.1" diff --git a/cffi/__init__.py b/cffi/__init__.py --- a/cffi/__init__.py +++ b/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "0.9.0" -__version_info__ = (0, 9, 0) +__version__ = "0.9.1" +__version_info__ = (0, 9, 1) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/doc/source/conf.py b/doc/source/conf.py --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -47,7 +47,7 @@ # The short X.Y version. version = '0.9' # The full version, including alpha/beta/rc tags. -release = '0.9.0' +release = '0.9.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/source/index.rst b/doc/source/index.rst --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -88,13 +88,13 @@ Download and Installation: -* http://pypi.python.org/packages/source/c/cffi/cffi-0.9.0.tar.gz +* http://pypi.python.org/packages/source/c/cffi/cffi-0.9.1.tar.gz - Or grab the most current version by following the instructions below. - - MD5: 3a2f6b9f16e8082271aed6dcac51a71a + - MD5: 8dbdf23c600845b75654024e434601ce - - SHA: 02e44ecada40cb859e18e0b628cc52deba064a39 + - SHA: 6a1b297ee7108803f54bb6333cf1e5630a193756 * Or get it from the `Bitbucket page`_: ``hg clone https://bitbucket.org/cffi/cffi`` diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -139,7 +139,7 @@ `Mailing list `_ """, - version='0.9.0', + version='0.9.1', packages=['cffi'], zip_safe=False, From noreply at buildbot.pypy.org Sun Mar 8 09:17:08 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 8 Mar 2015 09:17:08 +0100 (CET) Subject: [pypy-commit] pypy default: update the version number to cffi 0.9.1 Message-ID: <20150308081708.C07E81C019D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76277:7aee078b2e7c Date: 2015-03-08 09:17 +0100 http://bitbucket.org/pypy/pypy/changeset/7aee078b2e7c/ Log: update the version number to cffi 0.9.1 diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -8,7 +8,7 @@ appleveldefs = { } interpleveldefs = { - '__version__': 'space.wrap("0.9.0")', + '__version__': 'space.wrap("0.9.1")', 'load_library': 'libraryobj.load_library', diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -3249,4 +3249,4 @@ def test_version(): # this test is here mostly for PyPy - assert __version__ == "0.9.0" + assert __version__ == "0.9.1" From noreply at buildbot.pypy.org Sun Mar 8 09:47:41 2015 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 8 Mar 2015 09:47:41 +0100 (CET) Subject: [pypy-commit] pypy optresult: enough virtual array of structs support Message-ID: <20150308084741.4B3281C0107@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult Changeset: r76278:9508bc587256 Date: 2015-03-08 10:47 +0200 http://bitbucket.org/pypy/pypy/changeset/9508bc587256/ Log: enough virtual array of structs support diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py --- a/rpython/jit/backend/llgraph/runner.py +++ b/rpython/jit/backend/llgraph/runner.py @@ -151,7 +151,7 @@ rffi.cast(TYPE, -1) == -1) class ArrayDescr(AbstractDescr): - def __init__(self, A): + def __init__(self, A, runner): self.A = self.OUTERA = A if isinstance(A, lltype.Struct): self.A = A._flds[A._arrayfld] @@ -191,10 +191,12 @@ class InteriorFieldDescr(AbstractDescr): - def __init__(self, A, fieldname): + def __init__(self, A, fieldname, runner): self.A = A self.fieldname = fieldname self.FIELD = getattr(A.OF, fieldname) + self.arraydescr = runner.arraydescrof(A) + self.fielddescr = runner.fielddescrof(A.OF, fieldname) def __repr__(self): return 'InteriorFieldDescr(%r, %r)' % (self.A, self.fieldname) @@ -397,8 +399,12 @@ try: return self.descrs[key] except KeyError: - descr = ArrayDescr(A) + descr = ArrayDescr(A, self) self.descrs[key] = descr + if isinstance(A.OF, lltype.Struct): + descrs = heaptracker.all_interiorfielddescrs(self, + A, get_field_descr=LLGraphCPU.interiorfielddescrof) + descr.all_interiorfielddescrs = descrs return descr def interiorfielddescrof(self, A, fieldname): @@ -406,7 +412,7 @@ try: return self.descrs[key] except KeyError: - descr = InteriorFieldDescr(A, fieldname) + descr = InteriorFieldDescr(A, fieldname, self) self.descrs[key] = descr return descr diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py --- a/rpython/jit/codewriter/heaptracker.py +++ b/rpython/jit/codewriter/heaptracker.py @@ -157,6 +157,25 @@ res.append(get_field_descr(gccache, STRUCT, name)) return res +def all_interiorfielddescrs(gccache, ARRAY, get_field_descr=None): + from rpython.jit.backend.llsupport import descr + + if get_field_descr is None: + get_field_descr = descr.get_field_descr + # order is not relevant, except for tests + STRUCT = ARRAY.OF + res = [] + for name in STRUCT._names: + FIELD = getattr(STRUCT, name) + if FIELD is lltype.Void: + continue + if name == 'typeptr': + continue # dealt otherwise + elif isinstance(FIELD, lltype.Struct): + raise Exception("unexpected array(struct(struct))") + res.append(get_field_descr(gccache, ARRAY, name)) + return res + def gc_fielddescrs(gccache, STRUCT): return all_fielddescrs(gccache, STRUCT, True) diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py --- a/rpython/jit/metainterp/optimizeopt/info.py +++ b/rpython/jit/metainterp/optimizeopt/info.py @@ -53,9 +53,8 @@ def is_nonnull(self): return True - -class AbstractStructPtrInfo(NonNullPtrInfo): - _attrs_ = ('_is_virtual', '_fields') +class AbstractVirtualPtrInfo(NonNullPtrInfo): + _attrs_ = ('_is_virtual',) def force_box(self, op, optforce): if self._is_virtual: @@ -65,18 +64,16 @@ op.set_forwarded(newop) newop.set_forwarded(self) self._is_virtual = False - if self._fields is not None: - descr = op.getdescr() - for i, flddescr in enumerate(descr.all_fielddescrs): - fld = self._fields[i] - if fld is not None: - subbox = optforce.force_box(fld) - setfieldop = ResOperation(rop.SETFIELD_GC, [op, subbox], - descr=flddescr) - optforce.emit_operation(setfieldop) + self._force_elements(newop, optforce) return newop return op + def is_virtual(self): + return self._is_virtual + +class AbstractStructPtrInfo(AbstractVirtualPtrInfo): + _attrs_ = ('_is_virtual', '_fields') + def init_fields(self, descr): self._fields = [None] * len(descr.all_fielddescrs) @@ -86,8 +83,17 @@ def getfield_virtual(self, descr): return self._fields[descr.index] - def is_virtual(self): - return self._is_virtual + def _force_elements(self, op, optforce): + if self._fields is None: + return + descr = op.getdescr() + for i, flddescr in enumerate(descr.all_fielddescrs): + fld = self._fields[i] + if fld is not None: + subbox = optforce.force_box(fld) + setfieldop = ResOperation(rop.SETFIELD_GC, [op, subbox], + descr=flddescr) + optforce.emit_operation(setfieldop) class InstancePtrInfo(AbstractStructPtrInfo): _attrs_ = ('_known_class') @@ -100,13 +106,44 @@ def get_known_class(self, cpu): return self._known_class -class StructPtrInfo(NonNullPtrInfo): - _attrs_ = ('is_virtual', '_fields') +class StructPtrInfo(AbstractStructPtrInfo): + pass + +class ArrayPtrInfo(AbstractVirtualPtrInfo): + _attrs_ = ('_is_virtual', 'length', '_items', '_descr') - -class ArrayPtrInfo(NonNullPtrInfo): - _attrs_ = ('is_virtual', 'length', '_items') +class ArrayStructInfo(ArrayPtrInfo): + def __init__(self, descr, size, is_virtual): + self.length = size + lgt = len(descr.all_interiorfielddescrs) + self._is_virtual = is_virtual + self._items = [None] * (size * lgt) + def _compute_index(self, index, fielddescr): + one_size = len(fielddescr.arraydescr.all_interiorfielddescrs) + return index * one_size + fielddescr.fielddescr.index + + def setinteriorfield_virtual(self, index, fielddescr, fld): + index = self._compute_index(index, fielddescr) + self._items[index] = fld + + def getinteriorfield_virtual(self, index, fielddescr): + index = self._compute_index(index, fielddescr) + return self._items[index] + + def _force_elements(self, op, optforce): + i = 0 + fielddescrs = op.getdescr().all_interiorfielddescrs + for index in range(self.length): + for flddescr in fielddescrs: + fld = self._items[i] + if fld is not None: + subbox = optforce.force_box(fld) + setfieldop = ResOperation(rop.SETINTERIORFIELD_GC, + [op, subbox], + descr=flddescr) + optforce.emit_operation(setfieldop) + i += 1 class StrPtrInfo(NonNullPtrInfo): _attrs_ = () diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -205,10 +205,11 @@ # Constant fold f0 * 1.0 and turn f0 * -1.0 into a FLOAT_NEG, these # work in all cases, including NaN and inf for lhs, rhs in [(arg1, arg2), (arg2, arg1)]: - v1 = self.getvalue(lhs) - v2 = self.getvalue(rhs) + v1 = self.get_box_replacement(lhs) + v2 = self.get_box_replacement(rhs) if v1.is_constant(): + xxxx if v1.box.getfloatstorage() == 1.0: self.make_equal_to(op, v2) return diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -531,6 +531,9 @@ def make_varray(self, arraydescr, size, source_op, clear=False): if arraydescr.is_array_of_structs(): assert clear + opinfo = info.ArrayStructInfo(arraydescr, size, True) + source_op.set_forwarded(opinfo) + return opinfo vvalue = VArrayStructValue(arraydescr, size, source_op) else: constvalue = self.new_const_item(arraydescr) @@ -697,7 +700,8 @@ def optimize_SETFIELD_GC(self, op): opinfo = self.getptrinfo(op.getarg(0)) if opinfo is not None and opinfo.is_virtual(): - opinfo.setfield_virtual(op.getdescr(), op.getarg(1)) + opinfo.setfield_virtual(op.getdescr(), + self.get_box_replacement(op.getarg(1))) else: self.make_nonnull(op.getarg(0)) self.emit_operation(op) @@ -718,8 +722,7 @@ def optimize_NEW_ARRAY_CLEAR(self, op): sizebox = self.get_constant_box(op.getarg(0)) if sizebox is not None: - self.make_varray(op.getdescr(), sizebox.getint(), op, - clear=True) + self.make_varray(op.getdescr(), sizebox.getint(), op, clear=True) else: self.emit_operation(op) @@ -893,32 +896,33 @@ self.emit_operation(op) def optimize_GETINTERIORFIELD_GC_I(self, op): - value = self.getvalue(op.getarg(0)) - if value.is_virtual(): + opinfo = self.getptrinfo(op.getarg(0)) + if opinfo and opinfo.is_virtual(): indexbox = self.get_constant_box(op.getarg(1)) if indexbox is not None: descr = op.getdescr() - fieldvalue = value.getinteriorfield( - indexbox.getint(), descr, None - ) - if fieldvalue is None: + fld = opinfo.getinteriorfield_virtual(indexbox.getint(), descr) + if fld is None: + xxx fieldvalue = self.new_const(descr) - self.make_equal_to(op, fieldvalue) + self.make_equal_to(op, fld) return + xxx value.ensure_nonnull() self.emit_operation(op) optimize_GETINTERIORFIELD_GC_R = optimize_GETINTERIORFIELD_GC_I optimize_GETINTERIORFIELD_GC_F = optimize_GETINTERIORFIELD_GC_I def optimize_SETINTERIORFIELD_GC(self, op): - value = self.getvalue(op.getarg(0)) - if value.is_virtual(): + opinfo = self.getptrinfo(op.getarg(0)) + if opinfo and opinfo.is_virtual(): indexbox = self.get_constant_box(op.getarg(1)) if indexbox is not None: - value.setinteriorfield( - indexbox.getint(), op.getdescr(), self.getvalue(op.getarg(2)) - ) + opinfo.setinteriorfield_virtual(indexbox.getint(), + op.getdescr(), + self.get_box_replacement(op.getarg(2))) return + xxx value.ensure_nonnull() self.emit_operation(op) From noreply at buildbot.pypy.org Sun Mar 8 10:40:12 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 8 Mar 2015 10:40:12 +0100 (CET) Subject: [pypy-commit] pypy default: Print some info to tell which virtualizable array field the error is about Message-ID: <20150308094012.DF9EF1C0107@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76279:a1abf06bc4ed Date: 2015-03-08 10:40 +0100 http://bitbucket.org/pypy/pypy/changeset/a1abf06bc4ed/ Log: Print some info to tell which virtualizable array field the error is about diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -143,13 +143,16 @@ return for v in list: if v in self.vable_array_vars: + vars = self.vable_array_vars[v] + (v_base, arrayfielddescr, arraydescr) = vars raise AssertionError( "A virtualizable array is passed around; it should\n" "only be used immediately after being read. Note\n" "that a possible cause is indexing with an index not\n" "known non-negative, or catching IndexError, or\n" "not inlining at all (for tests: use listops=True).\n" - "Occurred in: %r" % self.graph) + "This is about: %r\n" + "Occurred in: %r" % (arrayfielddescr, self.graph)) # extra explanation: with the way things are organized in # rpython/rlist.py, the ll_getitem becomes a function call # that is typically meant to be inlined by the JIT, but diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py --- a/rpython/jit/codewriter/test/test_flatten.py +++ b/rpython/jit/codewriter/test/test_flatten.py @@ -999,6 +999,7 @@ e = py.test.raises(AssertionError, self.encoding_test, f, [], "!", transform=True) assert str(e.value).startswith("A virtualizable array is passed aroun") + assert "" in str(e.value) def test_vable_attribute_list_copied_around(self): class F: @@ -1014,6 +1015,7 @@ e = py.test.raises(AssertionError, self.encoding_test, f, [], "!", transform=True) assert str(e.value).startswith("A virtualizable array is passed aroun") + assert "" in str(e.value) def check_force_cast(FROM, TO, operations, value): From noreply at buildbot.pypy.org Sun Mar 8 18:03:36 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 8 Mar 2015 18:03:36 +0100 (CET) Subject: [pypy-commit] pypy vmprof: uh Message-ID: <20150308170336.C0ECD1C11F1@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof Changeset: r76280:4c0fa46303da Date: 2015-03-08 18:03 +0100 http://bitbucket.org/pypy/pypy/changeset/4c0fa46303da/ Log: uh diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -141,4 +141,4 @@ try: return space.wrap(ec.code_info_file.read()) finally: - ec.code_info_file.seek(2, 0) + ec.code_info_file.seek(0, 2) From noreply at buildbot.pypy.org Sun Mar 8 18:42:34 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 8 Mar 2015 18:42:34 +0100 (CET) Subject: [pypy-commit] stmgc c8-marker: in-progress Message-ID: <20150308174234.56D091C0196@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-marker Changeset: r1696:7d44874ec01b Date: 2015-03-08 18:43 +0100 http://bitbucket.org/pypy/stmgc/changeset/7d44874ec01b/ Log: in-progress diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -49,6 +49,8 @@ DEBUG_EXPECT_SEGFAULT(false); for (; undo < end; undo++) { + if (undo->type == TYPE_POSITION_MARKER) + continue; object_t *obj = undo->object; stm_char *oslice = ((stm_char *)obj) + SLICE_OFFSET(undo->slice); uintptr_t current_page_num = ((uintptr_t)oslice) / 4096; @@ -269,6 +271,11 @@ struct stm_undo_s *undo = cl->written; struct stm_undo_s *end = undo + cl->written_count; for (; undo < end; undo++) { + if (undo->type == TYPE_POSITION_MARKER) { + fprintf(stderr, " marker %p %lu\n", + undo->marker_object, undo->marker_odd_number); + continue; + } fprintf(stderr, " obj %p, size %d, ofs %lu: ", undo->object, SLICE_SIZE(undo->slice), SLICE_OFFSET(undo->slice)); /* long i; */ @@ -369,6 +376,7 @@ the old (but unmodified) version to the newer version. */ reset_modified_from_backup_copies(my_segnum); + timing_write_read_contention(cl->written, undo); needs_abort = true; dprintf(("_stm_validate() failed for obj %p\n", undo->object)); @@ -528,7 +536,6 @@ STM_PSEGMENT->safe_point = SP_NO_TRANSACTION; list_clear(STM_PSEGMENT->modified_old_objects); - list_clear(STM_PSEGMENT->modified_old_objects_markers); STM_PSEGMENT->last_commit_log_entry = new; release_modification_lock(STM_SEGMENT->segment_num); } @@ -558,7 +565,6 @@ STM_PSEGMENT->transaction_state = TS_NONE; STM_PSEGMENT->safe_point = SP_NO_TRANSACTION; list_clear(STM_PSEGMENT->modified_old_objects); - list_clear(STM_PSEGMENT->modified_old_objects_markers); STM_PSEGMENT->last_commit_log_entry = new; /* do it: */ @@ -601,6 +607,8 @@ { dprintf(("make_bk_slices_for_range(%p, %lu, %lu)\n", obj, start - (stm_char*)obj, end - start)); + timing_record_write_position(); + char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); uintptr_t first_page = ((uintptr_t)start) / 4096UL; uintptr_t end_page = ((uintptr_t)end) / 4096UL; @@ -632,7 +640,6 @@ (uintptr_t)obj, /* obj */ (uintptr_t)bk_slice, /* bk_addr */ NEW_SLICE(slice_off, slice_sz)); - timing_record_write(); dprintf(("> append slice %p, off=%lu, sz=%lu\n", bk_slice, slice_off, slice_sz)); release_modification_lock(STM_SEGMENT->segment_num); @@ -1075,7 +1082,6 @@ } assert(list_is_empty(STM_PSEGMENT->modified_old_objects)); - assert(list_is_empty(STM_PSEGMENT->modified_old_objects_markers)); assert(list_is_empty(STM_PSEGMENT->large_overflow_objects)); assert(list_is_empty(STM_PSEGMENT->objects_pointing_to_nursery)); assert(list_is_empty(STM_PSEGMENT->young_weakrefs)); @@ -1087,7 +1093,7 @@ assert(STM_PSEGMENT->finalizers == NULL); #ifndef NDEBUG /* this should not be used when objects_pointing_to_nursery == NULL */ - STM_PSEGMENT->modified_old_objects_markers_num_old = 99999999999999999L; + STM_PSEGMENT->position_markers_len_old = 99999999999999999L; #endif check_nursery_at_transaction_start(); @@ -1270,6 +1276,8 @@ struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count); for (; undo < end; undo++) { + if (undo->type == TYPE_POSITION_MARKER) + continue; object_t *obj = undo->object; char *dst = REAL_ADDRESS(pseg->pub.segment_base, obj); diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -84,21 +84,18 @@ track the STM status: these are old objects that where written to and that will need to be recorded in the commit log. The list contains three entries for every such object, in the same - format as 'struct stm_undo_s' below. + format as 'struct stm_undo_s' below. It can also represent a + position marker, like 'struct stm_undo_s'. */ struct list_s *modified_old_objects; + uintptr_t position_markers_last; /* index of most recent pos marker */ + uintptr_t position_markers_len_old; /* length of list at last minor col */ struct list_s *objects_pointing_to_nursery; struct list_s *old_objects_with_cards_set; struct tree_s *young_outside_nursery; struct tree_s *nursery_objects_shadows; - /* For each entry in 'modified_old_objects', we have two entries - in the following list, which give the marker at the time we added - the entry to modified_old_objects. */ - struct list_s *modified_old_objects_markers; - uintptr_t modified_old_objects_markers_num_old; - /* List of all young weakrefs to check in minor collections. These are the only weakrefs that may point to young objects and never contain NULL. */ @@ -180,18 +177,27 @@ /* Commit Log things */ struct stm_undo_s { - object_t *object; /* the object that is modified */ - char *backup; /* some backup data (a slice of the original obj) */ - uint64_t slice; /* location and size of this slice (cannot cross - pages). The size is in the lower 2 bytes, and - the offset in the remaining 6 bytes. */ + union { + struct { + object_t *object; /* the object that is modified */ + char *backup; /* some backup data (a slice of the original obj) */ + uint64_t slice; /* location and size of this slice (cannot cross + pages). The size is in the lower 2 bytes, and + the offset in the remaining 6 bytes. */ + }; + struct { + intptr_t type; /* TYPE_POSITION_MARKER */ + uintptr_t marker_odd_number; /* the odd number part of the marker */ + object_t *marker_object; /* the object part of the marker */ + }; + }; }; +#define TYPE_POSITION_MARKER (-1) #define SLICE_OFFSET(slice) ((slice) >> 16) #define SLICE_SIZE(slice) ((int)((slice) & 0xFFFF)) #define NEW_SLICE(offset, size) (((uint64_t)(offset)) << 16 | (size)) - /* The model is: we have a global chained list, from 'commit_log_root', of 'struct stm_commit_log_entry_s' entries. Every one is fully read-only apart from the 'next' field. Every one stands for one diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -348,6 +348,8 @@ struct stm_undo_s *modified = (struct stm_undo_s *)lst->items; struct stm_undo_s *end = (struct stm_undo_s *)(lst->items + lst->count); for (; modified < end; modified++) { + if (modified->type == TYPE_POSITION_MARKER) + continue; object_t *obj = modified->object; struct object_s *dst = (struct object_s*)REAL_ADDRESS(base, obj); @@ -388,14 +390,16 @@ static void mark_visit_from_markers(void) { - long j; - for (j = 1; j < NB_SEGMENTS; j++) { - struct stm_priv_segment_info_s *pseg = get_priv_segment(j); - struct list_s *lst = pseg->modified_old_objects_markers; - uintptr_t i; - for (i = list_count(lst); i > 0; i -= 2) { - mark_visit_possibly_new_object((object_t *)list_item(lst, i - 1), - pseg); + long i; + for (i = 1; i < NB_SEGMENTS; i++) { + struct stm_priv_segment_info_s *pseg = get_priv_segment(i); + struct list_s *lst = get_priv_segment(i)->modified_old_objects; + + struct stm_undo_s *modified = (struct stm_undo_s *)lst->items; + struct stm_undo_s *end = (struct stm_undo_s *)(lst->items + lst->count); + for (; modified < end; modified++) { + if (modified->type == TYPE_POSITION_MARKER) + mark_visit_possibly_new_object(modified->marker_object, pseg); } } } diff --git a/c8/stm/marker.c b/c8/stm/marker.c --- a/c8/stm/marker.c +++ b/c8/stm/marker.c @@ -3,11 +3,10 @@ #endif -static void marker_fetch(stm_loc_marker_t *out_marker) +static bool marker_fetch(stm_thread_local_t *tl, stm_loc_marker_t *out_marker) { - /* Fetch the current marker from the 'out_marker->tl's shadow stack, + /* Fetch the current marker from tl's shadow stack, and return it in 'out_marker->odd_number' and 'out_marker->object'. */ - stm_thread_local_t *tl = out_marker->tl; struct stm_shadowentry_s *current = tl->shadowstack - 1; struct stm_shadowentry_s *base = tl->shadowstack_base; @@ -23,68 +22,69 @@ /* found the odd marker */ out_marker->odd_number = (uintptr_t)current[0].ss; out_marker->object = current[1].ss; + return true; } else { /* no marker found */ - out_marker->odd_number = 0; - out_marker->object = NULL; + return false; } } -static void marker_fetch_obj_write(object_t *obj, stm_loc_marker_t *out_marker) +static void marker_fetch_obj_write(struct stm_undo_s *start, + struct stm_undo_s *contention, + stm_loc_marker_t *out_marker) { - /* From 'out_marker->tl', fill in 'out_marker->segment_base' and - 'out_marker->odd_number' and 'out_marker->object' from the - marker associated with writing the 'obj'. + /* Fill out 'out_marker->odd_number' and 'out_marker->object' from + the marker just before 'contention' in the list starting at + 'start'. */ - assert(_has_mutex()); - - long i, num; - int in_segment_num = out_marker->tl->associated_segment_num; - assert(in_segment_num >= 1); - struct stm_priv_segment_info_s *pseg = get_priv_segment(in_segment_num); - struct list_s *mlst = pseg->modified_old_objects; - struct list_s *mlstm = pseg->modified_old_objects_markers; - num = list_count(mlstm) / 2; - assert(num * 3 <= list_count(mlst)); - for (i = 0; i < num; i++) { - if (list_item(mlst, i * 3) == (uintptr_t)obj) { - out_marker->odd_number = list_item(mlstm, i * 2 + 0); - out_marker->object = (object_t *)list_item(mlstm, i * 2 + 1); + while (contention != start) { + --contention; + if (contention->type == TYPE_POSITION_MARKER) { + out_marker->odd_number = contention->marker_odd_number; + out_marker->object = contention->marker_object; return; } } + /* no position marker found... */ out_marker->odd_number = 0; out_marker->object = NULL; } -static void _timing_record_write(void) +static void _timing_record_write_position(void) { stm_loc_marker_t marker; - marker.tl = STM_SEGMENT->running_thread; - marker_fetch(&marker); + if (!marker_fetch(STM_SEGMENT->running_thread, &marker)) + return; - long base_count = list_count(STM_PSEGMENT->modified_old_objects) / 3; - struct list_s *mlstm = STM_PSEGMENT->modified_old_objects_markers; - while (list_count(mlstm) < 2 * base_count) { - mlstm = list_append2(mlstm, 0, 0); + struct list_s *list = STM_PSEGMENT->modified_old_objects; + uintptr_t i = STM_PSEGMENT->position_markers_last; + if (i < list_count(list)) { + struct stm_undo_s *undo = (struct stm_undo_s *)(list->items + i); + if (undo->type == TYPE_POSITION_MARKER && + undo->marker_odd_number == marker.odd_number && + undo->marker_object == marker.object) + return; /* already up-to-date */ } - mlstm = list_append2(mlstm, marker.odd_number, (uintptr_t)marker.object); - STM_PSEGMENT->modified_old_objects_markers = mlstm; + + STM_PSEGMENT->position_markers_last = list_count(list); + STM_PSEGMENT->modified_old_objects = list_append3( + list, + TYPE_POSITION_MARKER, /* type */ + marker.odd_number, /* marker_odd_number */ + (uintptr_t)marker.object); /* marker_object */ } -static void timing_write_read_contention(object_t *obj) +static void timing_write_read_contention(struct stm_undo_s *start, + struct stm_undo_s *contention) { if (stmcb_timing_event == NULL) return; - /* Collect the older location of the write from the current thread. */ stm_loc_marker_t marker; - marker.tl = STM_SEGMENT->running_thread; - marker.segment_base = STM_SEGMENT->segment_base; - marker_fetch_obj_write(obj, &marker); - - stmcb_timing_event(marker.tl, STM_CONTENTION_WRITE_READ, &marker); + marker_fetch_obj_write(start, contention, &marker); + stmcb_timing_event(STM_SEGMENT->running_thread, + STM_CONTENTION_WRITE_READ, &marker); } diff --git a/c8/stm/marker.h b/c8/stm/marker.h --- a/c8/stm/marker.h +++ b/c8/stm/marker.h @@ -1,10 +1,11 @@ -static void _timing_record_write(void); -static void timing_write_read_contention(object_t *obj); +static void _timing_record_write_position(void); +static void timing_write_read_contention(struct stm_undo_s *start, + struct stm_undo_s *contention); #define timing_event(tl, event) \ (stmcb_timing_event != NULL ? stmcb_timing_event(tl, event, NULL) : (void)0) -#define timing_record_write() \ - (stmcb_timing_event != NULL ? _timing_record_write() : (void)0) +#define timing_record_write_position() \ + (stmcb_timing_event != NULL ? _timing_record_write_position() : (void)0) diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c --- a/c8/stm/nursery.c +++ b/c8/stm/nursery.c @@ -410,16 +410,18 @@ } } -static void collect_roots_from_markers(uintptr_t num_old) +static void collect_roots_from_markers(uintptr_t len_old) { dprintf(("collect_roots_from_markers\n")); + /* visit the marker objects */ - struct list_s *mlst = STM_PSEGMENT->modified_old_objects_markers; - STM_PSEGMENT->modified_old_objects_markers_num_old = list_count(mlst); - uintptr_t i, total = list_count(mlst); - assert((total & 1) == 0); - for (i = num_old + 1; i < total; i += 2) { - minor_trace_if_young((object_t **)list_ptr_to_item(mlst, i)); + struct list_s *list = STM_PSEGMENT->modified_old_objects; + struct stm_undo_s *undo = (struct stm_undo_s *)(list->items + len_old); + struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count); + + for (; undo < end; undo++) { + if (undo->type == TYPE_POSITION_MARKER) + minor_trace_if_young(&undo->marker_object); } } @@ -507,21 +509,23 @@ STM_PSEGMENT->minor_collect_will_commit_now = commit; - uintptr_t num_old; + uintptr_t len_old; if (STM_PSEGMENT->overflow_number_has_been_used) - num_old = STM_PSEGMENT->modified_old_objects_markers_num_old; + len_old = STM_PSEGMENT->position_markers_len_old; else - num_old = 0; + len_old = 0; if (!commit) { /* 'STM_PSEGMENT->overflow_number' is used now by this collection, in the sense that it's copied to the overflow objects */ STM_PSEGMENT->overflow_number_has_been_used = true; + STM_PSEGMENT->position_markers_len_old = + list_count(STM_PSEGMENT->modified_old_objects); } collect_cardrefs_to_nursery(); - collect_roots_from_markers(num_old); + collect_roots_from_markers(len_old); collect_roots_in_nursery(); diff --git a/c8/stm/prof.c b/c8/stm/prof.c new file mode 100644 --- /dev/null +++ b/c8/stm/prof.c @@ -0,0 +1,122 @@ +#include +#include + + +static FILE *profiling_file; +static char *profiling_basefn = NULL; +static stm_expand_marker_fn profiling_expand_marker; + +#define MARKER_LEN_MAX 160 + + +static bool close_timing_log(void); /* forward */ + +static void _stm_profiling_event(stm_thread_local_t *tl, + enum stm_event_e event, + stm_loc_marker_t *marker) +{ + struct buf_s { + uint32_t tv_sec; + uint32_t tv_nsec; + uint32_t thread_num; + uint8_t event; + uint8_t marker_length; + char extra[MARKER_LEN_MAX+1]; + } __attribute__((packed)); + + struct buf_s buf; + struct timespec t; + clock_gettime(CLOCK_MONOTONIC, &t); + buf.tv_sec = t.tv_sec; + buf.tv_nsec = t.tv_nsec; + buf.thread_num = tl->thread_local_counter; + buf.event = event; + buf.marker_length = 0; + + if (marker != NULL && marker->odd_number != 0) { + buf.marker_length = profiling_expand_marker(get_segment_base(0), + marker, + buf.extra, MARKER_LEN_MAX); + } + + if (fwrite(&buf, offsetof(struct buf_s, extra) + buf.marker_length, + 1, profiling_file) != 1) { + fprintf(stderr, "stmgc: profiling log file closed unexpectedly: %m\n"); + close_timing_log(); + } +} + +static int default_expand_marker(stm_loc_marker_t *m, char *p, int s) +{ + *(uintptr_t *)p = m->odd_number; + return sizeof(uintptr_t); +} + +static bool open_timing_log(const char *filename) +{ + profiling_file = fopen(filename, "w"); + if (profiling_file == NULL) + return false; + + fwrite("STMGC-C7-PROF01\n", 16, 1, profiling_file); + stmcb_timing_event = _stm_profiling_event; + return true; +} + +static bool close_timing_log(void) +{ + if (stmcb_timing_event == &_stm_profiling_event) { + stmcb_timing_event = NULL; + fclose(profiling_file); + profiling_file = NULL; + return true; + } + return false; +} + +static void prof_forksupport_prepare(void) +{ + if (profiling_file != NULL) + fflush(profiling_file); +} + +static void prof_forksupport_child(void) +{ + if (close_timing_log() && profiling_basefn != NULL) { + char filename[1024]; + snprintf(filename, sizeof(filename), + "%s.fork%ld", profiling_basefn, (long)getpid()); + open_timing_log(filename); + } +} + +int stm_set_timing_log(const char *profiling_file_name, int fork_mode, + stm_expand_marker_fn expand_marker) +{ + close_timing_log(); + free(profiling_basefn); + profiling_basefn = NULL; + + if (profiling_file_name == NULL) + return 0; + + if (!expand_marker) + expand_marker = default_expand_marker; + profiling_expand_marker = expand_marker; + + static bool fork_support_ready = false; + if (!fork_support_ready) { + int res = pthread_atfork(prof_forksupport_prepare, + NULL, prof_forksupport_child); + if (res != 0) + stm_fatalerror("pthread_atfork() failed: %m"); + fork_support_ready = true; + } + + if (!open_timing_log(profiling_file_name)) + return -1; + + if (fork_mode != 0) + profiling_basefn = strdup(profiling_file_name); + return 0; +} diff --git a/c8/stm/setup.c b/c8/stm/setup.c --- a/c8/stm/setup.c +++ b/c8/stm/setup.c @@ -100,7 +100,6 @@ pr->pub.segment_num = i; pr->pub.segment_base = segment_base; pr->modified_old_objects = list_create(); - pr->modified_old_objects_markers = list_create(); pr->large_overflow_objects = list_create(); pr->young_weakrefs = list_create(); pr->old_weakrefs = list_create(); @@ -153,7 +152,6 @@ list_free(pr->objects_pointing_to_nursery); list_free(pr->old_objects_with_cards_set); list_free(pr->modified_old_objects); - list_free(pr->modified_old_objects_markers); assert(list_is_empty(pr->large_overflow_objects)); list_free(pr->large_overflow_objects); list_free(pr->young_weakrefs); @@ -223,6 +221,8 @@ return (pthread_t *)(tl->creating_pthread); } +static int thread_local_counters = 0; + void stm_register_thread_local(stm_thread_local_t *tl) { int num; @@ -244,6 +244,7 @@ numbers automatically. */ tl->associated_segment_num = -1; tl->last_associated_segment_num = num + 1; + tl->thread_local_counter = ++thread_local_counters; *_get_cpth(tl) = pthread_self(); _init_shadow_stack(tl); set_gs_register(get_segment_base(num + 1)); diff --git a/c8/stmgc.c b/c8/stmgc.c --- a/c8/stmgc.c +++ b/c8/stmgc.c @@ -36,5 +36,6 @@ #include "stm/extra.c" #include "stm/fprintcolor.c" #include "stm/marker.c" +#include "stm/prof.c" #include "stm/rewind_setjmp.c" #include "stm/finalizer.c" diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -68,6 +68,7 @@ /* the next fields are handled internally by the library */ int associated_segment_num; int last_associated_segment_num; + int thread_local_counter; struct stm_thread_local_s *prev, *next; void *creating_pthread[2]; } stm_thread_local_t; @@ -381,12 +382,10 @@ "gc major done" /* The markers pushed in the shadowstack are an odd number followed by a - regular pointer. */ + regular object pointer. */ typedef struct { - stm_thread_local_t *tl; - char *segment_base; /* base to interpret the 'object' below */ - uintptr_t odd_number; /* marker odd number, or 0 if marker is missing */ - object_t *object; /* marker object, or NULL if marker is missing */ + uintptr_t odd_number; + object_t *object; } stm_loc_marker_t; extern void (*stmcb_timing_event)(stm_thread_local_t *tl, /* the local thread */ enum stm_event_e event, @@ -399,10 +398,12 @@ 'profiling_file_name.fork' after a fork(). Call it with NULL to stop profiling. Returns -1 in case of error (see errno then). The optional 'expand_marker' function pointer is called to expand - the marker's odd_number and object into data, starting at the given - position and with the given maximum length. */ + the marker's odd_number and object into printable data, starting at + the given position and with the given maximum length. */ +typedef int (*stm_expand_marker_fn)(char *seg_base, stm_loc_marker_t *marker, + char *output, int output_size); int stm_set_timing_log(const char *profiling_file_name, int fork_mode, - int expand_marker(stm_loc_marker_t *, char *, int)); + stm_expand_marker_fn expand_marker); /* Convenience macros to push the markers into the shadowstack */ diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -149,10 +149,6 @@ }; typedef struct { - stm_thread_local_t *tl; - /* If segment_base==NULL, the remaining fields are undefined. If non-NULL, - the rest is a marker to interpret from this segment_base addr. */ - char *segment_base; uintptr_t odd_number; object_t *object; } stm_loc_marker_t; @@ -162,8 +158,10 @@ stm_loc_marker_t *markers); stmcb_timing_event_fn stmcb_timing_event; -int stm_set_timing_log(const char *profiling_file_name, int prof_mode, - int expand_marker(stm_loc_marker_t *, char *, int)); +typedef int (*stm_expand_marker_fn)(char *seg_base, stm_loc_marker_t *marker, + char *output, int output_size); +int stm_set_timing_log(const char *profiling_file_name, int fork_mode, + stm_expand_marker_fn expand_marker); long _stm_count_modified_old_objects(void); long _stm_count_objects_pointing_to_nursery(void); From noreply at buildbot.pypy.org Sun Mar 8 18:45:17 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 8 Mar 2015 18:45:17 +0100 (CET) Subject: [pypy-commit] stmgc c8-marker: fix Message-ID: <20150308174517.028911C0196@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-marker Changeset: r1697:f74408440140 Date: 2015-03-08 18:45 +0100 http://bitbucket.org/pypy/stmgc/changeset/f74408440140/ Log: fix diff --git a/c8/stm/prof.c b/c8/stm/prof.c --- a/c8/stm/prof.c +++ b/c8/stm/prof.c @@ -46,7 +46,7 @@ } } -static int default_expand_marker(stm_loc_marker_t *m, char *p, int s) +static int default_expand_marker(char *b, stm_loc_marker_t *m, char *p, int s) { *(uintptr_t *)p = m->odd_number; return sizeof(uintptr_t); From noreply at buildbot.pypy.org Sun Mar 8 18:58:35 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 8 Mar 2015 18:58:35 +0100 (CET) Subject: [pypy-commit] stmgc c8-marker: progress Message-ID: <20150308175835.D6AED1C071E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-marker Changeset: r1698:75ffbf6c36cb Date: 2015-03-08 18:59 +0100 http://bitbucket.org/pypy/stmgc/changeset/75ffbf6c36cb/ Log: progress diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -369,6 +369,8 @@ struct stm_undo_s *undo = cl->written; struct stm_undo_s *end = cl->written + cl->written_count; for (; undo < end; undo++) { + if (undo->type == TYPE_POSITION_MARKER) + continue; if (_stm_was_read(undo->object)) { /* first reset all modified objects from the backup copies as soon as the first conflict is detected; @@ -1031,6 +1033,8 @@ struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count); for (; undo < end; undo++) { + if (undo->type == TYPE_POSITION_MARKER) + continue; object_t *obj = undo->object; obj->stm_flags &= ~GCFLAG_WB_EXECUTED; } @@ -1044,6 +1048,8 @@ struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count); for (; undo < end; undo++) { + if (undo->type == TYPE_POSITION_MARKER) + continue; object_t *obj = undo->object; obj->stm_flags |= GCFLAG_WB_EXECUTED; } @@ -1163,6 +1169,8 @@ struct stm_undo_s *undo = (struct stm_undo_s *)list->items; struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count); for (; undo < end; undo++) { + if (undo->type == TYPE_POSITION_MARKER) + continue; object_t *obj = undo->object; struct object_s *dst = (struct object_s*)REAL_ADDRESS(segbase, obj); assert(dst->stm_flags & GCFLAG_WRITE_BARRIER); diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c --- a/c8/stm/nursery.c +++ b/c8/stm/nursery.c @@ -185,6 +185,8 @@ struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count); for (; undo < end; undo++) { + if (undo->type == TYPE_POSITION_MARKER) + continue; _cards_cleared_in_object(pseg, undo->object, false); } LIST_FOREACH_R( diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -163,6 +163,10 @@ int stm_set_timing_log(const char *profiling_file_name, int fork_mode, stm_expand_marker_fn expand_marker); +void stm_push_marker(stm_thread_local_t *, uintptr_t, object_t *); +void stm_update_marker_num(stm_thread_local_t *, uintptr_t); +void stm_pop_marker(stm_thread_local_t *); + long _stm_count_modified_old_objects(void); long _stm_count_objects_pointing_to_nursery(void); long _stm_count_old_objects_with_cards_set(void); @@ -185,7 +189,6 @@ void stm_enable_light_finalizer(object_t *); void (*stmcb_finalizer)(object_t *); - """) @@ -412,6 +415,21 @@ } } +void stm_push_marker(stm_thread_local_t *tl, uintptr_t onum, object_t *ob) +{ + STM_PUSH_MARKER(*tl, onum, ob); +} + +void stm_update_marker_num(stm_thread_local_t *tl, uintptr_t onum) +{ + STM_UPDATE_MARKER_NUM(*tl, onum); +} + +void stm_pop_marker(stm_thread_local_t *tl) +{ + STM_POP_MARKER(*tl); +} + long current_segment_num(void) { return STM_SEGMENT->segment_num; diff --git a/c8/test/test_marker.py b/c8/test/test_marker.py --- a/c8/test/test_marker.py +++ b/c8/test/test_marker.py @@ -7,16 +7,11 @@ def recording(self, *kinds): seen = [] @ffi.callback("stmcb_timing_event_fn") - def timing_event(tl, event, markers): + def timing_event(tl, event, marker): if len(kinds) > 0 and event not in kinds: return - if markers: - expanded = [] - for i in range(2): - expanded.append((markers[i].tl, - markers[i].segment_base, - markers[i].odd_number, - markers[i].object)) + if marker: + expanded = (marker.odd_number, marker.object) else: expanded = None seen.append((tl, event, expanded)) @@ -24,13 +19,11 @@ self.timing_event_keepalive = timing_event self.seen = seen - def check_recording(self, i1, o1, i2, o2, extra=None): + def check_recording(self, i1, o1, extra=None): seen = self.seen - tl, event, markers = seen[0] + tl, event, marker = seen[0] assert tl == self.tls[1] - segbase = lib._stm_get_segment_base - assert markers[0] == (self.tls[1], segbase(2), i1, o1) - assert markers[1] == (self.tls[0], segbase(1), i2, o2) + assert marker == (i1, o1) if extra is None: assert len(seen) == 1 else: @@ -47,9 +40,7 @@ assert int(ffi.cast("uintptr_t", x)) == 29 def test_abort_marker_no_shadowstack(self): - self.recording(lib.STM_CONTENTION_WRITE_WRITE, - lib.STM_WAIT_CONTENTION, - lib.STM_ABORTING_OTHER_CONTENTION) + self.recording(lib.STM_CONTENTION_WRITE_READ) p = stm_allocate_old(16) # self.start_transaction() @@ -57,9 +48,13 @@ # self.switch(1) self.start_transaction() - py.test.raises(Conflict, stm_set_char, p, 'B') + stm_set_char(p, 'B') # - self.check_recording(0, ffi.NULL, 0, ffi.NULL) + self.switch(0) + self.commit_transaction() + # + py.test.raises(Conflict, self.switch, 1) + self.check_recording(0, ffi.NULL) def test_macros(self): self.start_transaction() @@ -96,8 +91,9 @@ py.test.raises(EmptyStack, self.pop_root) def test_double_abort_markers_cb_write_write(self): - self.recording(lib.STM_CONTENTION_WRITE_WRITE) + self.recording(lib.STM_CONTENTION_WRITE_READ) p = stm_allocate_old(16) + p2 = stm_allocate_old(16) # self.start_transaction() self.push_root(ffi.cast("object_t *", 19)) @@ -107,15 +103,21 @@ self.pop_root() self.push_root(ffi.cast("object_t *", 17)) self.push_root(ffi.cast("object_t *", ffi.NULL)) + stm_set_char(p, 'B') + stm_set_char(p2, 'C') stm_minor_collect() # self.switch(1) self.start_transaction() self.push_root(ffi.cast("object_t *", 21)) self.push_root(ffi.cast("object_t *", ffi.NULL)) - py.test.raises(Conflict, stm_set_char, p, 'B') + stm_set_char(p, 'B') # - self.check_recording(21, ffi.NULL, 19, ffi.NULL) + self.switch(0) + self.commit_transaction() + # + py.test.raises(Conflict, self.switch, 1) + self.check_recording(19, ffi.NULL) def test_double_abort_markers_cb_inevitable(self): self.recording(lib.STM_CONTENTION_INEVITABLE) From noreply at buildbot.pypy.org Sun Mar 8 19:01:45 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 8 Mar 2015 19:01:45 +0100 (CET) Subject: [pypy-commit] stmgc c8-marker: fix the next test Message-ID: <20150308180145.617B81C071E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-marker Changeset: r1699:a93899e6c43e Date: 2015-03-08 19:02 +0100 http://bitbucket.org/pypy/stmgc/changeset/a93899e6c43e/ Log: fix the next test diff --git a/c8/test/test_marker.py b/c8/test/test_marker.py --- a/c8/test/test_marker.py +++ b/c8/test/test_marker.py @@ -149,10 +149,6 @@ p = stm_allocate_old(16) # self.start_transaction() - assert stm_get_char(p) == '\x00' - # - self.switch(1) - self.start_transaction() self.push_root(ffi.cast("object_t *", 19)) self.push_root(ffi.cast("object_t *", ffi.NULL)) stm_set_char(p, 'A') @@ -160,9 +156,16 @@ self.pop_root() self.push_root(ffi.cast("object_t *", 17)) self.push_root(ffi.cast("object_t *", ffi.NULL)) - py.test.raises(Conflict, self.commit_transaction) # - self.check_recording(19, ffi.NULL, 0, ffi.NULL) + self.switch(1) + self.start_transaction() + assert stm_get_char(p) == '\x00' + # + self.switch(0) + self.commit_transaction() + # + py.test.raises(Conflict, self.switch, 1) + self.check_recording(19, ffi.NULL) def test_double_remote_markers_cb_write_write(self): self.recording(lib.STM_CONTENTION_WRITE_WRITE, From noreply at buildbot.pypy.org Sun Mar 8 19:32:25 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 8 Mar 2015 19:32:25 +0100 (CET) Subject: [pypy-commit] pypy default: Add another hack around the hack of the global dictionary, Message-ID: <20150308183225.81B571C0221@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76281:537301906e33 Date: 2015-03-08 19:32 +0100 http://bitbucket.org/pypy/pypy/changeset/537301906e33/ Log: Add another hack around the hack of the global dictionary, for test_ztranslation diff --git a/pypy/module/_ssl/__init__.py b/pypy/module/_ssl/__init__.py --- a/pypy/module/_ssl/__init__.py +++ b/pypy/module/_ssl/__init__.py @@ -51,6 +51,11 @@ super(Module, cls).buildloaders() + def setup_after_space_initialization(self): + """NOT_RPYTHON""" + from pypy.module._ssl.interp_ssl import PWINFO_STORAGE + PWINFO_STORAGE.clear() + def startup(self, space): from rpython.rlib.ropenssl import init_ssl init_ssl() diff --git a/pypy/objspace/fake/checkmodule.py b/pypy/objspace/fake/checkmodule.py --- a/pypy/objspace/fake/checkmodule.py +++ b/pypy/objspace/fake/checkmodule.py @@ -10,6 +10,7 @@ mod = __import__('pypy.module.%s' % modname, None, None, ['__doc__']) # force computation and record what we wrap module = mod.Module(space, W_Root()) + module.setup_after_space_initialization() module.startup(space) for name in module.loaders: seeobj_w.append(module._load_lazily(space, name)) From noreply at buildbot.pypy.org Sun Mar 8 19:43:49 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 8 Mar 2015 19:43:49 +0100 (CET) Subject: [pypy-commit] pypy default: update the version number Message-ID: <20150308184349.36D281C0221@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76282:c7810718bc96 Date: 2015-03-08 19:43 +0100 http://bitbucket.org/pypy/pypy/changeset/c7810718bc96/ Log: update the version number diff --git a/lib_pypy/cffi.egg-info b/lib_pypy/cffi.egg-info --- a/lib_pypy/cffi.egg-info +++ b/lib_pypy/cffi.egg-info @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: cffi -Version: 0.9.0 +Version: 0.9.1 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski From noreply at buildbot.pypy.org Sun Mar 8 19:47:49 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 8 Mar 2015 19:47:49 +0100 (CET) Subject: [pypy-commit] pypy default: Add a test that fails here if we forget to update lib_pypy/cffi.egg-info Message-ID: <20150308184749.74E8F1C071E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76283:2849115f3479 Date: 2015-03-08 19:47 +0100 http://bitbucket.org/pypy/pypy/changeset/2849115f3479/ Log: Add a test that fails here if we forget to update lib_pypy/cffi.egg- info diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -2,13 +2,15 @@ from pypy.interpreter.mixedmodule import MixedModule from rpython.rlib import rdynload +VERSION = "0.9.1" + class Module(MixedModule): appleveldefs = { } interpleveldefs = { - '__version__': 'space.wrap("0.9.1")', + '__version__': 'space.wrap("%s")' % VERSION, 'load_library': 'libraryobj.load_library', diff --git a/pypy/module/_cffi_backend/test/test_file.py b/pypy/module/_cffi_backend/test/test_file.py --- a/pypy/module/_cffi_backend/test/test_file.py +++ b/pypy/module/_cffi_backend/test/test_file.py @@ -15,3 +15,10 @@ "Update test/_backend_test_c.py by copying it from " "https://bitbucket.org/cffi/cffi/raw/default/c/test_c.py " "and killing the import lines at the start") + +def test_egginfo_version(): + from pypy.module._cffi_backend import VERSION + line = "Version: %s\n" % VERSION + eggfile = py.path.local(__file__).join('..', '..', '..', '..', '..', + 'lib_pypy', 'cffi.egg-info') + assert line in eggfile.readlines() From noreply at buildbot.pypy.org Sun Mar 8 19:48:36 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 8 Mar 2015 19:48:36 +0100 (CET) Subject: [pypy-commit] stmgc c8-marker: Change the signature: the format changed Message-ID: <20150308184836.ECBEE1C071E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-marker Changeset: r1700:e3079f525e1a Date: 2015-03-08 19:39 +0100 http://bitbucket.org/pypy/stmgc/changeset/e3079f525e1a/ Log: Change the signature: the format changed diff --git a/c8/stm/prof.c b/c8/stm/prof.c --- a/c8/stm/prof.c +++ b/c8/stm/prof.c @@ -58,7 +58,7 @@ if (profiling_file == NULL) return false; - fwrite("STMGC-C7-PROF01\n", 16, 1, profiling_file); + fwrite("STMGC-C8-PROF01\n", 16, 1, profiling_file); stmcb_timing_event = _stm_profiling_event; return true; } From noreply at buildbot.pypy.org Sun Mar 8 19:55:50 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 8 Mar 2015 19:55:50 +0100 (CET) Subject: [pypy-commit] pypy.org extradoc: update the values Message-ID: <20150308185550.BB8141C0221@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r578:0fb16c46382d Date: 2015-03-08 19:56 +0100 http://bitbucket.org/pypy/pypy.org/changeset/0fb16c46382d/ Log: update the values diff --git a/don1.html b/don1.html --- a/don1.html +++ b/don1.html @@ -15,7 +15,7 @@ - $58873 of $105000 (56.1%) + $58892 of $105000 (56.1%)
diff --git a/don4.html b/don4.html --- a/don4.html +++ b/don4.html @@ -9,7 +9,7 @@ @@ -17,7 +17,7 @@ 2nd call: - $22473 of $80000 (28.1%) + $22583 of $80000 (28.2%)
From noreply at buildbot.pypy.org Sun Mar 8 20:03:48 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 8 Mar 2015 20:03:48 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c8: fix Message-ID: <20150308190348.3C5F01C0221@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c8 Changeset: r76285:89d3462f6005 Date: 2015-03-08 20:03 +0100 http://bitbucket.org/pypy/pypy/changeset/89d3462f6005/ Log: fix diff --git a/rpython/translator/stm/src_stm/extracode.h b/rpython/translator/stm/src_stm/extracode.h --- a/rpython/translator/stm/src_stm/extracode.h +++ b/rpython/translator/stm/src_stm/extracode.h @@ -157,8 +157,7 @@ { /* only for tests: XXX fishing */ stm_loc_marker_t marker; - marker.tl = &stm_thread_local; - marker.segment_base = STM_SEGMENT->segment_base; + char *segment_base = STM_SEGMENT->segment_base; struct stm_shadowentry_s *_ss = stm_thread_local.shadowstack - 2; while (!(((uintptr_t)(_ss->ss)) & 1)) { @@ -169,7 +168,8 @@ marker.object = (_ss + 1)->ss; static char buffer[80]; - int length = _stm_expand_marker_for_pypy(&marker, buffer, 80); + int length; + length = _stm_expand_marker_for_pypy(segment_base, &marker,buffer, 80); assert(length >= 0 && length < 80); buffer[length] = 0; return buffer; From noreply at buildbot.pypy.org Sun Mar 8 20:03:47 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 8 Mar 2015 20:03:47 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c8: import stmgc (c8-marker) Message-ID: <20150308190347.0F3B51C0221@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c8 Changeset: r76284:5a3f2855c123 Date: 2015-03-08 19:54 +0100 http://bitbucket.org/pypy/pypy/changeset/5a3f2855c123/ Log: import stmgc (c8-marker) diff --git a/rpython/translator/stm/src_stm/extracode.h b/rpython/translator/stm/src_stm/extracode.h --- a/rpython/translator/stm/src_stm/extracode.h +++ b/rpython/translator/stm/src_stm/extracode.h @@ -81,7 +81,8 @@ return seg_base + addr + rpy_items_ofs; } -static int _stm_expand_marker_for_pypy(stm_loc_marker_t *marker, +static int _stm_expand_marker_for_pypy(char *segment_base, + stm_loc_marker_t *marker, char *outputbuf, int outputbufsize) { if (marker->object == NULL) @@ -95,7 +96,6 @@ long fnlen = 1, nlen = 1, line = 0; char *fn = "?", *name = "?"; - char *segment_base = marker->segment_base; long o = (long)marker->object; co_filename = _fetch_long(segment_base, o + g_co_filename_ofs); diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision --- a/rpython/translator/stm/src_stm/revision +++ b/rpython/translator/stm/src_stm/revision @@ -1,1 +1,1 @@ -f1272b890ba0 +e3079f525e1a diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c --- a/rpython/translator/stm/src_stm/stm/core.c +++ b/rpython/translator/stm/src_stm/stm/core.c @@ -49,6 +49,8 @@ DEBUG_EXPECT_SEGFAULT(false); for (; undo < end; undo++) { + if (undo->type == TYPE_POSITION_MARKER) + continue; object_t *obj = undo->object; stm_char *oslice = ((stm_char *)obj) + SLICE_OFFSET(undo->slice); uintptr_t current_page_num = ((uintptr_t)oslice) / 4096; @@ -228,7 +230,7 @@ addr >= stm_object_pages+TOTAL_MEMORY) { /* actual segfault, unrelated to stmgc */ fprintf(stderr, "Segmentation fault: accessing %p\n", addr); - raise(SIGINT); + abort(); } int segnum = get_segment_of_linear_address(addr); @@ -236,7 +238,7 @@ if (segnum != STM_SEGMENT->segment_num) { fprintf(stderr, "Segmentation fault: accessing %p (seg %d) from" " seg %d\n", addr, segnum, STM_SEGMENT->segment_num); - raise(SIGINT); + abort(); } dprintf(("-> segment: %d\n", segnum)); @@ -245,7 +247,7 @@ if (pagenum < END_NURSERY_PAGE) { fprintf(stderr, "Segmentation fault: accessing %p (seg %d " "page %lu)\n", addr, segnum, pagenum); - raise(SIGINT); + abort(); } DEBUG_EXPECT_SEGFAULT(false); @@ -269,6 +271,11 @@ struct stm_undo_s *undo = cl->written; struct stm_undo_s *end = undo + cl->written_count; for (; undo < end; undo++) { + if (undo->type == TYPE_POSITION_MARKER) { + fprintf(stderr, " marker %p %lu\n", + undo->marker_object, undo->marker_odd_number); + continue; + } fprintf(stderr, " obj %p, size %d, ofs %lu: ", undo->object, SLICE_SIZE(undo->slice), SLICE_OFFSET(undo->slice)); /* long i; */ @@ -362,6 +369,8 @@ struct stm_undo_s *undo = cl->written; struct stm_undo_s *end = cl->written + cl->written_count; for (; undo < end; undo++) { + if (undo->type == TYPE_POSITION_MARKER) + continue; if (_stm_was_read(undo->object)) { /* first reset all modified objects from the backup copies as soon as the first conflict is detected; @@ -369,6 +378,7 @@ the old (but unmodified) version to the newer version. */ reset_modified_from_backup_copies(my_segnum); + timing_write_read_contention(cl->written, undo); needs_abort = true; dprintf(("_stm_validate() failed for obj %p\n", undo->object)); @@ -599,6 +609,8 @@ { dprintf(("make_bk_slices_for_range(%p, %lu, %lu)\n", obj, start - (stm_char*)obj, end - start)); + timing_record_write_position(); + char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); uintptr_t first_page = ((uintptr_t)start) / 4096UL; uintptr_t end_page = ((uintptr_t)end) / 4096UL; @@ -1021,6 +1033,8 @@ struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count); for (; undo < end; undo++) { + if (undo->type == TYPE_POSITION_MARKER) + continue; object_t *obj = undo->object; obj->stm_flags &= ~GCFLAG_WB_EXECUTED; } @@ -1034,6 +1048,8 @@ struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count); for (; undo < end; undo++) { + if (undo->type == TYPE_POSITION_MARKER) + continue; object_t *obj = undo->object; obj->stm_flags |= GCFLAG_WB_EXECUTED; } @@ -1051,6 +1067,7 @@ assert(STM_PSEGMENT->safe_point == SP_NO_TRANSACTION); assert(STM_PSEGMENT->transaction_state == TS_NONE); + timing_event(tl, STM_TRANSACTION_START); STM_PSEGMENT->transaction_state = TS_REGULAR; STM_PSEGMENT->safe_point = SP_RUNNING; #ifndef NDEBUG @@ -1080,6 +1097,10 @@ assert(tree_is_cleared(STM_PSEGMENT->callbacks_on_commit_and_abort[1])); assert(list_is_empty(STM_PSEGMENT->young_objects_with_light_finalizers)); assert(STM_PSEGMENT->finalizers == NULL); +#ifndef NDEBUG + /* this should not be used when objects_pointing_to_nursery == NULL */ + STM_PSEGMENT->position_markers_len_old = 99999999999999999L; +#endif check_nursery_at_transaction_start(); @@ -1125,7 +1146,7 @@ /************************************************************/ -static void _finish_transaction() +static void _finish_transaction(enum stm_event_e event) { stm_thread_local_t *tl = STM_SEGMENT->running_thread; @@ -1136,6 +1157,7 @@ list_clear(STM_PSEGMENT->objects_pointing_to_nursery); list_clear(STM_PSEGMENT->old_objects_with_cards_set); list_clear(STM_PSEGMENT->large_overflow_objects); + timing_event(tl, event); release_thread_segment(tl); /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ @@ -1147,6 +1169,8 @@ struct stm_undo_s *undo = (struct stm_undo_s *)list->items; struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count); for (; undo < end; undo++) { + if (undo->type == TYPE_POSITION_MARKER) + continue; object_t *obj = undo->object; struct object_s *dst = (struct object_s*)REAL_ADDRESS(segbase, obj); assert(dst->stm_flags & GCFLAG_WRITE_BARRIER); @@ -1236,7 +1260,7 @@ /* done */ stm_thread_local_t *tl = STM_SEGMENT->running_thread; - _finish_transaction(); + _finish_transaction(STM_TRANSACTION_COMMIT); /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ s_mutex_unlock(); @@ -1260,6 +1284,8 @@ struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count); for (; undo < end; undo++) { + if (undo->type == TYPE_POSITION_MARKER) + continue; object_t *obj = undo->object; char *dst = REAL_ADDRESS(pseg->pub.segment_base, obj); @@ -1369,7 +1395,7 @@ : NURSERY_END; } - _finish_transaction(); + _finish_transaction(STM_TRANSACTION_ABORT); /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ return tl; diff --git a/rpython/translator/stm/src_stm/stm/core.h b/rpython/translator/stm/src_stm/stm/core.h --- a/rpython/translator/stm/src_stm/stm/core.h +++ b/rpython/translator/stm/src_stm/stm/core.h @@ -84,9 +84,12 @@ track the STM status: these are old objects that where written to and that will need to be recorded in the commit log. The list contains three entries for every such object, in the same - format as 'struct stm_undo_s' below. + format as 'struct stm_undo_s' below. It can also represent a + position marker, like 'struct stm_undo_s'. */ struct list_s *modified_old_objects; + uintptr_t position_markers_last; /* index of most recent pos marker */ + uintptr_t position_markers_len_old; /* length of list at last minor col */ struct list_s *objects_pointing_to_nursery; struct list_s *old_objects_with_cards_set; @@ -174,18 +177,27 @@ /* Commit Log things */ struct stm_undo_s { - object_t *object; /* the object that is modified */ - char *backup; /* some backup data (a slice of the original obj) */ - uint64_t slice; /* location and size of this slice (cannot cross - pages). The size is in the lower 2 bytes, and - the offset in the remaining 6 bytes. */ + union { + struct { + object_t *object; /* the object that is modified */ + char *backup; /* some backup data (a slice of the original obj) */ + uint64_t slice; /* location and size of this slice (cannot cross + pages). The size is in the lower 2 bytes, and + the offset in the remaining 6 bytes. */ + }; + struct { + intptr_t type; /* TYPE_POSITION_MARKER */ + uintptr_t marker_odd_number; /* the odd number part of the marker */ + object_t *marker_object; /* the object part of the marker */ + }; + }; }; +#define TYPE_POSITION_MARKER (-1) #define SLICE_OFFSET(slice) ((slice) >> 16) #define SLICE_SIZE(slice) ((int)((slice) & 0xFFFF)) #define NEW_SLICE(offset, size) (((uint64_t)(offset)) << 16 | (size)) - /* The model is: we have a global chained list, from 'commit_log_root', of 'struct stm_commit_log_entry_s' entries. Every one is fully read-only apart from the 'next' field. Every one stands for one diff --git a/rpython/translator/stm/src_stm/stm/gcpage.c b/rpython/translator/stm/src_stm/stm/gcpage.c --- a/rpython/translator/stm/src_stm/stm/gcpage.c +++ b/rpython/translator/stm/src_stm/stm/gcpage.c @@ -348,6 +348,8 @@ struct stm_undo_s *modified = (struct stm_undo_s *)lst->items; struct stm_undo_s *end = (struct stm_undo_s *)(lst->items + lst->count); for (; modified < end; modified++) { + if (modified->type == TYPE_POSITION_MARKER) + continue; object_t *obj = modified->object; struct object_s *dst = (struct object_s*)REAL_ADDRESS(base, obj); @@ -386,6 +388,22 @@ LIST_FREE(uniques); } +static void mark_visit_from_markers(void) +{ + long i; + for (i = 1; i < NB_SEGMENTS; i++) { + struct stm_priv_segment_info_s *pseg = get_priv_segment(i); + struct list_s *lst = get_priv_segment(i)->modified_old_objects; + + struct stm_undo_s *modified = (struct stm_undo_s *)lst->items; + struct stm_undo_s *end = (struct stm_undo_s *)(lst->items + lst->count); + for (; modified < end; modified++) { + if (modified->type == TYPE_POSITION_MARKER) + mark_visit_possibly_new_object(modified->marker_object, pseg); + } + } +} + static void mark_visit_from_roots(void) { if (testing_prebuilt_objs != NULL) { @@ -683,6 +701,7 @@ /* marking */ LIST_CREATE(marked_objects_to_trace); mark_visit_from_modified_objects(); + mark_visit_from_markers(); mark_visit_from_roots(); mark_visit_from_finalizer_pending(); diff --git a/rpython/translator/stm/src_stm/stm/marker.c b/rpython/translator/stm/src_stm/stm/marker.c new file mode 100644 --- /dev/null +++ b/rpython/translator/stm/src_stm/stm/marker.c @@ -0,0 +1,93 @@ +/* Imported by rpython/translator/stm/import_stmgc.py */ +#ifndef _STM_CORE_H_ +# error "must be compiled via stmgc.c" +#endif + +static bool marker_fetch(stm_thread_local_t *tl, stm_loc_marker_t *out_marker) +{ + /* Fetch the current marker from tl's shadow stack, + and return it in 'out_marker->odd_number' and 'out_marker->object'. */ + struct stm_shadowentry_s *current = tl->shadowstack - 1; + struct stm_shadowentry_s *base = tl->shadowstack_base; + + /* The shadowstack_base contains -1, which is a convenient stopper for + the loop below but which shouldn't be returned. */ + assert(base->ss == (object_t *)-1); + + while (!(((uintptr_t)current->ss) & 1)) { + current--; + assert(current >= base); + } + if (current != base) { + /* found the odd marker */ + out_marker->odd_number = (uintptr_t)current[0].ss; + out_marker->object = current[1].ss; + return true; + } + else { + /* no marker found */ + return false; + } +} + +static void marker_fetch_obj_write(struct stm_undo_s *start, + struct stm_undo_s *contention, + stm_loc_marker_t *out_marker) +{ + /* Fill out 'out_marker->odd_number' and 'out_marker->object' from + the marker just before 'contention' in the list starting at + 'start'. + */ + while (contention != start) { + --contention; + if (contention->type == TYPE_POSITION_MARKER) { + out_marker->odd_number = contention->marker_odd_number; + out_marker->object = contention->marker_object; + return; + } + } + /* no position marker found... */ + out_marker->odd_number = 0; + out_marker->object = NULL; +} + +static void _timing_record_write_position(void) +{ + stm_loc_marker_t marker; + if (!marker_fetch(STM_SEGMENT->running_thread, &marker)) + return; + + struct list_s *list = STM_PSEGMENT->modified_old_objects; + uintptr_t i = STM_PSEGMENT->position_markers_last; + if (i < list_count(list)) { + struct stm_undo_s *undo = (struct stm_undo_s *)(list->items + i); + if (undo->type == TYPE_POSITION_MARKER && + undo->marker_odd_number == marker.odd_number && + undo->marker_object == marker.object) + return; /* already up-to-date */ + } + + STM_PSEGMENT->position_markers_last = list_count(list); + STM_PSEGMENT->modified_old_objects = list_append3( + list, + TYPE_POSITION_MARKER, /* type */ + marker.odd_number, /* marker_odd_number */ + (uintptr_t)marker.object); /* marker_object */ +} + +static void timing_write_read_contention(struct stm_undo_s *start, + struct stm_undo_s *contention) +{ + if (stmcb_timing_event == NULL) + return; + + stm_loc_marker_t marker; + marker_fetch_obj_write(start, contention, &marker); + stmcb_timing_event(STM_SEGMENT->running_thread, + STM_CONTENTION_WRITE_READ, &marker); +} + + +void (*stmcb_timing_event)(stm_thread_local_t *tl, /* the local thread */ + enum stm_event_e event, + stm_loc_marker_t *marker); diff --git a/rpython/translator/stm/src_stm/stm/marker.h b/rpython/translator/stm/src_stm/stm/marker.h new file mode 100644 --- /dev/null +++ b/rpython/translator/stm/src_stm/stm/marker.h @@ -0,0 +1,11 @@ +/* Imported by rpython/translator/stm/import_stmgc.py */ +static void _timing_record_write_position(void); +static void timing_write_read_contention(struct stm_undo_s *start, + struct stm_undo_s *contention); + + +#define timing_event(tl, event) \ + (stmcb_timing_event != NULL ? stmcb_timing_event(tl, event, NULL) : (void)0) + +#define timing_record_write_position() \ + (stmcb_timing_event != NULL ? _timing_record_write_position() : (void)0) diff --git a/rpython/translator/stm/src_stm/stm/nursery.c b/rpython/translator/stm/src_stm/stm/nursery.c --- a/rpython/translator/stm/src_stm/stm/nursery.c +++ b/rpython/translator/stm/src_stm/stm/nursery.c @@ -185,6 +185,8 @@ struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count); for (; undo < end; undo++) { + if (undo->type == TYPE_POSITION_MARKER) + continue; _cards_cleared_in_object(pseg, undo->object, false); } LIST_FOREACH_R( @@ -410,6 +412,20 @@ } } +static void collect_roots_from_markers(uintptr_t len_old) +{ + dprintf(("collect_roots_from_markers\n")); + + /* visit the marker objects */ + struct list_s *list = STM_PSEGMENT->modified_old_objects; + struct stm_undo_s *undo = (struct stm_undo_s *)(list->items + len_old); + struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count); + + for (; undo < end; undo++) { + if (undo->type == TYPE_POSITION_MARKER) + minor_trace_if_young(&undo->marker_object); + } +} static void collect_objs_still_young_but_with_finalizers(void) { @@ -494,14 +510,25 @@ dprintf(("minor_collection commit=%d\n", (int)commit)); STM_PSEGMENT->minor_collect_will_commit_now = commit; + + uintptr_t len_old; + if (STM_PSEGMENT->overflow_number_has_been_used) + len_old = STM_PSEGMENT->position_markers_len_old; + else + len_old = 0; + if (!commit) { /* 'STM_PSEGMENT->overflow_number' is used now by this collection, in the sense that it's copied to the overflow objects */ STM_PSEGMENT->overflow_number_has_been_used = true; + STM_PSEGMENT->position_markers_len_old = + list_count(STM_PSEGMENT->modified_old_objects); } collect_cardrefs_to_nursery(); + collect_roots_from_markers(len_old); + collect_roots_in_nursery(); if (STM_PSEGMENT->finalizers != NULL) diff --git a/rpython/translator/stm/src_stm/stm/prof.c b/rpython/translator/stm/src_stm/stm/prof.c new file mode 100644 --- /dev/null +++ b/rpython/translator/stm/src_stm/stm/prof.c @@ -0,0 +1,122 @@ +/* Imported by rpython/translator/stm/import_stmgc.py */ +#include +#include + +static FILE *profiling_file; +static char *profiling_basefn = NULL; +static stm_expand_marker_fn profiling_expand_marker; + +#define MARKER_LEN_MAX 160 + + +static bool close_timing_log(void); /* forward */ + +static void _stm_profiling_event(stm_thread_local_t *tl, + enum stm_event_e event, + stm_loc_marker_t *marker) +{ + struct buf_s { + uint32_t tv_sec; + uint32_t tv_nsec; + uint32_t thread_num; + uint8_t event; + uint8_t marker_length; + char extra[MARKER_LEN_MAX+1]; + } __attribute__((packed)); + + struct buf_s buf; + struct timespec t; + clock_gettime(CLOCK_MONOTONIC, &t); + buf.tv_sec = t.tv_sec; + buf.tv_nsec = t.tv_nsec; + buf.thread_num = tl->thread_local_counter; + buf.event = event; + buf.marker_length = 0; + + if (marker != NULL && marker->odd_number != 0) { + buf.marker_length = profiling_expand_marker(get_segment_base(0), + marker, + buf.extra, MARKER_LEN_MAX); + } + + if (fwrite(&buf, offsetof(struct buf_s, extra) + buf.marker_length, + 1, profiling_file) != 1) { + fprintf(stderr, "stmgc: profiling log file closed unexpectedly: %m\n"); + close_timing_log(); + } +} + +static int default_expand_marker(char *b, stm_loc_marker_t *m, char *p, int s) +{ + *(uintptr_t *)p = m->odd_number; + return sizeof(uintptr_t); +} + +static bool open_timing_log(const char *filename) +{ + profiling_file = fopen(filename, "w"); + if (profiling_file == NULL) + return false; + + fwrite("STMGC-C8-PROF01\n", 16, 1, profiling_file); + stmcb_timing_event = _stm_profiling_event; + return true; +} + +static bool close_timing_log(void) +{ + if (stmcb_timing_event == &_stm_profiling_event) { + stmcb_timing_event = NULL; + fclose(profiling_file); + profiling_file = NULL; + return true; + } + return false; +} + +static void prof_forksupport_prepare(void) +{ + if (profiling_file != NULL) + fflush(profiling_file); +} + +static void prof_forksupport_child(void) +{ + if (close_timing_log() && profiling_basefn != NULL) { + char filename[1024]; + snprintf(filename, sizeof(filename), + "%s.fork%ld", profiling_basefn, (long)getpid()); + open_timing_log(filename); + } +} + +int stm_set_timing_log(const char *profiling_file_name, int fork_mode, + stm_expand_marker_fn expand_marker) +{ + close_timing_log(); + free(profiling_basefn); + profiling_basefn = NULL; + + if (profiling_file_name == NULL) + return 0; + + if (!expand_marker) + expand_marker = default_expand_marker; + profiling_expand_marker = expand_marker; + + static bool fork_support_ready = false; + if (!fork_support_ready) { + int res = pthread_atfork(prof_forksupport_prepare, + NULL, prof_forksupport_child); + if (res != 0) + stm_fatalerror("pthread_atfork() failed: %m"); + fork_support_ready = true; + } + + if (!open_timing_log(profiling_file_name)) + return -1; + + if (fork_mode != 0) + profiling_basefn = strdup(profiling_file_name); + return 0; +} diff --git a/rpython/translator/stm/src_stm/stm/setup.c b/rpython/translator/stm/src_stm/stm/setup.c --- a/rpython/translator/stm/src_stm/stm/setup.c +++ b/rpython/translator/stm/src_stm/stm/setup.c @@ -221,6 +221,8 @@ return (pthread_t *)(tl->creating_pthread); } +static int thread_local_counters = 0; + void stm_register_thread_local(stm_thread_local_t *tl) { int num; @@ -242,6 +244,7 @@ numbers automatically. */ tl->associated_segment_num = -1; tl->last_associated_segment_num = num + 1; + tl->thread_local_counter = ++thread_local_counters; *_get_cpth(tl) = pthread_self(); _init_shadow_stack(tl); set_gs_register(get_segment_base(num + 1)); diff --git a/rpython/translator/stm/src_stm/stmgc.c b/rpython/translator/stm/src_stm/stmgc.c --- a/rpython/translator/stm/src_stm/stmgc.c +++ b/rpython/translator/stm/src_stm/stmgc.c @@ -15,6 +15,7 @@ #include "stm/gcpage.h" #include "stm/extra.h" #include "stm/fprintcolor.h" +#include "stm/marker.h" #include "stm/rewind_setjmp.h" #include "stm/finalizer.h" #include "stm/misc.c" @@ -34,5 +35,7 @@ #include "stm/core.c" #include "stm/extra.c" #include "stm/fprintcolor.c" +#include "stm/marker.c" +#include "stm/prof.c" #include "stm/rewind_setjmp.c" #include "stm/finalizer.c" diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h --- a/rpython/translator/stm/src_stm/stmgc.h +++ b/rpython/translator/stm/src_stm/stmgc.h @@ -68,6 +68,7 @@ /* the next fields are handled internally by the library */ int associated_segment_num; int last_associated_segment_num; + int thread_local_counter; struct stm_thread_local_s *prev, *next; void *creating_pthread[2]; } stm_thread_local_t; @@ -342,6 +343,69 @@ void stm_resume_all_other_threads(void); +/* Profiling events. In the comments: content of the markers, if any */ +enum stm_event_e { + /* always STM_TRANSACTION_START followed later by one of COMMIT or ABORT */ + STM_TRANSACTION_START, + STM_TRANSACTION_COMMIT, + STM_TRANSACTION_ABORT, + + /* write-read contention: a "marker" is included in the PYPYSTM file + saying where the write was done. Followed by STM_TRANSACTION_ABORT. */ + STM_CONTENTION_WRITE_READ, + + /* always one STM_WAIT_xxx followed later by STM_WAIT_DONE */ + STM_WAIT_FREE_SEGMENT, + STM_WAIT_OTHER_INEVITABLE, + STM_WAIT_DONE, + + /* start and end of GC cycles */ + STM_GC_MINOR_START, + STM_GC_MINOR_DONE, + STM_GC_MAJOR_START, + STM_GC_MAJOR_DONE, + + _STM_EVENT_N +}; + +#define STM_EVENT_NAMES \ + "transaction start", \ + "transaction commit", \ + "transaction abort", \ + "contention write read", \ + "wait free segment", \ + "wait other inevitable", \ + "wait done", \ + "gc minor start", \ + "gc minor done", \ + "gc major start", \ + "gc major done" + +/* The markers pushed in the shadowstack are an odd number followed by a + regular object pointer. */ +typedef struct { + uintptr_t odd_number; + object_t *object; +} stm_loc_marker_t; +extern void (*stmcb_timing_event)(stm_thread_local_t *tl, /* the local thread */ + enum stm_event_e event, + stm_loc_marker_t *marker); + +/* Calling this sets up a stmcb_timing_event callback that will produce + a binary file called 'profiling_file_name'. Call it with + 'fork_mode == 0' for only the main process, and with + 'fork_mode == 1' to also write files called + 'profiling_file_name.fork' after a fork(). Call it with NULL to + stop profiling. Returns -1 in case of error (see errno then). + The optional 'expand_marker' function pointer is called to expand + the marker's odd_number and object into printable data, starting at + the given position and with the given maximum length. */ +typedef int (*stm_expand_marker_fn)(char *seg_base, stm_loc_marker_t *marker, + char *output, int output_size); +int stm_set_timing_log(const char *profiling_file_name, int fork_mode, + stm_expand_marker_fn expand_marker); + + /* Convenience macros to push the markers into the shadowstack */ #define STM_PUSH_MARKER(tl, odd_num, p) do { \ uintptr_t _odd_num = (odd_num); \ diff --git a/rpython/translator/stm/src_stm/stmgcintf.h b/rpython/translator/stm/src_stm/stmgcintf.h --- a/rpython/translator/stm/src_stm/stmgcintf.h +++ b/rpython/translator/stm/src_stm/stmgcintf.h @@ -42,14 +42,6 @@ /* C8: not implemented properly yet: */ extern void stmcb_commit_soon(void); -typedef struct { - stm_thread_local_t *tl; - char *segment_base; /* base to interpret the 'object' below */ - uintptr_t odd_number; /* marker odd number, or 0 if marker is missing */ - object_t *object; /* marker object, or NULL if marker is missing */ -} stm_loc_marker_t; -static inline int stm_set_timing_log(const char *profiling_file_name, int fork_mode, - int expand_marker(stm_loc_marker_t *, char *, int)) {return 0;} /* C8: not implemented properly yet ^^^^^^^^^^^^^^^^^^ */ From noreply at buildbot.pypy.org Sun Mar 8 21:11:23 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 8 Mar 2015 21:11:23 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c8: Adapt print_stm_log Message-ID: <20150308201123.9DF1C1C0107@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c8 Changeset: r76286:b4c1304ddd02 Date: 2015-03-08 21:11 +0100 http://bitbucket.org/pypy/pypy/changeset/b4c1304ddd02/ Log: Adapt print_stm_log diff --git a/pypy/stm/print_stm_log.py b/pypy/stm/print_stm_log.py --- a/pypy/stm/print_stm_log.py +++ b/pypy/stm/print_stm_log.py @@ -8,29 +8,22 @@ STM_TRANSACTION_COMMIT = 1 STM_TRANSACTION_ABORT = 2 -# contention; see details at the start of contention.c -STM_CONTENTION_WRITE_WRITE = 3 # markers: self loc / other written loc -STM_CONTENTION_WRITE_READ = 4 # markers: self written loc / other missing -STM_CONTENTION_INEVITABLE = 5 # markers: self loc / other inev loc - -# following a contention, we get from the same thread one of -# STM_ABORTING_OTHER_CONTENTION, STM_TRANSACTION_ABORT (self-abort), -# or STM_WAIT_CONTENTION (self-wait). -STM_ABORTING_OTHER_CONTENTION = 6 +# write-read contention: a "marker" is included in the PYPYSTM file +# saying where the write was done. Followed by STM_TRANSACTION_ABORT. +STM_CONTENTION_WRITE_READ = 3 # always one STM_WAIT_xxx followed later by STM_WAIT_DONE -STM_WAIT_FREE_SEGMENT = 7 -STM_WAIT_SYNC_PAUSE = 8 -STM_WAIT_CONTENTION = 9 -STM_WAIT_DONE = 10 +STM_WAIT_FREE_SEGMENT = 4 +STM_WAIT_OTHER_INEVITABLE = 5 +STM_WAIT_DONE = 6 # start and end of GC cycles -STM_GC_MINOR_START = 11 -STM_GC_MINOR_DONE = 12 -STM_GC_MAJOR_START = 13 -STM_GC_MAJOR_DONE = 14 +STM_GC_MINOR_START = 7 +STM_GC_MINOR_DONE = 8 +STM_GC_MAJOR_START = 9 +STM_GC_MAJOR_DONE = 10 -_STM_EVENT_N = 15 +_STM_EVENT_N = 11 PAUSE_AFTER_ABORT = 0.000001 # usleep(1) after every abort @@ -44,24 +37,18 @@ class LogEntry(object): - def __init__(self, timestamp, threadnum, otherthreadnum, - event, marker1, marker2, frac): + def __init__(self, timestamp, threadnum, event, marker, frac): self.timestamp = timestamp self.threadnum = threadnum - self.otherthreadnum = otherthreadnum self.event = event - self.marker1 = marker1 - self.marker2 = marker2 + self.marker = marker self.frac = frac def __str__(self): - s = '[%.3f][%s->%s]\t%s' % ( - self.timestamp, self.threadnum, self.otherthreadnum, - event_name[self.event]) - if self.marker1: - s += ':\n%s' % print_marker(self.marker1) - if self.marker2: - s += '\n%s' % print_marker(self.marker2) + s = '[%.3f][%s]\t%s' % ( + self.timestamp, self.threadnum, event_name[self.event]) + if self.marker: + s += ':\n%s' % print_marker(self.marker) return s @@ -69,23 +56,22 @@ f = open(filename, 'rb') try: header = f.read(16) - if header != "STMGC-C7-PROF01\n": + if header != "STMGC-C8-PROF01\n": raise ValueError("wrong format in file %r" % (filename,)) f.seek(0, 2) frac = 1.0 / f.tell() f.seek(16, 0) result = [] while True: - packet = f.read(19) - if len(packet) < 19: break - sec, nsec, threadnum, otherthreadnum, event, len1, len2 = \ - struct.unpack("IIIIBBB", packet) + packet = f.read(14) + if len(packet) < 14: break + sec, nsec, threadnum, event, markerlen = \ + struct.unpack("IIIBB", packet) if event >= _STM_EVENT_N: raise ValueError("the file %r appears corrupted" % (filename,)) - m1 = f.read(len1) - m2 = f.read(len2) + marker = f.read(markerlen) yield LogEntry(sec + 0.000000001 * nsec, - threadnum, otherthreadnum, event, m1, m2, + threadnum, event, marker, f.tell() * frac) finally: f.close() @@ -134,10 +120,9 @@ class ConflictSummary(object): - def __init__(self, event, marker1, marker2): + def __init__(self, event, marker): self.event = event - self.marker1 = marker1 - self.marker2 = marker2 + self.marker = marker self.aborted_time = 0.0 self.paused_time = 0.0 self.num_events = 0 @@ -149,24 +134,18 @@ def get_event_name(self): return event_name[self.event] - def get_marker1(self): - return print_marker(self.marker1) - - def get_marker2(self): - return print_marker(self.marker2) + def get_marker(self): + return print_marker(self.marker) def __str__(self): s = '%.3fs lost in aborts, %.3fs paused (%dx %s)\n' % ( self.aborted_time, self.paused_time, self.num_events, self.get_event_name()) - s += print_marker(self.marker1) - if self.marker2: - s += '\n%s' % print_marker(self.marker2) + s += print_marker(self.marker) return s - r_marker = re.compile(r'File "(.+)", line (\d+)') def print_marker(marker): @@ -207,10 +186,11 @@ t = threads.get(entry.threadnum) if t is not None and t.in_transaction(): t.transaction_stop(entry) - elif entry.event in (STM_CONTENTION_WRITE_WRITE, + elif entry.event in (#STM_CONTENTION_WRITE_WRITE, STM_CONTENTION_WRITE_READ, - STM_CONTENTION_INEVITABLE): - summary = (entry.event, entry.marker1, entry.marker2) + #STM_CONTENTION_INEVITABLE, + ): + summary = (entry.event, entry.marker) c = conflicts.get(summary) if c is None: c = conflicts[summary] = ConflictSummary(*summary) @@ -219,16 +199,15 @@ t = threads.get(entry.threadnum) if t is not None and t.in_transaction(): t._conflict = ("local", c, entry) - elif entry.event == STM_ABORTING_OTHER_CONTENTION: - t = threads.get(entry.threadnum) - if t is not None and t._conflict and t._conflict[0] == "local": - _, c, entry = t._conflict - t._conflict = None - t2 = threads.get(entry.otherthreadnum) - if t2 is not None and t2.in_transaction(): - t2._conflict = ("remote", c, entry) - elif entry.event in (STM_WAIT_SYNC_PAUSE, STM_WAIT_CONTENTION, - STM_WAIT_FREE_SEGMENT): + ## elif entry.event == STM_ABORTING_OTHER_CONTENTION: + ## t = threads.get(entry.threadnum) + ## if t is not None and t._conflict and t._conflict[0] == "local": + ## _, c, entry = t._conflict + ## t._conflict = None + ## t2 = threads.get(entry.otherthreadnum) + ## if t2 is not None and t2.in_transaction(): + ## t2._conflict = ("remote", c, entry) + elif entry.event in (STM_WAIT_FREE_SEGMENT, STM_WAIT_OTHER_INEVITABLE): t = threads.get(entry.threadnum) if t is not None and t.in_transaction(): t.transaction_pause(entry) From noreply at buildbot.pypy.org Sun Mar 8 21:23:09 2015 From: noreply at buildbot.pypy.org (mattip) Date: Sun, 8 Mar 2015 21:23:09 +0100 (CET) Subject: [pypy-commit] cffi win32-zintegration: fix test harness for win32 Message-ID: <20150308202309.0E4081C019D@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: win32-zintegration Changeset: r1661:bcf702d6e17f Date: 2015-03-08 22:24 +0200 http://bitbucket.org/cffi/cffi/changeset/bcf702d6e17f/ Log: fix test harness for win32 diff --git a/testing/test_zintegration.py b/testing/test_zintegration.py --- a/testing/test_zintegration.py +++ b/testing/test_zintegration.py @@ -12,6 +12,20 @@ except OSError as e: py.test.skip("Cannot execute virtualenv: %s" % (e,)) + try: + deepcopy = os.symlink + except: + import shutil, errno + def deepcopy(src, dst): + try: + shutil.copytree(src, dst) + except OSError as e: + if e.errno in (errno.ENOTDIR, errno.EINVAL): + shutil.copy(src, dst) + else: + print 'got errno',e.errno,'not',errno.ENOTDIR + raise + site_packages = None for dirpath, dirnames, filenames in os.walk(str(tmpdir)): if os.path.basename(dirpath) == 'site-packages': @@ -31,7 +45,7 @@ modules += ('ply',) # needed for older versions of pycparser for module in modules: target = imp.find_module(module)[1] - os.symlink(target, os.path.join(site_packages, + deepcopy(target, os.path.join(site_packages, os.path.basename(target))) return tmpdir @@ -50,7 +64,11 @@ python_f.write(py.code.Source(python_snippet)) try: os.chdir(str(SNIPPET_DIR.join(dirname))) - vp = str(venv_dir.join('bin/python')) + if os.name == 'nt': + bindir = 'Scripts' + else: + bindir = 'bin' + vp = str(venv_dir.join(bindir).join('python')) subprocess.check_call((vp, 'setup.py', 'clean')) subprocess.check_call((vp, 'setup.py', 'install')) subprocess.check_call((vp, str(python_f))) From noreply at buildbot.pypy.org Mon Mar 9 09:41:48 2015 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 9 Mar 2015 09:41:48 +0100 (CET) Subject: [pypy-commit] pypy optresult: pass some more tests Message-ID: <20150309084148.904E51C023F@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult Changeset: r76287:643f3c00c690 Date: 2015-03-08 10:55 +0200 http://bitbucket.org/pypy/pypy/changeset/643f3c00c690/ Log: pass some more tests diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py --- a/rpython/jit/metainterp/optimizeopt/info.py +++ b/rpython/jit/metainterp/optimizeopt/info.py @@ -1,6 +1,7 @@ from rpython.jit.metainterp.resoperation import AbstractValue, ResOperation,\ rop +from rpython.jit.metainterp.history import ConstInt """ The tag field on PtrOptInfo has a following meaning: @@ -112,6 +113,9 @@ class ArrayPtrInfo(AbstractVirtualPtrInfo): _attrs_ = ('_is_virtual', 'length', '_items', '_descr') + def getlength(self): + return self.length + class ArrayStructInfo(ArrayPtrInfo): def __init__(self, descr, size, is_virtual): self.length = size @@ -140,7 +144,7 @@ if fld is not None: subbox = optforce.force_box(fld) setfieldop = ResOperation(rop.SETINTERIORFIELD_GC, - [op, subbox], + [op, ConstInt(index), subbox], descr=flddescr) optforce.emit_operation(setfieldop) i += 1 diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py --- a/rpython/jit/metainterp/optimizeopt/intbounds.py +++ b/rpython/jit/metainterp/optimizeopt/intbounds.py @@ -352,26 +352,27 @@ self.emit_operation(op) def optimize_INT_SIGNEXT(self, op): - value = self.getvalue(op.getarg(0)) + b = self.getintbound(op.getarg(0)) numbits = op.getarg(1).getint() * 8 start = -(1 << (numbits - 1)) stop = 1 << (numbits - 1) bounds = IntBound(start, stop - 1) - if bounds.contains_bound(value.getintbound()): - self.make_equal_to(op, value) + if bounds.contains_bound(b): + self.make_equal_to(op, op.getarg(0)) else: self.emit_operation(op) - vres = self.getvalue(op) - vres.getintbound().intersect(bounds) + bres = self.getintbound(op) + bres.intersect(bounds) def optimize_ARRAYLEN_GC(self, op): self.emit_operation(op) - array = self.getvalue(op.getarg(0)) - result = self.getvalue(op) - array.make_len_gt(MODE_ARRAY, op.getdescr(), -1) - array.getlenbound().bound.intersect(result.getintbound()) - assert isinstance(result, IntOptValue) - result.intbound = array.getlenbound().bound + # XXX + #array = self.getvalue(op.getarg(0)) + #result = self.getvalue(op) + #array.make_len_gt(MODE_ARRAY, op.getdescr(), -1) + #array.getlenbound().bound.intersect(result.getintbound()) + #assert isinstance(result, IntOptValue) + #result.intbound = array.getlenbound().bound def optimize_STRLEN(self, op): self.emit_operation(op) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -963,8 +963,8 @@ [f0, f1] f2 = float_mul(f0, f1) p0 = new_array_clear(1, descr=complexarraydescr) + setinteriorfield_gc(p0, 0, f0, descr=complexrealdescr) setinteriorfield_gc(p0, 0, f1, descr=compleximagdescr) - setinteriorfield_gc(p0, 0, f0, descr=complexrealdescr) i0 = escape_i(f2, p0) finish(i0) """ diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -777,11 +777,11 @@ self.emit_operation(op) def optimize_ARRAYLEN_GC(self, op): - value = self.getvalue(op.getarg(0)) - if value.is_virtual(): - self.make_constant_int(op, value.getlength()) + opinfo = self.getptrinfo(op.getarg(0)) + if opinfo and opinfo.is_virtual(): + self.make_constant_int(op, opinfo.getlength()) else: - value.ensure_nonnull() + self.make_nonnull(op.getarg(0)) self.emit_operation(op) def optimize_GETARRAYITEM_GC_I(self, op): From noreply at buildbot.pypy.org Mon Mar 9 09:41:49 2015 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 9 Mar 2015 09:41:49 +0100 (CET) Subject: [pypy-commit] pypy optresult: implement some parts of varray Message-ID: <20150309084149.BC8D51C023F@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult Changeset: r76288:bb7065b9e03c Date: 2015-03-09 10:41 +0200 http://bitbucket.org/pypy/pypy/changeset/bb7065b9e03c/ Log: implement some parts of varray diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py --- a/rpython/jit/metainterp/optimizeopt/info.py +++ b/rpython/jit/metainterp/optimizeopt/info.py @@ -113,6 +113,20 @@ class ArrayPtrInfo(AbstractVirtualPtrInfo): _attrs_ = ('_is_virtual', 'length', '_items', '_descr') + def __init__(self, descr, const, size, clear, is_virtual): + self._is_virtual = is_virtual + self.length = size + if clear: + self._items = [const] * size + else: + self._items = [None] * size + + def setitem_virtual(self, index, item): + self._items[index] = item + + def getitem_virtual(self, index): + return self._items[index] + def getlength(self): return self.length diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -567,49 +567,29 @@ def make_nonnull(self, op): op = self.get_box_replacement(op) + if op.is_constant(): + return opinfo = op.get_forwarded() if opinfo is not None: assert opinfo.is_nonnull() return op.set_forwarded(info.NonNullPtrInfo()) - def new_ptr_box(self): - xxx - return self.cpu.ts.BoxRef() - - def new_box(self, fieldofs): - xxx - if fieldofs.is_pointer_field(): - return self.new_ptr_box() - elif fieldofs.is_float_field(): - return BoxFloat() - else: - return BoxInt() - def new_const(self, fieldofs): if fieldofs.is_pointer_field(): - return self.cpu.ts.CVAL_NULLREF + return self.cpu.ts.CONST_NULL elif fieldofs.is_float_field(): - return CVAL_ZERO_FLOAT + return CONST_ZERO_FLOAT else: - return CVAL_ZERO - - def new_box_item(self, arraydescr): - xxx - if arraydescr.is_array_of_pointers(): - return self.new_ptr_box() - elif arraydescr.is_array_of_floats(): - return BoxFloat() - else: - return BoxInt() + return CONST_0 def new_const_item(self, arraydescr): if arraydescr.is_array_of_pointers(): - return self.cpu.ts.CVAL_NULLREF + return self.cpu.ts.CONST_NULL elif arraydescr.is_array_of_floats(): - return CVAL_ZERO_FLOAT + return CONST_ZERO_FLOAT else: - return CVAL_ZERO + return CONST_0 def propagate_all_forward(self, clear=True): if clear: diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -7,7 +7,8 @@ from rpython.jit.metainterp.optimizeopt.intutils import IntBound from rpython.jit.metainterp.optimizeopt.optimizer import (Optimization, REMOVED, CONST_0, CONST_1) -from rpython.jit.metainterp.optimizeopt.info import INFO_NONNULL, INFO_NULL +from rpython.jit.metainterp.optimizeopt.info import INFO_NONNULL, INFO_NULL,\ + ArrayPtrInfo from rpython.jit.metainterp.optimizeopt.util import _findall, make_dispatcher_method from rpython.jit.metainterp.resoperation import rop, ResOperation, opclasses,\ OpHelpers @@ -502,22 +503,25 @@ if length and length.getint() == 0: return True # 0-length arraycopy - source_value = self.getvalue(op.getarg(1)) - dest_value = self.getvalue(op.getarg(2)) + source_info = self.getptrinfo(op.getarg(1)) + dest_info = self.getptrinfo(op.getarg(2)) source_start_box = self.get_constant_box(op.getarg(3)) dest_start_box = self.get_constant_box(op.getarg(4)) extrainfo = op.getdescr().get_extra_info() if (source_start_box and dest_start_box - and length and (dest_value.is_virtual() or length.getint() <= 8) and - (source_value.is_virtual() or length.getint() <= 8) and - len(extrainfo.write_descrs_arrays) == 1): # <-sanity check - from rpython.jit.metainterp.optimizeopt.virtualize import VArrayValue + and length and ((dest_info and dest_info.is_virtual()) or + length.getint() <= 8) and + ((source_info and source_info.is_virtual()) or length.getint() <= 8) + and len(extrainfo.write_descrs_arrays) == 1): # <-sanity check source_start = source_start_box.getint() dest_start = dest_start_box.getint() - # XXX fish fish fish arraydescr = extrainfo.write_descrs_arrays[0] if arraydescr.is_array_of_structs(): return False # not supported right now + + xxx + from rpython.jit.metainterp.optimizeopt.virtualize import VArrayValue + # XXX fish fish fish for index in range(length.getint()): if source_value.is_virtual(): assert isinstance(source_value, VArrayValue) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -987,8 +987,8 @@ ops = """ [f0, f1] p0 = new_array_clear(3, descr=complexarraydescr) + setinteriorfield_gc(p0, 0, f1, descr=complexrealdescr) setinteriorfield_gc(p0, 0, f0, descr=compleximagdescr) - setinteriorfield_gc(p0, 0, f1, descr=complexrealdescr) call_n(0, p0, p0, 0, 2, 1, descr=complexarraycopydescr) f2 = getinteriorfield_gc_f(p0, 2, descr=complexrealdescr) f3 = getinteriorfield_gc_f(p0, 2, descr=compleximagdescr) @@ -1020,7 +1020,7 @@ def test_nonvirtual_1(self): ops = """ [i] - p1 = new_with_vtable(ConstClass(node_vtable)) + p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) setfield_gc(p1, i, descr=valuedescr) i0 = getfield_gc_i(p1, descr=valuedescr) i1 = int_add(i0, 1) @@ -1031,7 +1031,7 @@ expected = """ [i] i1 = int_add(i, 1) - p1 = new_with_vtable(ConstClass(node_vtable)) + p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) setfield_gc(p1, i, descr=valuedescr) escape_n(p1) escape_n(p1) @@ -1045,7 +1045,7 @@ i0 = getfield_gc_i(p0, descr=valuedescr) escape_n(p0) i1 = int_add(i0, i) - p1 = new_with_vtable(ConstClass(node_vtable)) + p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) setfield_gc(p1, i1, descr=valuedescr) jump(i, p1) """ @@ -1055,7 +1055,7 @@ def test_nonvirtual_later(self): ops = """ [i] - p1 = new_with_vtable(ConstClass(node_vtable)) + p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) setfield_gc(p1, i, descr=valuedescr) i1 = getfield_gc_i(p1, descr=valuedescr) escape_n(p1) @@ -1065,7 +1065,7 @@ """ expected = """ [i] - p1 = new_with_vtable(ConstClass(node_vtable)) + p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) setfield_gc(p1, i, descr=valuedescr) escape_n(p1) i2 = getfield_gc_i(p1, descr=valuedescr) @@ -1077,7 +1077,7 @@ def test_nonvirtual_write_null_fields_on_force(self): ops = """ [i] - p1 = new_with_vtable(ConstClass(node_vtable)) + p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) setfield_gc(p1, i, descr=valuedescr) i1 = getfield_gc_i(p1, descr=valuedescr) setfield_gc(p1, 0, descr=valuedescr) @@ -1087,7 +1087,7 @@ """ expected = """ [i] - p1 = new_with_vtable(ConstClass(node_vtable)) + p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) setfield_gc(p1, 0, descr=valuedescr) escape_n(p1) i2 = getfield_gc_i(p1, descr=valuedescr) @@ -1098,7 +1098,7 @@ def test_getfield_gc_pure_1(self): ops = """ [i] - p1 = new_with_vtable(ConstClass(node_vtable)) + p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) setfield_gc(p1, i, descr=valuedescr) i1 = getfield_gc_pure_i(p1, descr=valuedescr) jump(i1) diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -532,15 +532,11 @@ if arraydescr.is_array_of_structs(): assert clear opinfo = info.ArrayStructInfo(arraydescr, size, True) - source_op.set_forwarded(opinfo) - return opinfo - vvalue = VArrayStructValue(arraydescr, size, source_op) else: - constvalue = self.new_const_item(arraydescr) - vvalue = VArrayValue(arraydescr, constvalue, size, source_op, - clear=clear) - self.make_equal_to(source_op, vvalue) - return vvalue + const = self.new_const_item(arraydescr) + opinfo = info.ArrayPtrInfo(arraydescr, const, size, clear, True) + source_op.set_forwarded(opinfo) + return opinfo def make_vstruct(self, structdescr, source_op): vvalue = VStructValue(self.optimizer.cpu, structdescr, source_op) @@ -670,15 +666,17 @@ def optimize_GETFIELD_GC_I(self, op): opinfo = self.getptrinfo(op.getarg(0)) + # XXX dealt with by heapcache # If this is an immutable field (as indicated by op.is_always_pure()) # then it's safe to reuse the virtual's field, even if it has been # forced, because it should never be written to again. - if op.is_always_pure(): - if value.is_forced_virtual() and op.is_always_pure(): - fieldvalue = value.getfield(op.getdescr(), None) - if fieldvalue is not None: - self.make_equal_to(op, fieldvalue) - return + #if op.is_always_pure(): + # + # if value.is_forced_virtual() and op.is_always_pure(): + # fieldvalue = value.getfield(op.getdescr(), None) + # if fieldvalue is not None: + # self.make_equal_to(op, fieldvalue) + # return if opinfo and opinfo.is_virtual(): fieldop = opinfo.getfield_virtual(op.getdescr()) if fieldop is None: @@ -785,17 +783,17 @@ self.emit_operation(op) def optimize_GETARRAYITEM_GC_I(self, op): - value = self.getvalue(op.getarg(0)) - if value.is_virtual(): - assert isinstance(value, VArrayValue) + opinfo = self.getptrinfo(op.getarg(0)) + if opinfo and opinfo.is_virtual(): indexbox = self.get_constant_box(op.getarg(1)) if indexbox is not None: - itemvalue = value.getitem(indexbox.getint()) - if itemvalue is None: # reading uninitialized array items? + item = opinfo.getitem_virtual(indexbox.getint()) + if item is None: # reading uninitialized array items? + assert False, "can't read uninitialized items" itemvalue = value.constvalue # bah, just return 0 - self.make_equal_to(op, itemvalue) + self.make_equal_to(op, item) return - value.ensure_nonnull() + self.make_nonnull(op.getarg(0)) self.emit_operation(op) optimize_GETARRAYITEM_GC_R = optimize_GETARRAYITEM_GC_I optimize_GETARRAYITEM_GC_F = optimize_GETARRAYITEM_GC_I @@ -807,13 +805,14 @@ optimize_GETARRAYITEM_GC_PURE_F = optimize_GETARRAYITEM_GC_I def optimize_SETARRAYITEM_GC(self, op): - value = self.getvalue(op.getarg(0)) - if value.is_virtual(): + opinfo = self.getptrinfo(op.getarg(0)) + if opinfo and opinfo.is_virtual(): indexbox = self.get_constant_box(op.getarg(1)) if indexbox is not None: - value.setitem(indexbox.getint(), self.getvalue(op.getarg(2))) + opinfo.setitem_virtual(indexbox.getint(), + self.get_box_replacement(op.getarg(2))) return - value.ensure_nonnull() + self.make_nonnull(op.getarg(0)) self.emit_operation(op) def _unpack_arrayitem_raw_op(self, op, indexbox): @@ -907,8 +906,7 @@ fieldvalue = self.new_const(descr) self.make_equal_to(op, fld) return - xxx - value.ensure_nonnull() + self.make_nonnull(op.getarg(0)) self.emit_operation(op) optimize_GETINTERIORFIELD_GC_R = optimize_GETINTERIORFIELD_GC_I optimize_GETINTERIORFIELD_GC_F = optimize_GETINTERIORFIELD_GC_I diff --git a/rpython/jit/metainterp/typesystem.py b/rpython/jit/metainterp/typesystem.py --- a/rpython/jit/metainterp/typesystem.py +++ b/rpython/jit/metainterp/typesystem.py @@ -38,7 +38,6 @@ loops_done_with_this_frame_ref = None # patched by compile.py NULLREF = history.ConstPtr.value CONST_NULL = history.ConstPtr(NULLREF) - CVAL_NULLREF = None # patched by optimizeopt.py def new_ConstRef(self, x): ptrval = lltype.cast_opaque_ptr(llmemory.GCREF, x) From noreply at buildbot.pypy.org Mon Mar 9 10:04:50 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 9 Mar 2015 10:04:50 +0100 (CET) Subject: [pypy-commit] stmgc c8-marker: hg merge default Message-ID: <20150309090450.A239A1C023F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-marker Changeset: r1701:4b7e576c7437 Date: 2015-03-09 08:51 +0100 http://bitbucket.org/pypy/stmgc/changeset/4b7e576c7437/ Log: hg merge default diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -508,6 +508,10 @@ break; /* success! */ } else if (old->next == INEV_RUNNING) { /* we failed because there is an INEV transaction running */ + /* XXXXXX for now just sleep (XXX with the lock acquired? + isn't it a bad idea?). We should really ask to inev + transaction to do the commit for us, and then we can + continue running. */ usleep(10); } From noreply at buildbot.pypy.org Mon Mar 9 10:04:51 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 9 Mar 2015 10:04:51 +0100 (CET) Subject: [pypy-commit] stmgc c8-marker: Generate the markers in case of inevitable transactions Message-ID: <20150309090451.DCF591C023F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-marker Changeset: r1702:5a56584dbe4b Date: 2015-03-09 10:05 +0100 http://bitbucket.org/pypy/stmgc/changeset/5a56584dbe4b/ Log: Generate the markers in case of inevitable transactions diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -5,6 +5,7 @@ /* *** MISC *** */ static void free_bk(struct stm_undo_s *undo) { + assert(undo->type != TYPE_POSITION_MARKER); free(undo->backup); assert(undo->backup = (char*)-88); increment_total_allocated(-SLICE_SIZE(undo->slice)); @@ -455,6 +456,17 @@ } +static void wait_for_other_inevitable(struct stm_commit_log_entry_s *old) +{ + timing_wait_other_inevitable(); + + while (old->next == INEV_RUNNING && !safe_point_requested()) { + spin_loop(); + usleep(10); /* XXXXXX */ + } + timing_event(STM_SEGMENT->running_thread, STM_WAIT_DONE); +} + static void reset_wb_executed_flags(void); static void readd_wb_executed_flags(void); static void check_all_write_barrier_flags(char *segbase, struct list_s *list); @@ -506,13 +518,6 @@ if (__sync_bool_compare_and_swap(&old->next, NULL, new)) break; /* success! */ - } else if (old->next == INEV_RUNNING) { - /* we failed because there is an INEV transaction running */ - /* XXXXXX for now just sleep (XXX with the lock acquired? - isn't it a bad idea?). We should really ask to inev - transaction to do the commit for us, and then we can - continue running. */ - usleep(10); } if (is_commit) { @@ -523,6 +528,16 @@ readd_wb_executed_flags(); } + if (old->next == INEV_RUNNING && !safe_point_requested()) { + /* we failed because there is an INEV transaction running */ + /* XXXXXX for now just sleep. We should really ask to inev + transaction to do the commit for us, and then we can + continue running. */ + dprintf(("_validate_and_attach(%p) failed, " + "waiting for inevitable\n", new)); + wait_for_other_inevitable(old); + } + dprintf(("_validate_and_attach(%p) failed, enter safepoint\n", new)); /* check for requested safe point. otherwise an INEV transaction @@ -1150,7 +1165,7 @@ /************************************************************/ -static void _finish_transaction(enum stm_event_e event) +static void _finish_transaction(enum stm_event_e event, bool was_inev) { stm_thread_local_t *tl = STM_SEGMENT->running_thread; @@ -1161,7 +1176,10 @@ list_clear(STM_PSEGMENT->objects_pointing_to_nursery); list_clear(STM_PSEGMENT->old_objects_with_cards_set); list_clear(STM_PSEGMENT->large_overflow_objects); - timing_event(tl, event); + if (was_inev) + timing_commit_inev_position(); + else + timing_event(tl, event); release_thread_segment(tl); /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ @@ -1223,6 +1241,7 @@ push_large_overflow_objects_to_other_segments(); /* push before validate. otherwise they are reachable too early */ + bool was_inev = STM_PSEGMENT->transaction_state == TS_INEVITABLE; _validate_and_add_to_commit_log(); @@ -1264,7 +1283,7 @@ /* done */ stm_thread_local_t *tl = STM_SEGMENT->running_thread; - _finish_transaction(STM_TRANSACTION_COMMIT); + _finish_transaction(STM_TRANSACTION_COMMIT, was_inev); /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ s_mutex_unlock(); @@ -1399,7 +1418,7 @@ : NURSERY_END; } - _finish_transaction(STM_TRANSACTION_ABORT); + _finish_transaction(STM_TRANSACTION_ABORT, false); /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ return tl; @@ -1441,6 +1460,8 @@ _validate_and_turn_inevitable(); STM_PSEGMENT->transaction_state = TS_INEVITABLE; + timing_record_inev_position(); + stm_rewind_jmp_forget(STM_SEGMENT->running_thread); invoke_and_clear_user_callbacks(0); /* for commit */ } diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -141,6 +141,9 @@ pthread_t running_pthread; #endif + /* marker where this thread became inevitable */ + stm_loc_marker_t marker_inev; + /* light finalizers */ struct list_s *young_objects_with_light_finalizers; struct list_s *old_objects_with_light_finalizers; diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -401,6 +401,9 @@ if (modified->type == TYPE_POSITION_MARKER) mark_visit_possibly_new_object(modified->marker_object, pseg); } + + if (pseg->transaction_state == TS_INEVITABLE) + mark_visit_possibly_new_object(pseg->marker_inev.object, pseg); } } diff --git a/c8/stm/marker.c b/c8/stm/marker.c --- a/c8/stm/marker.c +++ b/c8/stm/marker.c @@ -26,6 +26,8 @@ } else { /* no marker found */ + out_marker->odd_number = 0; + out_marker->object = NULL; return false; } } @@ -87,6 +89,34 @@ STM_CONTENTION_WRITE_READ, &marker); } +static void _timing_record_inev_position(void) +{ + stm_loc_marker_t marker; + marker_fetch(STM_SEGMENT->running_thread, &marker); + STM_PSEGMENT->marker_inev.odd_number = marker.odd_number; + STM_PSEGMENT->marker_inev.object = marker.object; +} + +static void _timing_commit_inev_position(void) +{ + stm_loc_marker_t marker; + marker.odd_number = STM_PSEGMENT->marker_inev.odd_number; + marker.object = STM_PSEGMENT->marker_inev.object; + stmcb_timing_event(STM_SEGMENT->running_thread, + STM_TRANSACTION_COMMIT, &marker); +} + +static void timing_wait_other_inevitable(void) +{ + if (stmcb_timing_event == NULL) + return; + + stm_loc_marker_t marker; + marker_fetch(STM_SEGMENT->running_thread, &marker); + stmcb_timing_event(STM_SEGMENT->running_thread, + STM_WAIT_OTHER_INEVITABLE, &marker); +} + void (*stmcb_timing_event)(stm_thread_local_t *tl, /* the local thread */ enum stm_event_e event, diff --git a/c8/stm/marker.h b/c8/stm/marker.h --- a/c8/stm/marker.h +++ b/c8/stm/marker.h @@ -2,10 +2,21 @@ static void _timing_record_write_position(void); static void timing_write_read_contention(struct stm_undo_s *start, struct stm_undo_s *contention); +static void _timing_record_inev_position(void); +static void _timing_commit_inev_position(void); +static void timing_wait_other_inevitable(void); +#define timing_enabled() (stmcb_timing_event != NULL) + #define timing_event(tl, event) \ - (stmcb_timing_event != NULL ? stmcb_timing_event(tl, event, NULL) : (void)0) + (timing_enabled() ? stmcb_timing_event(tl, event, NULL) : (void)0) #define timing_record_write_position() \ - (stmcb_timing_event != NULL ? _timing_record_write_position() : (void)0) + (timing_enabled() ? _timing_record_write_position() : (void)0) + +#define timing_record_inev_position() \ + (timing_enabled() ? _timing_record_inev_position() : (void)0) + +#define timing_commit_inev_position() \ + (timing_enabled() ? _timing_commit_inev_position() : (void)0) diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c --- a/c8/stm/nursery.c +++ b/c8/stm/nursery.c @@ -425,6 +425,13 @@ if (undo->type == TYPE_POSITION_MARKER) minor_trace_if_young(&undo->marker_object); } + + if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) { + object_t **pmarker_inev_obj = (object_t **) + REAL_ADDRESS(STM_SEGMENT->segment_base, + &STM_PSEGMENT->marker_inev.object); + minor_trace_if_young(pmarker_inev_obj); + } } static void collect_objs_still_young_but_with_finalizers(void) diff --git a/c8/stm/sync.h b/c8/stm/sync.h --- a/c8/stm/sync.h +++ b/c8/stm/sync.h @@ -33,3 +33,5 @@ static bool pause_signalled, globally_unique_transaction; static void enter_safe_point_if_requested(void); + +#define safe_point_requested() (STM_SEGMENT->nursery_end != NURSERY_END) diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -354,6 +354,12 @@ saying where the write was done. Followed by STM_TRANSACTION_ABORT. */ STM_CONTENTION_WRITE_READ, + /* inevitable contention: the thread that waited is + STM_WAIT_OTHER_INEVITABLE (with a marker) and the thread that + it waited for is the next STM_TRANSACTION_COMMIT (with a marker + as well; both markers point to the place that made each + transaction inevitable). */ + /* always one STM_WAIT_xxx followed later by STM_WAIT_DONE */ STM_WAIT_FREE_SEGMENT, STM_WAIT_OTHER_INEVITABLE, diff --git a/c8/test/test_marker.py b/c8/test/test_marker.py --- a/c8/test/test_marker.py +++ b/c8/test/test_marker.py @@ -119,20 +119,25 @@ py.test.raises(Conflict, self.switch, 1) self.check_recording(19, ffi.NULL) - def test_double_abort_markers_cb_inevitable(self): - self.recording(lib.STM_CONTENTION_INEVITABLE) + def test_commit_marker_for_inev(self): + self.recording(lib.STM_TRANSACTION_COMMIT) + # + self.switch(1) + self.start_transaction() + self.push_root(ffi.cast("object_t *", 19)) + self.push_root(ffi.cast("object_t *", ffi.NULL)) + self.become_inevitable() + self.pop_root() + self.pop_root() + self.commit_transaction() + # + self.check_recording(19, ffi.NULL) + + def test_abort_markers_cb_inevitable(self): + self.recording(lib.STM_WAIT_OTHER_INEVITABLE) # self.start_transaction() - p = stm_allocate(16) - stm_set_char(p, 'A') - self.push_root(ffi.cast("object_t *", 19)) - self.push_root(ffi.cast("object_t *", p)) self.become_inevitable() - self.pop_root() - self.pop_root() - self.push_root(ffi.cast("object_t *", 17)) - self.push_root(ffi.cast("object_t *", ffi.NULL)) - stm_minor_collect() # self.switch(1) self.start_transaction() @@ -142,7 +147,9 @@ self.push_root(ffi.cast("object_t *", p)) py.test.raises(Conflict, self.become_inevitable) # - self.check_recording(21, p, 19, p) + py.test.skip("XXX only during tests does become_inevitable() abort" + " and then it doesn't record anything") + self.check_recording(21, p) def test_read_write_contention(self): self.recording(lib.STM_CONTENTION_WRITE_READ) @@ -167,63 +174,6 @@ py.test.raises(Conflict, self.switch, 1) self.check_recording(19, ffi.NULL) - def test_double_remote_markers_cb_write_write(self): - self.recording(lib.STM_CONTENTION_WRITE_WRITE, - lib.STM_ABORTING_OTHER_CONTENTION) - p = stm_allocate_old(16) - # - self.start_transaction() - self.push_root(ffi.cast("object_t *", 19)) - self.push_root(ffi.cast("object_t *", ffi.NULL)) - stm_set_char(p, 'A') - self.pop_root() - self.pop_root() - self.push_root(ffi.cast("object_t *", 17)) - self.push_root(ffi.cast("object_t *", ffi.NULL)) - tl0 = self.get_stm_thread_local() - # - self.switch(1) - self.start_transaction() - self.become_inevitable() - self.push_root(ffi.cast("object_t *", 21)) - self.push_root(ffi.cast("object_t *", ffi.NULL)) - stm_set_char(p, 'B') # aborts in #0 - self.pop_root() - self.pop_root() - self.push_root(ffi.cast("object_t *", 23)) - self.push_root(ffi.cast("object_t *", ffi.NULL)) - # - py.test.raises(Conflict, self.switch, 0) - # - self.check_recording(21, ffi.NULL, 19, ffi.NULL, - extra=lib.STM_ABORTING_OTHER_CONTENTION) - - def test_double_remote_markers_cb_write_read(self): - self.recording(lib.STM_CONTENTION_WRITE_READ, - lib.STM_ABORTING_OTHER_CONTENTION) - p = stm_allocate_old(16) - # - self.start_transaction() - assert stm_get_char(p) == '\x00' # read - tl0 = self.get_stm_thread_local() - # - self.switch(1) - self.start_transaction() - self.become_inevitable() - self.push_root(ffi.cast("object_t *", 21)) - self.push_root(ffi.cast("object_t *", ffi.NULL)) - stm_set_char(p, 'B') # write, will abort #0 - self.pop_root() - self.pop_root() - self.push_root(ffi.cast("object_t *", 23)) - self.push_root(ffi.cast("object_t *", ffi.NULL)) - self.commit_transaction() - # - py.test.raises(Conflict, self.switch, 0) - # - self.check_recording(21, ffi.NULL, 0, ffi.NULL, - extra=lib.STM_ABORTING_OTHER_CONTENTION) - def test_all(self): self.recording() # all events self.start_transaction() From noreply at buildbot.pypy.org Mon Mar 9 10:13:47 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 9 Mar 2015 10:13:47 +0100 (CET) Subject: [pypy-commit] stmgc c8-marker: Re-add the remining timing_events Message-ID: <20150309091347.543FD1C04BE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-marker Changeset: r1703:a26d30f127f1 Date: 2015-03-09 10:14 +0100 http://bitbucket.org/pypy/stmgc/changeset/a26d30f127f1/ Log: Re-add the remining timing_events diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -1268,11 +1268,7 @@ /* if a major collection is required, do it here */ if (is_major_collection_requested()) { - synchronize_all_threads(STOP_OTHERS_UNTIL_MUTEX_UNLOCK); - - if (is_major_collection_requested()) { /* if *still* true */ - major_collection_now_at_safe_point(); - } + major_collection_with_mutex(); } _verify_cards_cleared_in_all_lists(get_priv_segment(STM_SEGMENT->segment_num)); diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -131,6 +131,19 @@ /************************************************************/ +static void major_collection_with_mutex(void) +{ + timing_event(STM_SEGMENT->running_thread, STM_GC_MAJOR_START); + + synchronize_all_threads(STOP_OTHERS_UNTIL_MUTEX_UNLOCK); + + if (is_major_collection_requested()) { /* if *still* true */ + major_collection_now_at_safe_point(); + } + + timing_event(STM_SEGMENT->running_thread, STM_GC_MAJOR_DONE); +} + static void major_collection_if_requested(void) { assert(!_has_mutex()); @@ -140,13 +153,7 @@ s_mutex_lock(); if (is_major_collection_requested()) { /* if still true */ - - synchronize_all_threads(STOP_OTHERS_UNTIL_MUTEX_UNLOCK); - - if (is_major_collection_requested()) { /* if *still* true */ - major_collection_now_at_safe_point(); - } - + major_collection_with_mutex(); } s_mutex_unlock(); diff --git a/c8/stm/gcpage.h b/c8/stm/gcpage.h --- a/c8/stm/gcpage.h +++ b/c8/stm/gcpage.h @@ -19,5 +19,6 @@ static void major_collection_if_requested(void); static void major_collection_now_at_safe_point(void); +static void major_collection_with_mutex(void); static bool largemalloc_keep_object_at(char *data); /* for largemalloc.c */ static bool smallmalloc_keep_object_at(char *data); /* for smallmalloc.c */ diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c --- a/c8/stm/nursery.c +++ b/c8/stm/nursery.c @@ -563,7 +563,11 @@ stm_safe_point(); + timing_event(STM_SEGMENT->running_thread, STM_GC_MINOR_START); + _do_minor_collection(commit); + + timing_event(STM_SEGMENT->running_thread, STM_GC_MINOR_DONE); } void stm_collect(long level) diff --git a/c8/stm/sync.c b/c8/stm/sync.c --- a/c8/stm/sync.c +++ b/c8/stm/sync.c @@ -139,10 +139,13 @@ } /* No segment available. Wait until release_thread_segment() signals that one segment has been freed. */ + timing_event(tl, STM_WAIT_FREE_SEGMENT); cond_wait(C_SEGMENT_FREE); + timing_event(tl, STM_WAIT_DONE); /* Return false to the caller, which will call us again */ return false; + got_num: OPT_ASSERT(num >= 0 && num < NB_SEGMENTS-1); sync_ctl.in_use1[num+1] = 1; @@ -296,10 +299,12 @@ #ifdef STM_TESTS abort_with_mutex(); #endif + timing_event(STM_SEGMENT->running_thread, STM_WAIT_SYNC_PAUSE); cond_signal(C_AT_SAFE_POINT); STM_PSEGMENT->safe_point = SP_WAIT_FOR_C_REQUEST_REMOVED; cond_wait(C_REQUEST_REMOVED); STM_PSEGMENT->safe_point = SP_RUNNING; + timing_event(STM_SEGMENT->running_thread, STM_WAIT_DONE); } } diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -362,6 +362,7 @@ /* always one STM_WAIT_xxx followed later by STM_WAIT_DONE */ STM_WAIT_FREE_SEGMENT, + STM_WAIT_SYNC_PAUSE, STM_WAIT_OTHER_INEVITABLE, STM_WAIT_DONE, diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -137,6 +137,7 @@ /* always one STM_WAIT_xxx followed later by STM_WAIT_DONE */ STM_WAIT_FREE_SEGMENT, + STM_WAIT_SYNC_PAUSE, STM_WAIT_OTHER_INEVITABLE, STM_WAIT_DONE, From noreply at buildbot.pypy.org Mon Mar 9 14:13:58 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 9 Mar 2015 14:13:58 +0100 (CET) Subject: [pypy-commit] pypy default: Another test for another part of cffi which can be at the wrong version... Message-ID: <20150309131358.4AE091C0221@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76292:f4b4cfce3e5b Date: 2015-03-09 14:13 +0100 http://bitbucket.org/pypy/pypy/changeset/f4b4cfce3e5b/ Log: Another test for another part of cffi which can be at the wrong version... diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "0.9.0" -__version_info__ = (0, 9, 0) +__version__ = "0.9.1" +__version_info__ = (0, 9, 1) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/pypy/module/_cffi_backend/test/test_file.py b/pypy/module/_cffi_backend/test/test_file.py --- a/pypy/module/_cffi_backend/test/test_file.py +++ b/pypy/module/_cffi_backend/test/test_file.py @@ -22,3 +22,8 @@ eggfile = py.path.local(__file__).join('..', '..', '..', '..', '..', 'lib_pypy', 'cffi.egg-info') assert line in eggfile.readlines() + +def test_app_version(): + from pypy.module import _cffi_backend + from lib_pypy import cffi + assert _cffi_backend.VERSION == cffi.__version__ From noreply at buildbot.pypy.org Mon Mar 9 15:56:26 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Mon, 9 Mar 2015 15:56:26 +0100 (CET) Subject: [pypy-commit] stmgc cl-collector-thread: branch for a separate commit-log collector thread to avoid triggering too many Message-ID: <20150309145626.B63D81C023F@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: cl-collector-thread Changeset: r1706:837fe4394dfc Date: 2015-03-09 13:24 +0100 http://bitbucket.org/pypy/stmgc/changeset/837fe4394dfc/ Log: branch for a separate commit-log collector thread to avoid triggering too many major collections From noreply at buildbot.pypy.org Mon Mar 9 15:56:27 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Mon, 9 Mar 2015 15:56:27 +0100 (CET) Subject: [pypy-commit] stmgc cl-collector-thread: move some commit log things to separate files Message-ID: <20150309145627.D55861C023F@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: cl-collector-thread Changeset: r1707:6f9eed8aeaa2 Date: 2015-03-09 13:35 +0100 http://bitbucket.org/pypy/stmgc/changeset/6f9eed8aeaa2/ Log: move some commit log things to separate files diff --git a/c8/stm/commitlog.c b/c8/stm/commitlog.c new file mode 100644 --- /dev/null +++ b/c8/stm/commitlog.c @@ -0,0 +1,55 @@ +#ifndef _STM_CORE_H_ +# error "must be compiled via stmgc.c" +#endif + + +static void free_bk(struct stm_undo_s *undo) +{ + free(undo->backup); + assert(undo->backup = (char*)-88); + increment_total_allocated(-SLICE_SIZE(undo->slice)); +} + +static struct stm_commit_log_entry_s *malloc_cle(long entries) +{ + size_t byte_len = sizeof(struct stm_commit_log_entry_s) + + entries * sizeof(struct stm_undo_s); + struct stm_commit_log_entry_s *result = malloc(byte_len); + increment_total_allocated(byte_len); + return result; +} + +static void free_cle(struct stm_commit_log_entry_s *e) +{ + size_t byte_len = sizeof(struct stm_commit_log_entry_s) + + e->written_count * sizeof(struct stm_undo_s); + increment_total_allocated(-byte_len); + free(e); +} + + +void _dbg_print_commit_log() +{ + struct stm_commit_log_entry_s *cl = &commit_log_root; + + fprintf(stderr, "commit log:\n"); + while (cl) { + fprintf(stderr, " entry at %p: seg %d, rev %lu\n", cl, cl->segment_num, cl->rev_num); + struct stm_undo_s *undo = cl->written; + struct stm_undo_s *end = undo + cl->written_count; + for (; undo < end; undo++) { + fprintf(stderr, " obj %p, size %d, ofs %lu: ", undo->object, + SLICE_SIZE(undo->slice), SLICE_OFFSET(undo->slice)); + /* long i; */ + /* for (i=0; islice); i += 8) */ + /* fprintf(stderr, " 0x%016lx", *(long *)(undo->backup + i)); */ + fprintf(stderr, "\n"); + } + + cl = cl->next; + if (cl == INEV_RUNNING) { + fprintf(stderr, " INEVITABLE\n"); + return; + } + } +} diff --git a/c8/stm/commitlog.h b/c8/stm/commitlog.h new file mode 100644 --- /dev/null +++ b/c8/stm/commitlog.h @@ -0,0 +1,40 @@ + + +/* Commit Log things */ +struct stm_undo_s { + object_t *object; /* the object that is modified */ + char *backup; /* some backup data (a slice of the original obj) */ + uint64_t slice; /* location and size of this slice (cannot cross + pages). The size is in the lower 2 bytes, and + the offset in the remaining 6 bytes. */ +}; +#define SLICE_OFFSET(slice) ((slice) >> 16) +#define SLICE_SIZE(slice) ((int)((slice) & 0xFFFF)) +#define NEW_SLICE(offset, size) (((uint64_t)(offset)) << 16 | (size)) + + + +/* The model is: we have a global chained list, from 'commit_log_root', + of 'struct stm_commit_log_entry_s' entries. Every one is fully + read-only apart from the 'next' field. Every one stands for one + commit that occurred. It lists the old objects that were modified + in this commit, and their attached "undo logs" --- that is, the + data from 'written[n].backup' is the content of (slices of) the + object as they were *before* that commit occurred. +*/ +#define INEV_RUNNING ((void*)-1) +struct stm_commit_log_entry_s { + struct stm_commit_log_entry_s *volatile next; + int segment_num; + uint64_t rev_num; + size_t written_count; + struct stm_undo_s written[]; +}; +static struct stm_commit_log_entry_s commit_log_root; + + +static void free_bk(struct stm_undo_s *undo); +static struct stm_commit_log_entry_s *malloc_cle(long entries); +static void free_cle(struct stm_commit_log_entry_s *e); + +void _dbg_print_commit_log(); diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -2,31 +2,6 @@ # error "must be compiled via stmgc.c" #endif -/* *** MISC *** */ -static void free_bk(struct stm_undo_s *undo) -{ - free(undo->backup); - assert(undo->backup = (char*)-88); - increment_total_allocated(-SLICE_SIZE(undo->slice)); -} - -static struct stm_commit_log_entry_s *malloc_cle(long entries) -{ - size_t byte_len = sizeof(struct stm_commit_log_entry_s) + - entries * sizeof(struct stm_undo_s); - struct stm_commit_log_entry_s *result = malloc(byte_len); - increment_total_allocated(byte_len); - return result; -} - -static void free_cle(struct stm_commit_log_entry_s *e) -{ - size_t byte_len = sizeof(struct stm_commit_log_entry_s) + - e->written_count * sizeof(struct stm_undo_s); - increment_total_allocated(-byte_len); - free(e); -} -/* *** MISC *** */ /* General helper: copies objects into our own segment, from some @@ -259,31 +234,6 @@ /* ############# commit log ############# */ -void _dbg_print_commit_log() -{ - struct stm_commit_log_entry_s *cl = &commit_log_root; - - fprintf(stderr, "commit log:\n"); - while (cl) { - fprintf(stderr, " entry at %p: seg %d, rev %lu\n", cl, cl->segment_num, cl->rev_num); - struct stm_undo_s *undo = cl->written; - struct stm_undo_s *end = undo + cl->written_count; - for (; undo < end; undo++) { - fprintf(stderr, " obj %p, size %d, ofs %lu: ", undo->object, - SLICE_SIZE(undo->slice), SLICE_OFFSET(undo->slice)); - /* long i; */ - /* for (i=0; islice); i += 8) */ - /* fprintf(stderr, " 0x%016lx", *(long *)(undo->backup + i)); */ - fprintf(stderr, "\n"); - } - - cl = cl->next; - if (cl == INEV_RUNNING) { - fprintf(stderr, " INEVITABLE\n"); - return; - } - } -} static void reset_modified_from_backup_copies(int segment_num); /* forward */ diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -172,43 +172,9 @@ TS_INEVITABLE, }; -/* Commit Log things */ -struct stm_undo_s { - object_t *object; /* the object that is modified */ - char *backup; /* some backup data (a slice of the original obj) */ - uint64_t slice; /* location and size of this slice (cannot cross - pages). The size is in the lower 2 bytes, and - the offset in the remaining 6 bytes. */ -}; -#define SLICE_OFFSET(slice) ((slice) >> 16) -#define SLICE_SIZE(slice) ((int)((slice) & 0xFFFF)) -#define NEW_SLICE(offset, size) (((uint64_t)(offset)) << 16 | (size)) -/* The model is: we have a global chained list, from 'commit_log_root', - of 'struct stm_commit_log_entry_s' entries. Every one is fully - read-only apart from the 'next' field. Every one stands for one - commit that occurred. It lists the old objects that were modified - in this commit, and their attached "undo logs" --- that is, the - data from 'written[n].backup' is the content of (slices of) the - object as they were *before* that commit occurred. -*/ -#define INEV_RUNNING ((void*)-1) -struct stm_commit_log_entry_s { - struct stm_commit_log_entry_s *volatile next; - int segment_num; - uint64_t rev_num; - size_t written_count; - struct stm_undo_s written[]; -}; -static struct stm_commit_log_entry_s commit_log_root; - - -static void free_bk(struct stm_undo_s *undo); -static struct stm_commit_log_entry_s *malloc_cle(long entries); -static void free_cle(struct stm_commit_log_entry_s *e); - #ifndef STM_TESTS static char *stm_object_pages; diff --git a/c8/stmgc.c b/c8/stmgc.c --- a/c8/stmgc.c +++ b/c8/stmgc.c @@ -3,6 +3,7 @@ #include "stm/atomic.h" #include "stm/list.h" #include "stm/smallmalloc.h" +#include "stm/commitlog.h" #include "stm/core.h" #include "stm/pagecopy.h" #include "stm/pages.h" @@ -20,6 +21,7 @@ #include "stm/misc.c" #include "stm/list.c" #include "stm/smallmalloc.c" +#include "stm/commitlog.c" #include "stm/pagecopy.c" #include "stm/pages.c" #include "stm/prebuilt.c" From noreply at buildbot.pypy.org Mon Mar 9 15:56:28 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Mon, 9 Mar 2015 15:56:28 +0100 (CET) Subject: [pypy-commit] stmgc cl-collector-thread: move some more code, separate accounting for cl-entries, and adapt tests Message-ID: <20150309145628.E44241C023F@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: cl-collector-thread Changeset: r1708:3387438f8cc4 Date: 2015-03-09 13:57 +0100 http://bitbucket.org/pypy/stmgc/changeset/3387438f8cc4/ Log: move some more code, separate accounting for cl-entries, and adapt tests diff --git a/c8/stm/commitlog.c b/c8/stm/commitlog.c --- a/c8/stm/commitlog.c +++ b/c8/stm/commitlog.c @@ -2,12 +2,44 @@ # error "must be compiled via stmgc.c" #endif +uint64_t cle_allocated; + +static void setup_commitlog(void) +{ + cle_allocated = 0; + commit_log_root.next = NULL; + commit_log_root.segment_num = -1; + commit_log_root.rev_num = 0; + commit_log_root.written_count = 0; +} + +static void teardown_commitlog(void) +{ + cle_allocated = 0; + commit_log_root.next = NULL; /* xxx:free them */ + commit_log_root.segment_num = -1; +} + +static void add_cle_allocated(ssize_t add_or_remove) +{ + __sync_add_and_fetch(&cle_allocated, add_or_remove); +} + +uint64_t _stm_cle_allocated(void) { + return cle_allocated; +} + +static char *malloc_bk(size_t bk_size) +{ + add_cle_allocated(bk_size); + return malloc(bk_size); +} static void free_bk(struct stm_undo_s *undo) { free(undo->backup); assert(undo->backup = (char*)-88); - increment_total_allocated(-SLICE_SIZE(undo->slice)); + add_cle_allocated(-SLICE_SIZE(undo->slice)); } static struct stm_commit_log_entry_s *malloc_cle(long entries) @@ -15,7 +47,7 @@ size_t byte_len = sizeof(struct stm_commit_log_entry_s) + entries * sizeof(struct stm_undo_s); struct stm_commit_log_entry_s *result = malloc(byte_len); - increment_total_allocated(byte_len); + add_cle_allocated(byte_len); return result; } @@ -23,7 +55,7 @@ { size_t byte_len = sizeof(struct stm_commit_log_entry_s) + e->written_count * sizeof(struct stm_undo_s); - increment_total_allocated(-byte_len); + add_cle_allocated(-byte_len); free(e); } diff --git a/c8/stm/commitlog.h b/c8/stm/commitlog.h --- a/c8/stm/commitlog.h +++ b/c8/stm/commitlog.h @@ -1,3 +1,7 @@ + + +/* when to trigger a CLE collection */ +#define CLE_COLLECT_BOUND (1*1024*1024) /* 1 MiB */ /* Commit Log things */ @@ -33,8 +37,13 @@ static struct stm_commit_log_entry_s commit_log_root; +static char *malloc_bk(size_t bk_size); static void free_bk(struct stm_undo_s *undo); static struct stm_commit_log_entry_s *malloc_cle(long entries); static void free_cle(struct stm_commit_log_entry_s *e); void _dbg_print_commit_log(); + +#ifdef STM_TESTS +uint64_t _stm_cle_allocated(void); +#endif diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -231,7 +231,6 @@ /* now return and retry */ } -/* ############# commit log ############# */ @@ -573,8 +572,7 @@ in_page_offset = (in_page_offset + slice_sz) % 4096UL; /* mostly 0 */ /* make backup slice: */ - char *bk_slice = malloc(slice_sz); - increment_total_allocated(slice_sz); + char *bk_slice = malloc_bk(slice_sz); memcpy(bk_slice, realobj + slice_off, slice_sz); acquire_modification_lock(STM_SEGMENT->segment_num); diff --git a/c8/stm/setup.c b/c8/stm/setup.c --- a/c8/stm/setup.c +++ b/c8/stm/setup.c @@ -78,10 +78,6 @@ setup_protection_settings(); setup_signal_handler(); - commit_log_root.next = NULL; - commit_log_root.segment_num = -1; - commit_log_root.rev_num = 0; - commit_log_root.written_count = 0; long i; /* including seg0 */ @@ -133,6 +129,7 @@ setup_pages(); setup_forksupport(); setup_finalizer(); + setup_commitlog(); set_gs_register(get_segment_base(0)); } @@ -166,14 +163,13 @@ munmap(stm_object_pages, TOTAL_MEMORY); stm_object_pages = NULL; - commit_log_root.next = NULL; /* xxx:free them */ - commit_log_root.segment_num = -1; teardown_finalizer(); teardown_sync(); teardown_gcpage(); teardown_smallmalloc(); teardown_pages(); + teardown_commitlog(); } static void _shadowstack_trap_page(char *start, int prot) diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -138,6 +138,7 @@ long _stm_count_old_objects_with_cards_set(void); object_t *_stm_enum_old_objects_with_cards_set(long index); uint64_t _stm_total_allocated(void); +uint64_t _stm_cle_allocated(void); #endif /* ==================== HELPERS ==================== */ diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -104,6 +104,7 @@ /* void stm_collect(long level); */ long _check_stm_collect(long level); uint64_t _stm_total_allocated(void); +uint64_t _stm_cle_allocated(void); void _stm_set_nursery_free_count(uint64_t free_count); void _stm_largemalloc_init_arena(char *data_start, size_t data_size); diff --git a/c8/test/test_gcpage.py b/c8/test/test_gcpage.py --- a/c8/test/test_gcpage.py +++ b/c8/test/test_gcpage.py @@ -140,7 +140,8 @@ def test_account_for_everything(self): self.start_transaction() self.commit_transaction() - assert lib._stm_total_allocated() == CLEO + assert lib._stm_total_allocated() == 0 + assert lib._stm_cle_allocated() == CLEO self.start_transaction() o = stm_allocate(5008) @@ -148,7 +149,8 @@ self.commit_transaction() assert last_commit_log_entry_objs() == [] # 2 CLEs, 1 old object - assert lib._stm_total_allocated() == 2*CLEO + (5008 + LMO) + assert lib._stm_total_allocated() == 5008 + LMO + assert lib._stm_cle_allocated() == 2*CLEO self.start_transaction() o = self.pop_root() @@ -158,13 +160,16 @@ assert last_commit_log_entry_objs() == [o]*2 # 3 CLEs, 1 old object # also, 2 slices of bk_copy and thus 2 CLE entries - assert lib._stm_total_allocated() == 3*CLEO + (5008+LMO) + (5008 + CLEEO*2) + assert lib._stm_total_allocated() == 5008+LMO + assert lib._stm_cle_allocated() == 3*CLEO + (5008 + CLEEO*2) self.start_transaction() - assert lib._stm_total_allocated() == 3*CLEO + (5008+LMO) + (5008 + CLEEO*2) + assert lib._stm_total_allocated() == 5008+LMO + assert lib._stm_cle_allocated() == 3*CLEO + (5008 + CLEEO*2) stm_major_collect() # all CLE and CLE entries freed: assert lib._stm_total_allocated() == (5008+LMO) + assert lib._stm_cle_allocated() == 0 self.commit_transaction() @@ -202,12 +207,14 @@ stm_set_char(x, 'a', 4999) self.push_root(x) self.commit_transaction() - assert lib._stm_total_allocated() == 5008 + LMO + CLEO + assert lib._stm_total_allocated() == 5008 + LMO + assert lib._stm_cle_allocated() == CLEO self.start_transaction() x = self.pop_root() self.push_root(x) - assert lib._stm_total_allocated() == 5008 + LMO + CLEO + assert lib._stm_total_allocated() == 5008 + LMO + assert lib._stm_cle_allocated() == CLEO stm_set_char(x, 'B') stm_set_char(x, 'b', 4999) From noreply at buildbot.pypy.org Mon Mar 9 15:56:29 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Mon, 9 Mar 2015 15:56:29 +0100 (CET) Subject: [pypy-commit] stmgc cl-collector-thread: fixes, call CL-collection on segment release, do validation of non-running Message-ID: <20150309145629.E53201C023F@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: cl-collector-thread Changeset: r1709:caba34127c17 Date: 2015-03-09 15:47 +0100 http://bitbucket.org/pypy/stmgc/changeset/caba34127c17/ Log: fixes, call CL-collection on segment release, do validation of non- running segments in CL-collection diff --git a/c8/stm/commitlog.c b/c8/stm/commitlog.c --- a/c8/stm/commitlog.c +++ b/c8/stm/commitlog.c @@ -25,7 +25,13 @@ __sync_add_and_fetch(&cle_allocated, add_or_remove); } -uint64_t _stm_cle_allocated(void) { +static bool is_cle_collection_requested(void) +{ + return cle_allocated > CLE_COLLECT_BOUND; +} + +uint64_t _stm_cle_allocated(void) +{ return cle_allocated; } @@ -53,6 +59,7 @@ static void free_cle(struct stm_commit_log_entry_s *e) { + dprintf(("free_cle(%p): written=%lu\n", e, e->written_count)); size_t byte_len = sizeof(struct stm_commit_log_entry_s) + e->written_count * sizeof(struct stm_undo_s); add_cle_allocated(-byte_len); @@ -85,3 +92,110 @@ } } } + + + +void free_commit_log_entries_up_to(struct stm_commit_log_entry_s *end) +{ + struct stm_commit_log_entry_s *cl, *next; + + cl = &commit_log_root; + assert(cl->next != NULL && cl->next != INEV_RUNNING); + assert(end->next != NULL && end->next != INEV_RUNNING); + assert(cl != end); + + uint64_t rev_num = -1; + next = cl->next; /* guaranteed to exist */ + do { + cl = next; + rev_num = cl->rev_num; + + /* free bk copies of entries: */ + long count = cl->written_count; + while (count-->0) { + free_bk(&cl->written[count]); + } + + next = cl->next; + free_cle(cl); + + assert(next != INEV_RUNNING); + if (cl == end) { + /* was the last one to free */ + break; + } + } while (next != NULL); + + /* set the commit_log_root to the last, common cl entry: */ + commit_log_root.next = next; + commit_log_root.rev_num = rev_num; +} + +void maybe_collect_commit_log() +{ + /* XXX: maybe use other lock, but right now we must make sure + that we do not run a major GC in parallel, since we do a + validate in some segments. */ + assert(_has_mutex()); + + if (!is_cle_collection_requested()) + return; + + /* do validation in segments with no threads running, as some + of them may rarely run a thread and thus rarely advance in + the commit log. */ + int original_num = STM_SEGMENT->segment_num; + for (long i = 0; i < NB_SEGMENTS; i++) { + struct stm_priv_segment_info_s *pseg = get_priv_segment(i); + + if (pseg->pub.running_thread == NULL) { + assert(pseg->transaction_state == TS_NONE); + + set_gs_register(get_segment_base(i)); + bool ok = _stm_validate(); + OPT_ASSERT(IMPLY(STM_PSEGMENT->transaction_state != TS_NONE, + ok)); + /* only TS_NONE segments may have stale read-marker data + that reports a conflict here. but it is fine to ignore it. */ + } + } + set_gs_register(get_segment_base(original_num)); + + + /* look for the last common commit log entry. However, + don't free the last common one, as it may still be + used by running threads. */ + struct stm_commit_log_entry_s *cl, *next; + + cl = &commit_log_root; + next = cl; + do { + bool found = false; + for (int i = 0; i < NB_SEGMENTS; i++) { + struct stm_commit_log_entry_s *tmp; + tmp = get_priv_segment(i)->last_commit_log_entry; + if (next == tmp) { + found = true; + break; + } + } + + if (found) { + /* cl is the last one to consider */ + break; + } else { + cl = next; + } + } while ((next = cl->next) && next != INEV_RUNNING); + + + if (cl == &commit_log_root) { + dprintf(("WARN: triggered CLE collection w/o anything to do\n")); + return; /* none found that is newer */ + } + + /* cl should never be the last (common) one */ + assert(cl->next != NULL && cl->next != INEV_RUNNING); + + free_commit_log_entries_up_to(cl); +} diff --git a/c8/stm/commitlog.h b/c8/stm/commitlog.h --- a/c8/stm/commitlog.h +++ b/c8/stm/commitlog.h @@ -1,7 +1,11 @@ /* when to trigger a CLE collection */ -#define CLE_COLLECT_BOUND (1*1024*1024) /* 1 MiB */ +#ifndef STM_TESTS +#define CLE_COLLECT_BOUND (30*1024*1024) /* 30 MiB */ +#else +#define CLE_COLLECT_BOUND 0 /* ASAP! */ +#endif /* Commit Log things */ @@ -37,6 +41,8 @@ static struct stm_commit_log_entry_s commit_log_root; +static bool is_cle_collection_requested(void); + static char *malloc_bk(size_t bk_size); static void free_bk(struct stm_undo_s *undo); static struct stm_commit_log_entry_s *malloc_cle(long entries); diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -573,62 +573,6 @@ _stm_smallmalloc_sweep(); } -static void clean_up_commit_log_entries() -{ - struct stm_commit_log_entry_s *cl, *next; - -#ifndef NDEBUG - /* check that all segments are at the same revision: */ - cl = get_priv_segment(0)->last_commit_log_entry; - for (long i = 0; i < NB_SEGMENTS; i++) { - struct stm_priv_segment_info_s *pseg = get_priv_segment(i); - assert(pseg->last_commit_log_entry == cl); - } -#endif - - /* if there is only one element, we don't have to do anything: */ - cl = &commit_log_root; - - if (cl->next == NULL || cl->next == INEV_RUNNING) { - assert(get_priv_segment(0)->last_commit_log_entry == cl); - return; - } - - bool was_inev = false; - uint64_t rev_num = -1; - - next = cl->next; /* guaranteed to exist */ - do { - cl = next; - rev_num = cl->rev_num; - - /* free bk copies of entries: */ - long count = cl->written_count; - while (count-->0) { - free_bk(&cl->written[count]); - } - - next = cl->next; - free_cle(cl); - if (next == INEV_RUNNING) { - was_inev = true; - break; - } - } while (next != NULL); - - /* set the commit_log_root to the last, common cl entry: */ - commit_log_root.next = was_inev ? INEV_RUNNING : NULL; - commit_log_root.rev_num = rev_num; - - /* update in all segments: */ - for (long i = 0; i < NB_SEGMENTS; i++) { - get_priv_segment(i)->last_commit_log_entry = &commit_log_root; - } - - assert(_stm_count_cl_entries() == 0); -} - - static void major_collection_now_at_safe_point(void) { @@ -644,37 +588,6 @@ dprintf((" | commit log entries before: %ld\n", _stm_count_cl_entries())); - - /* free all commit log entries. all segments are on the most recent - revision now. */ - uint64_t allocd_before = pages_ctl.total_allocated; - clean_up_commit_log_entries(); - /* check if freeing the log entries actually freed a considerable - amount itself. Then we don't want to also trace the whole heap - and just leave major gc right here. - The problem is apparent from raytrace.py, but may disappear if - we have card marking that also reduces the size of commit log - entries */ - if ((pages_ctl.total_allocated < pages_ctl.total_allocated_bound) - && (allocd_before - pages_ctl.total_allocated > 0.3 * allocd_before)) { - /* 0.3 should mean that we are at about 50% of the way to the - allocated_bound again */ -#ifndef STM_TESTS - /* we freed a considerable amount just by freeing commit log entries */ - pages_ctl.major_collection_requested = false; // reset_m_gc_requested - - dprintf(("STOP AFTER FREEING CL ENTRIES: -%ld\n", - (long)(allocd_before - pages_ctl.total_allocated))); - dprintf((" | used after collection: %ld\n", - (long)pages_ctl.total_allocated)); - dprintf((" `----------------------------------------------\n")); - if (must_abort()) - abort_with_mutex(); - - return; -#endif - } - /* only necessary because of assert that fails otherwise (XXX) */ acquire_all_privatization_locks(); diff --git a/c8/stm/sync.c b/c8/stm/sync.c --- a/c8/stm/sync.c +++ b/c8/stm/sync.c @@ -166,6 +166,10 @@ assert(sync_ctl.in_use1[tl->last_associated_segment_num] == 1); sync_ctl.in_use1[tl->last_associated_segment_num] = 0; + + /* check for cle collection here after we released the segment + while we still hold the mutex */ + maybe_collect_commit_log(); } __attribute__((unused)) diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -533,7 +533,6 @@ def stm_major_collect(): res = lib._check_stm_collect(1) - assert count_commit_log_entries() == 0 if res == 1: raise Conflict() return res diff --git a/c8/test/test_basic.py b/c8/test/test_basic.py --- a/c8/test/test_basic.py +++ b/c8/test/test_basic.py @@ -40,7 +40,7 @@ self.commit_transaction() assert last_commit_log_entry_objs() == [] - assert count_commit_log_entries() == 2 + assert count_commit_log_entries() == 1 def test_simple_read(self): self.start_transaction() diff --git a/c8/test/test_gcpage.py b/c8/test/test_gcpage.py --- a/c8/test/test_gcpage.py +++ b/c8/test/test_gcpage.py @@ -150,7 +150,8 @@ assert last_commit_log_entry_objs() == [] # 2 CLEs, 1 old object assert lib._stm_total_allocated() == 5008 + LMO - assert lib._stm_cle_allocated() == 2*CLEO + # however, on commit, we could free 1/2 CLE + assert lib._stm_cle_allocated() == 1*CLEO self.start_transaction() o = self.pop_root() @@ -158,18 +159,18 @@ self.push_root(o) self.commit_transaction() assert last_commit_log_entry_objs() == [o]*2 - # 3 CLEs, 1 old object + # 2 CLEs, 1 old object # also, 2 slices of bk_copy and thus 2 CLE entries assert lib._stm_total_allocated() == 5008+LMO - assert lib._stm_cle_allocated() == 3*CLEO + (5008 + CLEEO*2) + # however, on commit, we could free 1/2 CLE + assert lib._stm_cle_allocated() == 1*CLEO + (5008 + CLEEO*2) self.start_transaction() assert lib._stm_total_allocated() == 5008+LMO - assert lib._stm_cle_allocated() == 3*CLEO + (5008 + CLEEO*2) + assert lib._stm_cle_allocated() == 1*CLEO + (5008 + CLEEO*2) stm_major_collect() - # all CLE and CLE entries freed: assert lib._stm_total_allocated() == (5008+LMO) - assert lib._stm_cle_allocated() == 0 + assert lib._stm_cle_allocated() == 1*CLEO + (5008 + CLEEO*2) self.commit_transaction() @@ -456,14 +457,16 @@ self.start_transaction() stm_major_collect() self.commit_transaction() + assert count_commit_log_entries() == 1 self.switch(1) self.start_transaction() self.commit_transaction() + assert count_commit_log_entries() == 1 + py.test.xfail("XXX: we never completely free the CLEs anymore") self.switch(0) self.start_transaction() - assert count_commit_log_entries() == 2 stm_major_collect() assert count_commit_log_entries() == 0 From noreply at buildbot.pypy.org Tue Mar 10 02:51:31 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Tue, 10 Mar 2015 02:51:31 +0100 (CET) Subject: [pypy-commit] pypy default: move a test to the right location Message-ID: <20150310015131.A57C71C0196@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r76293:1d15f3b8dad9 Date: 2015-03-10 01:52 +0000 http://bitbucket.org/pypy/pypy/changeset/1d15f3b8dad9/ Log: move a test to the right location diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py --- a/rpython/annotator/test/test_annrpython.py +++ b/rpython/annotator/test/test_annrpython.py @@ -4326,13 +4326,6 @@ assert isinstance(s, annmodel.SomeString) assert not s.can_be_none() - def test_nonnulify(self): - s = annmodel.SomeString(can_be_None=True).nonnulify() - assert s.can_be_None is True - assert s.no_nul is True - s = annmodel.SomeChar().nonnulify() - assert s.no_nul is True - def g(n): return [0, 1, 2, n] diff --git a/rpython/annotator/test/test_model.py b/rpython/annotator/test/test_model.py --- a/rpython/annotator/test/test_model.py +++ b/rpython/annotator/test/test_model.py @@ -117,3 +117,11 @@ assert s_int != SomeInteger() assert not_const(s_int) == SomeInteger() assert not_const(s_None) == s_None + + +def test_nonnulify(): + s = SomeString(can_be_None=True).nonnulify() + assert s.can_be_None is True + assert s.no_nul is True + s = SomeChar().nonnulify() + assert s.no_nul is True From noreply at buildbot.pypy.org Tue Mar 10 08:56:40 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 10 Mar 2015 08:56:40 +0100 (CET) Subject: [pypy-commit] pypy.org extradoc: update the values Message-ID: <20150310075640.3F2F41C1150@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r579:5bcbea801995 Date: 2015-03-10 08:57 +0100 http://bitbucket.org/pypy/pypy.org/changeset/5bcbea801995/ Log: update the values diff --git a/don4.html b/don4.html --- a/don4.html +++ b/don4.html @@ -17,7 +17,7 @@ 2nd call: - $22583 of $80000 (28.2%) + $22593 of $80000 (28.2%)
From noreply at buildbot.pypy.org Tue Mar 10 15:56:18 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 10 Mar 2015 15:56:18 +0100 (CET) Subject: [pypy-commit] stmgc c8-marker: Improve the markers for inevitable transactions Message-ID: <20150310145618.DDEC51C015A@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-marker Changeset: r1710:a4e4d3ad014a Date: 2015-03-10 15:56 +0100 http://bitbucket.org/pypy/stmgc/changeset/a4e4d3ad014a/ Log: Improve the markers for inevitable transactions diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -464,7 +464,7 @@ static void wait_for_other_inevitable(struct stm_commit_log_entry_s *old) { - timing_wait_other_inevitable(); + timing_event(STM_SEGMENT->running_thread, STM_WAIT_OTHER_INEVITABLE); while (old->next == INEV_RUNNING && !safe_point_requested()) { spin_loop(); @@ -1171,7 +1171,7 @@ /************************************************************/ -static void _finish_transaction(enum stm_event_e event, bool was_inev) +static void _finish_transaction(enum stm_event_e event) { stm_thread_local_t *tl = STM_SEGMENT->running_thread; @@ -1182,10 +1182,7 @@ list_clear(STM_PSEGMENT->objects_pointing_to_nursery); list_clear(STM_PSEGMENT->old_objects_with_cards_set); list_clear(STM_PSEGMENT->large_overflow_objects); - if (was_inev) - timing_commit_inev_position(); - else - timing_event(tl, event); + timing_event(tl, event); release_thread_segment(tl); /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ @@ -1285,7 +1282,7 @@ /* done */ stm_thread_local_t *tl = STM_SEGMENT->running_thread; - _finish_transaction(STM_TRANSACTION_COMMIT, was_inev); + _finish_transaction(STM_TRANSACTION_COMMIT); /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ s_mutex_unlock(); @@ -1420,7 +1417,7 @@ : NURSERY_END; } - _finish_transaction(STM_TRANSACTION_ABORT, false); + _finish_transaction(STM_TRANSACTION_ABORT); /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ return tl; @@ -1459,10 +1456,10 @@ if (STM_PSEGMENT->transaction_state == TS_REGULAR) { dprintf(("become_inevitable: %s\n", msg)); _stm_collectable_safe_point(); + timing_become_inevitable(); _validate_and_turn_inevitable(); STM_PSEGMENT->transaction_state = TS_INEVITABLE; - timing_record_inev_position(); stm_rewind_jmp_forget(STM_SEGMENT->running_thread); invoke_and_clear_user_callbacks(0); /* for commit */ diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -141,9 +141,6 @@ pthread_t running_pthread; #endif - /* marker where this thread became inevitable */ - stm_loc_marker_t marker_inev; - /* light finalizers */ struct list_s *young_objects_with_light_finalizers; struct list_s *old_objects_with_light_finalizers; diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -408,9 +408,6 @@ if (modified->type == TYPE_POSITION_MARKER) mark_visit_possibly_new_object(modified->marker_object, pseg); } - - if (pseg->transaction_state == TS_INEVITABLE) - mark_visit_possibly_new_object(pseg->marker_inev.object, pseg); } } diff --git a/c8/stm/marker.c b/c8/stm/marker.c --- a/c8/stm/marker.c +++ b/c8/stm/marker.c @@ -89,32 +89,12 @@ STM_CONTENTION_WRITE_READ, &marker); } -static void _timing_record_inev_position(void) +static void _timing_become_inevitable(void) { stm_loc_marker_t marker; marker_fetch(STM_SEGMENT->running_thread, &marker); - STM_PSEGMENT->marker_inev.odd_number = marker.odd_number; - STM_PSEGMENT->marker_inev.object = marker.object; -} - -static void _timing_commit_inev_position(void) -{ - stm_loc_marker_t marker; - marker.odd_number = STM_PSEGMENT->marker_inev.odd_number; - marker.object = STM_PSEGMENT->marker_inev.object; stmcb_timing_event(STM_SEGMENT->running_thread, - STM_TRANSACTION_COMMIT, &marker); -} - -static void timing_wait_other_inevitable(void) -{ - if (stmcb_timing_event == NULL) - return; - - stm_loc_marker_t marker; - marker_fetch(STM_SEGMENT->running_thread, &marker); - stmcb_timing_event(STM_SEGMENT->running_thread, - STM_WAIT_OTHER_INEVITABLE, &marker); + STM_BECOME_INEVITABLE, &marker); } diff --git a/c8/stm/marker.h b/c8/stm/marker.h --- a/c8/stm/marker.h +++ b/c8/stm/marker.h @@ -2,9 +2,7 @@ static void _timing_record_write_position(void); static void timing_write_read_contention(struct stm_undo_s *start, struct stm_undo_s *contention); -static void _timing_record_inev_position(void); -static void _timing_commit_inev_position(void); -static void timing_wait_other_inevitable(void); +static void _timing_become_inevitable(void); #define timing_enabled() (stmcb_timing_event != NULL) @@ -15,8 +13,5 @@ #define timing_record_write_position() \ (timing_enabled() ? _timing_record_write_position() : (void)0) -#define timing_record_inev_position() \ - (timing_enabled() ? _timing_record_inev_position() : (void)0) - -#define timing_commit_inev_position() \ - (timing_enabled() ? _timing_commit_inev_position() : (void)0) +#define timing_become_inevitable() \ + (timing_enabled() ? _timing_become_inevitable() : (void)0) diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c --- a/c8/stm/nursery.c +++ b/c8/stm/nursery.c @@ -425,13 +425,6 @@ if (undo->type == TYPE_POSITION_MARKER) minor_trace_if_young(&undo->marker_object); } - - if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) { - object_t **pmarker_inev_obj = (object_t **) - REAL_ADDRESS(STM_SEGMENT->segment_base, - &STM_PSEGMENT->marker_inev.object); - minor_trace_if_young(pmarker_inev_obj); - } } static void collect_objs_still_young_but_with_finalizers(void) diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -354,11 +354,12 @@ saying where the write was done. Followed by STM_TRANSACTION_ABORT. */ STM_CONTENTION_WRITE_READ, - /* inevitable contention: the thread that waited is - STM_WAIT_OTHER_INEVITABLE (with a marker) and the thread that - it waited for is the next STM_TRANSACTION_COMMIT (with a marker - as well; both markers point to the place that made each - transaction inevitable). */ + /* inevitable contention: all threads that try to become inevitable + have a STM_BECOME_INEVITABLE event with a position marker. Then, + if it waits it gets a STM_WAIT_OTHER_INEVITABLE. It is possible + that a thread gets STM_BECOME_INEVITABLE followed by + STM_TRANSACTION_ABORT if it fails to become inevitable. */ + STM_BECOME_INEVITABLE, /* always one STM_WAIT_xxx followed later by STM_WAIT_DONE */ STM_WAIT_FREE_SEGMENT, diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -135,6 +135,13 @@ saying where the write was done. Followed by STM_TRANSACTION_ABORT. */ STM_CONTENTION_WRITE_READ, + /* inevitable contention: all threads that try to become inevitable + have a STM_BECOME_INEVITABLE event with a position marker. Then, + if it waits it gets a STM_WAIT_OTHER_INEVITABLE. It is possible + that a thread gets STM_BECOME_INEVITABLE followed by + STM_TRANSACTION_ABORT if it fails to become inevitable. */ + STM_BECOME_INEVITABLE, + /* always one STM_WAIT_xxx followed later by STM_WAIT_DONE */ STM_WAIT_FREE_SEGMENT, STM_WAIT_SYNC_PAUSE, diff --git a/c8/test/test_marker.py b/c8/test/test_marker.py --- a/c8/test/test_marker.py +++ b/c8/test/test_marker.py @@ -19,16 +19,13 @@ self.timing_event_keepalive = timing_event self.seen = seen - def check_recording(self, i1, o1, extra=None): + def check_recording(self, i1, o1, generating_thread_num=1): seen = self.seen tl, event, marker = seen[0] - assert tl == self.tls[1] + assert tl == self.tls[generating_thread_num] assert marker == (i1, o1) - if extra is None: - assert len(seen) == 1 - else: - assert seen[1] == (self.tls[1], extra, None) - assert len(seen) == 2 + assert len(seen) == 1 + del self.seen[:] def test_marker_odd_simple(self): self.start_transaction() @@ -119,25 +116,23 @@ py.test.raises(Conflict, self.switch, 1) self.check_recording(19, ffi.NULL) - def test_commit_marker_for_inev(self): - self.recording(lib.STM_TRANSACTION_COMMIT) + def test_become_inevitable_marker(self): + self.recording(lib.STM_BECOME_INEVITABLE) # self.switch(1) self.start_transaction() self.push_root(ffi.cast("object_t *", 19)) self.push_root(ffi.cast("object_t *", ffi.NULL)) self.become_inevitable() - self.pop_root() - self.pop_root() - self.commit_transaction() # self.check_recording(19, ffi.NULL) def test_abort_markers_cb_inevitable(self): - self.recording(lib.STM_WAIT_OTHER_INEVITABLE) + self.recording(lib.STM_BECOME_INEVITABLE) # self.start_transaction() self.become_inevitable() + self.check_recording(0, ffi.NULL, generating_thread_num=0) # self.switch(1) self.start_transaction() @@ -147,9 +142,10 @@ self.push_root(ffi.cast("object_t *", p)) py.test.raises(Conflict, self.become_inevitable) # - py.test.skip("XXX only during tests does become_inevitable() abort" - " and then it doesn't record anything") - self.check_recording(21, p) + # only during tests does become_inevitable() abort because + # another thread is already inevitable; but it should have + # recorded the marker first + self.check_recording(21, p, generating_thread_num=1) def test_read_write_contention(self): self.recording(lib.STM_CONTENTION_WRITE_READ) From noreply at buildbot.pypy.org Tue Mar 10 16:02:15 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 10 Mar 2015 16:02:15 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c8: import stmgc (branch c8-marker) Message-ID: <20150310150215.6E1F01C1150@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c8 Changeset: r76294:c92a99a4f5f0 Date: 2015-03-10 15:57 +0100 http://bitbucket.org/pypy/pypy/changeset/c92a99a4f5f0/ Log: import stmgc (branch c8-marker) diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision --- a/rpython/translator/stm/src_stm/revision +++ b/rpython/translator/stm/src_stm/revision @@ -1,1 +1,1 @@ -321802556392 +a4e4d3ad014a diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c --- a/rpython/translator/stm/src_stm/stm/core.c +++ b/rpython/translator/stm/src_stm/stm/core.c @@ -464,7 +464,7 @@ static void wait_for_other_inevitable(struct stm_commit_log_entry_s *old) { - timing_wait_other_inevitable(); + timing_event(STM_SEGMENT->running_thread, STM_WAIT_OTHER_INEVITABLE); while (old->next == INEV_RUNNING && !safe_point_requested()) { spin_loop(); @@ -1171,7 +1171,7 @@ /************************************************************/ -static void _finish_transaction(enum stm_event_e event, bool was_inev) +static void _finish_transaction(enum stm_event_e event) { stm_thread_local_t *tl = STM_SEGMENT->running_thread; @@ -1182,10 +1182,7 @@ list_clear(STM_PSEGMENT->objects_pointing_to_nursery); list_clear(STM_PSEGMENT->old_objects_with_cards_set); list_clear(STM_PSEGMENT->large_overflow_objects); - if (was_inev) - timing_commit_inev_position(); - else - timing_event(tl, event); + timing_event(tl, event); release_thread_segment(tl); /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ @@ -1285,7 +1282,7 @@ /* done */ stm_thread_local_t *tl = STM_SEGMENT->running_thread; - _finish_transaction(STM_TRANSACTION_COMMIT, was_inev); + _finish_transaction(STM_TRANSACTION_COMMIT); /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ s_mutex_unlock(); @@ -1420,7 +1417,7 @@ : NURSERY_END; } - _finish_transaction(STM_TRANSACTION_ABORT, false); + _finish_transaction(STM_TRANSACTION_ABORT); /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ return tl; @@ -1459,10 +1456,10 @@ if (STM_PSEGMENT->transaction_state == TS_REGULAR) { dprintf(("become_inevitable: %s\n", msg)); _stm_collectable_safe_point(); + timing_become_inevitable(); _validate_and_turn_inevitable(); STM_PSEGMENT->transaction_state = TS_INEVITABLE; - timing_record_inev_position(); stm_rewind_jmp_forget(STM_SEGMENT->running_thread); invoke_and_clear_user_callbacks(0); /* for commit */ diff --git a/rpython/translator/stm/src_stm/stm/core.h b/rpython/translator/stm/src_stm/stm/core.h --- a/rpython/translator/stm/src_stm/stm/core.h +++ b/rpython/translator/stm/src_stm/stm/core.h @@ -141,9 +141,6 @@ pthread_t running_pthread; #endif - /* marker where this thread became inevitable */ - stm_loc_marker_t marker_inev; - /* light finalizers */ struct list_s *young_objects_with_light_finalizers; struct list_s *old_objects_with_light_finalizers; diff --git a/rpython/translator/stm/src_stm/stm/gcpage.c b/rpython/translator/stm/src_stm/stm/gcpage.c --- a/rpython/translator/stm/src_stm/stm/gcpage.c +++ b/rpython/translator/stm/src_stm/stm/gcpage.c @@ -408,9 +408,6 @@ if (modified->type == TYPE_POSITION_MARKER) mark_visit_possibly_new_object(modified->marker_object, pseg); } - - if (pseg->transaction_state == TS_INEVITABLE) - mark_visit_possibly_new_object(pseg->marker_inev.object, pseg); } } diff --git a/rpython/translator/stm/src_stm/stm/marker.c b/rpython/translator/stm/src_stm/stm/marker.c --- a/rpython/translator/stm/src_stm/stm/marker.c +++ b/rpython/translator/stm/src_stm/stm/marker.c @@ -89,32 +89,12 @@ STM_CONTENTION_WRITE_READ, &marker); } -static void _timing_record_inev_position(void) +static void _timing_become_inevitable(void) { stm_loc_marker_t marker; marker_fetch(STM_SEGMENT->running_thread, &marker); - STM_PSEGMENT->marker_inev.odd_number = marker.odd_number; - STM_PSEGMENT->marker_inev.object = marker.object; -} - -static void _timing_commit_inev_position(void) -{ - stm_loc_marker_t marker; - marker.odd_number = STM_PSEGMENT->marker_inev.odd_number; - marker.object = STM_PSEGMENT->marker_inev.object; stmcb_timing_event(STM_SEGMENT->running_thread, - STM_TRANSACTION_COMMIT, &marker); -} - -static void timing_wait_other_inevitable(void) -{ - if (stmcb_timing_event == NULL) - return; - - stm_loc_marker_t marker; - marker_fetch(STM_SEGMENT->running_thread, &marker); - stmcb_timing_event(STM_SEGMENT->running_thread, - STM_WAIT_OTHER_INEVITABLE, &marker); + STM_BECOME_INEVITABLE, &marker); } diff --git a/rpython/translator/stm/src_stm/stm/marker.h b/rpython/translator/stm/src_stm/stm/marker.h --- a/rpython/translator/stm/src_stm/stm/marker.h +++ b/rpython/translator/stm/src_stm/stm/marker.h @@ -2,9 +2,7 @@ static void _timing_record_write_position(void); static void timing_write_read_contention(struct stm_undo_s *start, struct stm_undo_s *contention); -static void _timing_record_inev_position(void); -static void _timing_commit_inev_position(void); -static void timing_wait_other_inevitable(void); +static void _timing_become_inevitable(void); #define timing_enabled() (stmcb_timing_event != NULL) @@ -15,8 +13,5 @@ #define timing_record_write_position() \ (timing_enabled() ? _timing_record_write_position() : (void)0) -#define timing_record_inev_position() \ - (timing_enabled() ? _timing_record_inev_position() : (void)0) - -#define timing_commit_inev_position() \ - (timing_enabled() ? _timing_commit_inev_position() : (void)0) +#define timing_become_inevitable() \ + (timing_enabled() ? _timing_become_inevitable() : (void)0) diff --git a/rpython/translator/stm/src_stm/stm/nursery.c b/rpython/translator/stm/src_stm/stm/nursery.c --- a/rpython/translator/stm/src_stm/stm/nursery.c +++ b/rpython/translator/stm/src_stm/stm/nursery.c @@ -425,13 +425,6 @@ if (undo->type == TYPE_POSITION_MARKER) minor_trace_if_young(&undo->marker_object); } - - if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) { - object_t **pmarker_inev_obj = (object_t **) - REAL_ADDRESS(STM_SEGMENT->segment_base, - &STM_PSEGMENT->marker_inev.object); - minor_trace_if_young(pmarker_inev_obj); - } } static void collect_objs_still_young_but_with_finalizers(void) diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h --- a/rpython/translator/stm/src_stm/stmgc.h +++ b/rpython/translator/stm/src_stm/stmgc.h @@ -354,11 +354,12 @@ saying where the write was done. Followed by STM_TRANSACTION_ABORT. */ STM_CONTENTION_WRITE_READ, - /* inevitable contention: the thread that waited is - STM_WAIT_OTHER_INEVITABLE (with a marker) and the thread that - it waited for is the next STM_TRANSACTION_COMMIT (with a marker - as well; both markers point to the place that made each - transaction inevitable). */ + /* inevitable contention: all threads that try to become inevitable + have a STM_BECOME_INEVITABLE event with a position marker. Then, + if it waits it gets a STM_WAIT_OTHER_INEVITABLE. It is possible + that a thread gets STM_BECOME_INEVITABLE followed by + STM_TRANSACTION_ABORT if it fails to become inevitable. */ + STM_BECOME_INEVITABLE, /* always one STM_WAIT_xxx followed later by STM_WAIT_DONE */ STM_WAIT_FREE_SEGMENT, From noreply at buildbot.pypy.org Tue Mar 10 17:15:40 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 10 Mar 2015 17:15:40 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c8: Update the logic in print_stm_log. Still missing: inevitables Message-ID: <20150310161540.5294C1C015A@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c8 Changeset: r76295:7031b770c8e1 Date: 2015-03-10 17:15 +0100 http://bitbucket.org/pypy/pypy/changeset/7031b770c8e1/ Log: Update the logic in print_stm_log. Still missing: inevitables diff --git a/pypy/stm/print_stm_log.py b/pypy/stm/print_stm_log.py --- a/pypy/stm/print_stm_log.py +++ b/pypy/stm/print_stm_log.py @@ -12,19 +12,26 @@ # saying where the write was done. Followed by STM_TRANSACTION_ABORT. STM_CONTENTION_WRITE_READ = 3 +# inevitable contention: all threads that try to become inevitable +# have a STM_BECOME_INEVITABLE event with a position marker. Then, +# if it waits it gets a STM_WAIT_OTHER_INEVITABLE. It is possible +# that a thread gets STM_BECOME_INEVITABLE followed by +# STM_TRANSACTION_ABORT if it fails to become inevitable. +STM_BECOME_INEVITABLE = 4 + # always one STM_WAIT_xxx followed later by STM_WAIT_DONE -STM_WAIT_FREE_SEGMENT = 4 -STM_WAIT_SYNC_PAUSE = 5 -STM_WAIT_OTHER_INEVITABLE = 6 -STM_WAIT_DONE = 7 +STM_WAIT_FREE_SEGMENT = 5 +STM_WAIT_SYNC_PAUSE = 6 +STM_WAIT_OTHER_INEVITABLE = 7 +STM_WAIT_DONE = 8 # start and end of GC cycles -STM_GC_MINOR_START = 8 -STM_GC_MINOR_DONE = 9 -STM_GC_MAJOR_START = 10 -STM_GC_MAJOR_DONE = 11 +STM_GC_MINOR_START = 9 +STM_GC_MINOR_DONE = 10 +STM_GC_MAJOR_START = 11 +STM_GC_MAJOR_DONE = 12 -_STM_EVENT_N = 12 +_STM_EVENT_N = 13 PAUSE_AFTER_ABORT = 0.000001 # usleep(1) after every abort @@ -81,43 +88,65 @@ class ThreadState(object): def __init__(self, threadnum): self.threadnum = threadnum - self.cpu_time = 0.0 + self.cpu_time_committed = 0.0 + self.cpu_time_aborted = 0.0 + self._prev = (0.0, "stop") + self.reset_counters() + + def reset_counters(self): + self._transaction_cpu_time = 0.0 + self._transaction_pause_time = 0.0 + self._transaction_aborting = False def transaction_start(self, entry): - self._start = entry - self._conflict = None - self._pause = None - self._paused_time = 0.0 + self.reset_counters() + self.progress(entry, "run") - def transaction_stop(self, entry): - transaction_time = entry.timestamp - self._start.timestamp - transaction_time -= self._paused_time - assert transaction_time >= 0.0 - self.cpu_time += transaction_time - self._start = None - self._pause = None - if self._conflict and entry.event == STM_TRANSACTION_ABORT: - c = self._conflict[1] - c.aborted_time += transaction_time - c.paused_time += PAUSE_AFTER_ABORT - self._conflict = None + def progress(self, entry, new_state): + prev_time, prev_state = self._prev + add_time = entry.timestamp - prev_time + assert add_time >= 0.0 + if prev_state == "run": + self._transaction_cpu_time += add_time + elif prev_state == "pause": + self._transaction_pause_time += add_time + self._prev = entry.timestamp, new_state + + def transaction_commit(self, entry): + assert not self._transaction_aborting + self.progress(entry, "stop") + self.cpu_time_committed += self._transaction_cpu_time + + def transaction_abort(self, entry): + self.progress(entry, "stop") + self.cpu_time_aborted += self._transaction_cpu_time + + def become_inevitable(self, entry): + "XXX" def transaction_pause(self, entry): - self._pause = entry + self.progress(entry, "pause") def transaction_unpause(self, entry): - if self._pause is None: + self.progress(entry, "run") + + def contention_write_read(self, entry, out_conflicts): + # This thread is aborted because it has read an object, but + # another thread already committed a change to that object. + # The marker is pointing inside the other thread's write. + if self._transaction_aborting: + print >> sys.stderr, "note: double STM_CONTENTION_WRITE_READ" return - pause_time = entry.timestamp - self._pause.timestamp - self._paused_time += pause_time - self._pause = None - if self._conflict and self._conflict[0] == "local": - c = self._conflict[1] - c.paused_time += pause_time - self._conflict = None - - def in_transaction(self): - return self._start is not None + self._transaction_aborting = True + summary = (entry.event, entry.marker) + c = out_conflicts.get(summary) + if c is None: + c = out_conflicts[summary] = ConflictSummary(*summary) + c.num_events += 1 + c.timestamps.append(entry.timestamp) + self.progress(entry, "run") + c.aborted_time += self._transaction_cpu_time + c.paused_time += self._transaction_pause_time class ConflictSummary(object): @@ -153,10 +182,12 @@ s = ' %s' % marker match = r_marker.match(marker) if match: - line = linecache.getline(match.group(1), int(match.group(2))) - line = line.strip() - if line: - s += '\n %s' % line + filename = match.group(1) + if not (filename.endswith('.pyc') or filename.endswith('.pyo')): + line = linecache.getline(filename, int(match.group(2))) + line = line.strip() + if line: + s += '\n %s' % line return s def percent(fraction, total): @@ -177,47 +208,26 @@ print >> sys.stderr, '%.0f%%' % (entry.frac * 100.0,), cnt += 1 # + t = threads.get(entry.threadnum) + if t is None: + t = threads[entry.threadnum] = ThreadState(entry.threadnum) + # if entry.event == STM_TRANSACTION_START: - t = threads.get(entry.threadnum) - if t is None: - t = threads[entry.threadnum] = ThreadState(entry.threadnum) t.transaction_start(entry) - elif (entry.event == STM_TRANSACTION_COMMIT or - entry.event == STM_TRANSACTION_ABORT): - t = threads.get(entry.threadnum) - if t is not None and t.in_transaction(): - t.transaction_stop(entry) - elif entry.event in (#STM_CONTENTION_WRITE_WRITE, - STM_CONTENTION_WRITE_READ, - #STM_CONTENTION_INEVITABLE, - ): - summary = (entry.event, entry.marker) - c = conflicts.get(summary) - if c is None: - c = conflicts[summary] = ConflictSummary(*summary) - c.num_events += 1 - c.timestamps.append(entry.timestamp) - t = threads.get(entry.threadnum) - if t is not None and t.in_transaction(): - t._conflict = ("local", c, entry) - ## elif entry.event == STM_ABORTING_OTHER_CONTENTION: - ## t = threads.get(entry.threadnum) - ## if t is not None and t._conflict and t._conflict[0] == "local": - ## _, c, entry = t._conflict - ## t._conflict = None - ## t2 = threads.get(entry.otherthreadnum) - ## if t2 is not None and t2.in_transaction(): - ## t2._conflict = ("remote", c, entry) + elif entry.event == STM_TRANSACTION_COMMIT: + t.transaction_commit(entry) + elif entry.event == STM_TRANSACTION_ABORT: + t.transaction_abort(entry) + elif entry.event == STM_BECOME_INEVITABLE: + t.become_inevitable(entry) + elif entry.event == STM_CONTENTION_WRITE_READ: + t.contention_write_read(entry, conflicts) elif entry.event in (STM_WAIT_FREE_SEGMENT, STM_WAIT_SYNC_PAUSE, STM_WAIT_OTHER_INEVITABLE): - t = threads.get(entry.threadnum) - if t is not None and t.in_transaction(): - t.transaction_pause(entry) + t.transaction_pause(entry) elif entry.event == STM_WAIT_DONE: - t = threads.get(entry.threadnum) - if t is not None and t.in_transaction(): - t.transaction_unpause(entry) + t.transaction_unpause(entry) # if cnt == 0: raise Exception("empty file") @@ -234,9 +244,15 @@ print print 'Total real time: %.3fs' % (total_time,) # - total_cpu_time = stmlog.get_total_cpu_time() - print 'Total CPU time in STM mode: %.3fs (%s)' % ( - total_cpu_time, percent(total_cpu_time, total_time)) + total_cpu_time_committed = stmlog.get_total_cpu_time_committed() + total_cpu_time_aborted = stmlog.get_total_cpu_time_aborted() + total_cpu_time_total = total_cpu_time_committed + total_cpu_time_aborted + print 'Total CPU time in STM mode: %.3fs (%s) committed' % ( + total_cpu_time_committed, percent(total_cpu_time_committed, total_time)) + print ' %.3fs (%s) aborted' % ( + total_cpu_time_aborted, percent(total_cpu_time_aborted, total_time)) + print ' %.3fs (%s) total' % ( + total_cpu_time_total, percent(total_cpu_time_total, total_time)) print # values = stmlog.get_conflicts() @@ -256,8 +272,11 @@ def __init__(self, filename): summarize_log_entries(parse_log(filename), self) - def get_total_cpu_time(self): - return sum([v.cpu_time for v in self.threads.values()]) + def get_total_cpu_time_committed(self): + return sum([v.cpu_time_committed for v in self.threads.values()]) + + def get_total_cpu_time_aborted(self): + return sum([v.cpu_time_aborted for v in self.threads.values()]) def get_conflicts(self): values = self.conflicts.values() From noreply at buildbot.pypy.org Tue Mar 10 17:39:19 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 10 Mar 2015 17:39:19 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c8: Report inevitable pauses Message-ID: <20150310163919.18B581C0196@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c8 Changeset: r76296:84499372529f Date: 2015-03-10 17:38 +0100 http://bitbucket.org/pypy/pypy/changeset/84499372529f/ Log: Report inevitable pauses diff --git a/pypy/stm/print_stm_log.py b/pypy/stm/print_stm_log.py --- a/pypy/stm/print_stm_log.py +++ b/pypy/stm/print_stm_log.py @@ -90,6 +90,7 @@ self.threadnum = threadnum self.cpu_time_committed = 0.0 self.cpu_time_aborted = 0.0 + self.cpu_time_paused = 0.0 self._prev = (0.0, "stop") self.reset_counters() @@ -97,38 +98,59 @@ self._transaction_cpu_time = 0.0 self._transaction_pause_time = 0.0 self._transaction_aborting = False + self._transaction_inev = None def transaction_start(self, entry): self.reset_counters() - self.progress(entry, "run") + self.progress(entry.timestamp, "run") - def progress(self, entry, new_state): + def progress(self, now, new_state): prev_time, prev_state = self._prev - add_time = entry.timestamp - prev_time + add_time = now - prev_time assert add_time >= 0.0 if prev_state == "run": self._transaction_cpu_time += add_time elif prev_state == "pause": self._transaction_pause_time += add_time - self._prev = entry.timestamp, new_state + self._prev = now, new_state def transaction_commit(self, entry): assert not self._transaction_aborting - self.progress(entry, "stop") + self.progress(entry.timestamp, "stop") self.cpu_time_committed += self._transaction_cpu_time + self.cpu_time_paused += self._transaction_pause_time def transaction_abort(self, entry): - self.progress(entry, "stop") + self.progress(entry.timestamp, "stop") self.cpu_time_aborted += self._transaction_cpu_time + self.cpu_time_paused += self._transaction_pause_time def become_inevitable(self, entry): - "XXX" + self.progress(entry.timestamp, "run") + if self._transaction_inev is None: + self._transaction_inev = [entry, None] def transaction_pause(self, entry): - self.progress(entry, "pause") + self.progress(entry.timestamp, "pause") + if (entry.event == STM_WAIT_OTHER_INEVITABLE and + self._transaction_inev is not None): + self._transaction_inev[1] = entry.timestamp - def transaction_unpause(self, entry): - self.progress(entry, "run") + def transaction_unpause(self, entry, out_conflicts): + self.progress(entry.timestamp, "run") + if self._transaction_inev and self._transaction_inev[1] is not None: + wait_time = entry.timestamp - self._transaction_inev[1] + self.wait_for_other_inev(wait_time, out_conflicts) + self._transaction_inev[1] = None + + def get_conflict(self, entry, out_conflicts): + summary = (entry.event, entry.marker) + c = out_conflicts.get(summary) + if c is None: + c = out_conflicts[summary] = ConflictSummary(*summary) + c.num_events += 1 + c.timestamps.append(entry.timestamp) + return c def contention_write_read(self, entry, out_conflicts): # This thread is aborted because it has read an object, but @@ -138,16 +160,16 @@ print >> sys.stderr, "note: double STM_CONTENTION_WRITE_READ" return self._transaction_aborting = True - summary = (entry.event, entry.marker) - c = out_conflicts.get(summary) - if c is None: - c = out_conflicts[summary] = ConflictSummary(*summary) - c.num_events += 1 - c.timestamps.append(entry.timestamp) - self.progress(entry, "run") + c = self.get_conflict(entry, out_conflicts) + self.progress(entry.timestamp, "run") c.aborted_time += self._transaction_cpu_time c.paused_time += self._transaction_pause_time + def wait_for_other_inev(self, wait_time, out_conflicts): + c = self.get_conflict(self._transaction_inev[0], out_conflicts) + assert wait_time >= 0 + c.paused_time += wait_time + class ConflictSummary(object): def __init__(self, event, marker): @@ -227,7 +249,7 @@ STM_WAIT_OTHER_INEVITABLE): t.transaction_pause(entry) elif entry.event == STM_WAIT_DONE: - t.transaction_unpause(entry) + t.transaction_unpause(entry, conflicts) # if cnt == 0: raise Exception("empty file") @@ -242,16 +264,21 @@ start_time = stmlog.start_time total_time = stmlog.total_time print - print 'Total real time: %.3fs' % (total_time,) + print 'Total real time: %.3fs' % (total_time,) # total_cpu_time_committed = stmlog.get_total_cpu_time_committed() total_cpu_time_aborted = stmlog.get_total_cpu_time_aborted() - total_cpu_time_total = total_cpu_time_committed + total_cpu_time_aborted - print 'Total CPU time in STM mode: %.3fs (%s) committed' % ( + total_cpu_time_paused = stmlog.get_total_cpu_time_paused() + total_cpu_time_total = (total_cpu_time_committed + + total_cpu_time_aborted + + total_cpu_time_paused) + print 'CPU time in STM mode: %.3fs (%s) committed' % ( total_cpu_time_committed, percent(total_cpu_time_committed, total_time)) - print ' %.3fs (%s) aborted' % ( + print ' %.3fs (%s) aborted' % ( total_cpu_time_aborted, percent(total_cpu_time_aborted, total_time)) - print ' %.3fs (%s) total' % ( + print ' %.3fs (%s) paused' % ( + total_cpu_time_paused, percent(total_cpu_time_paused, total_time)) + print ' %.3fs (%s) total' % ( total_cpu_time_total, percent(total_cpu_time_total, total_time)) print # @@ -278,6 +305,9 @@ def get_total_cpu_time_aborted(self): return sum([v.cpu_time_aborted for v in self.threads.values()]) + def get_total_cpu_time_paused(self): + return sum([v.cpu_time_paused for v in self.threads.values()]) + def get_conflicts(self): values = self.conflicts.values() values.sort(key=ConflictSummary.sortkey) From noreply at buildbot.pypy.org Tue Mar 10 17:39:36 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 10 Mar 2015 17:39:36 +0100 (CET) Subject: [pypy-commit] stmgc default: hg merge c8-marker Message-ID: <20150310163936.854B01C0196@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1711:cf7b3fc03a25 Date: 2015-03-10 17:40 +0100 http://bitbucket.org/pypy/stmgc/changeset/cf7b3fc03a25/ Log: hg merge c8-marker Reintroduce "markers", produced by pypy-stm if the PYPYSTM environment variable is set to a file name. diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -5,6 +5,7 @@ /* *** MISC *** */ static void free_bk(struct stm_undo_s *undo) { + assert(undo->type != TYPE_POSITION_MARKER); free(undo->backup); assert(undo->backup = (char*)-88); increment_total_allocated(-SLICE_SIZE(undo->slice)); @@ -49,6 +50,8 @@ DEBUG_EXPECT_SEGFAULT(false); for (; undo < end; undo++) { + if (undo->type == TYPE_POSITION_MARKER) + continue; object_t *obj = undo->object; stm_char *oslice = ((stm_char *)obj) + SLICE_OFFSET(undo->slice); uintptr_t current_page_num = ((uintptr_t)oslice) / 4096; @@ -269,6 +272,11 @@ struct stm_undo_s *undo = cl->written; struct stm_undo_s *end = undo + cl->written_count; for (; undo < end; undo++) { + if (undo->type == TYPE_POSITION_MARKER) { + fprintf(stderr, " marker %p %lu\n", + undo->marker_object, undo->marker_odd_number); + continue; + } fprintf(stderr, " obj %p, size %d, ofs %lu: ", undo->object, SLICE_SIZE(undo->slice), SLICE_OFFSET(undo->slice)); /* long i; */ @@ -312,6 +320,12 @@ bool needs_abort = false; + if (STM_PSEGMENT->transaction_state == TS_NONE) { + /* can be seen from major_do_validation_and_minor_collections(); + don't try to detect pseudo-conflicts in this case */ + needs_abort = true; + } + while(1) { /* retry IF: */ /* if at the time of "HERE" (s.b.) there happen to be @@ -362,6 +376,8 @@ struct stm_undo_s *undo = cl->written; struct stm_undo_s *end = cl->written + cl->written_count; for (; undo < end; undo++) { + if (undo->type == TYPE_POSITION_MARKER) + continue; if (_stm_was_read(undo->object)) { /* first reset all modified objects from the backup copies as soon as the first conflict is detected; @@ -369,6 +385,7 @@ the old (but unmodified) version to the newer version. */ reset_modified_from_backup_copies(my_segnum); + timing_write_read_contention(cl->written, undo); needs_abort = true; dprintf(("_stm_validate() failed for obj %p\n", undo->object)); @@ -445,6 +462,17 @@ } +static void wait_for_other_inevitable(struct stm_commit_log_entry_s *old) +{ + timing_event(STM_SEGMENT->running_thread, STM_WAIT_OTHER_INEVITABLE); + + while (old->next == INEV_RUNNING && !safe_point_requested()) { + spin_loop(); + usleep(10); /* XXXXXX */ + } + timing_event(STM_SEGMENT->running_thread, STM_WAIT_DONE); +} + static void reset_wb_executed_flags(void); static void readd_wb_executed_flags(void); static void check_all_write_barrier_flags(char *segbase, struct list_s *list); @@ -496,13 +524,6 @@ if (__sync_bool_compare_and_swap(&old->next, NULL, new)) break; /* success! */ - } else if (old->next == INEV_RUNNING) { - /* we failed because there is an INEV transaction running */ - /* XXXXXX for now just sleep (XXX with the lock acquired? - isn't it a bad idea?). We should really ask to inev - transaction to do the commit for us, and then we can - continue running. */ - usleep(10); } if (is_commit) { @@ -513,6 +534,16 @@ readd_wb_executed_flags(); } + if (old->next == INEV_RUNNING && !safe_point_requested()) { + /* we failed because there is an INEV transaction running */ + /* XXXXXX for now just sleep. We should really ask to inev + transaction to do the commit for us, and then we can + continue running. */ + dprintf(("_validate_and_attach(%p) failed, " + "waiting for inevitable\n", new)); + wait_for_other_inevitable(old); + } + dprintf(("_validate_and_attach(%p) failed, enter safepoint\n", new)); /* check for requested safe point. otherwise an INEV transaction @@ -603,6 +634,8 @@ { dprintf(("make_bk_slices_for_range(%p, %lu, %lu)\n", obj, start - (stm_char*)obj, end - start)); + timing_record_write_position(); + char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); uintptr_t first_page = ((uintptr_t)start) / 4096UL; uintptr_t end_page = ((uintptr_t)end) / 4096UL; @@ -1025,6 +1058,8 @@ struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count); for (; undo < end; undo++) { + if (undo->type == TYPE_POSITION_MARKER) + continue; object_t *obj = undo->object; obj->stm_flags &= ~GCFLAG_WB_EXECUTED; } @@ -1038,6 +1073,8 @@ struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count); for (; undo < end; undo++) { + if (undo->type == TYPE_POSITION_MARKER) + continue; object_t *obj = undo->object; obj->stm_flags |= GCFLAG_WB_EXECUTED; } @@ -1055,6 +1092,7 @@ assert(STM_PSEGMENT->safe_point == SP_NO_TRANSACTION); assert(STM_PSEGMENT->transaction_state == TS_NONE); + timing_event(tl, STM_TRANSACTION_START); STM_PSEGMENT->transaction_state = TS_REGULAR; STM_PSEGMENT->safe_point = SP_RUNNING; #ifndef NDEBUG @@ -1084,6 +1122,10 @@ assert(tree_is_cleared(STM_PSEGMENT->callbacks_on_commit_and_abort[1])); assert(list_is_empty(STM_PSEGMENT->young_objects_with_light_finalizers)); assert(STM_PSEGMENT->finalizers == NULL); +#ifndef NDEBUG + /* this should not be used when objects_pointing_to_nursery == NULL */ + STM_PSEGMENT->position_markers_len_old = 99999999999999999L; +#endif check_nursery_at_transaction_start(); @@ -1129,7 +1171,7 @@ /************************************************************/ -static void _finish_transaction() +static void _finish_transaction(enum stm_event_e event) { stm_thread_local_t *tl = STM_SEGMENT->running_thread; @@ -1140,6 +1182,7 @@ list_clear(STM_PSEGMENT->objects_pointing_to_nursery); list_clear(STM_PSEGMENT->old_objects_with_cards_set); list_clear(STM_PSEGMENT->large_overflow_objects); + timing_event(tl, event); release_thread_segment(tl); /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ @@ -1151,6 +1194,8 @@ struct stm_undo_s *undo = (struct stm_undo_s *)list->items; struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count); for (; undo < end; undo++) { + if (undo->type == TYPE_POSITION_MARKER) + continue; object_t *obj = undo->object; struct object_s *dst = (struct object_s*)REAL_ADDRESS(segbase, obj); assert(dst->stm_flags & GCFLAG_WRITE_BARRIER); @@ -1199,6 +1244,7 @@ push_large_overflow_objects_to_other_segments(); /* push before validate. otherwise they are reachable too early */ + bool was_inev = STM_PSEGMENT->transaction_state == TS_INEVITABLE; _validate_and_add_to_commit_log(); @@ -1225,11 +1271,7 @@ /* if a major collection is required, do it here */ if (is_major_collection_requested()) { - synchronize_all_threads(STOP_OTHERS_UNTIL_MUTEX_UNLOCK); - - if (is_major_collection_requested()) { /* if *still* true */ - major_collection_now_at_safe_point(); - } + major_collection_with_mutex(); } _verify_cards_cleared_in_all_lists(get_priv_segment(STM_SEGMENT->segment_num)); @@ -1240,7 +1282,7 @@ /* done */ stm_thread_local_t *tl = STM_SEGMENT->running_thread; - _finish_transaction(); + _finish_transaction(STM_TRANSACTION_COMMIT); /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ s_mutex_unlock(); @@ -1264,6 +1306,8 @@ struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count); for (; undo < end; undo++) { + if (undo->type == TYPE_POSITION_MARKER) + continue; object_t *obj = undo->object; char *dst = REAL_ADDRESS(pseg->pub.segment_base, obj); @@ -1373,7 +1417,7 @@ : NURSERY_END; } - _finish_transaction(); + _finish_transaction(STM_TRANSACTION_ABORT); /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ return tl; @@ -1412,9 +1456,11 @@ if (STM_PSEGMENT->transaction_state == TS_REGULAR) { dprintf(("become_inevitable: %s\n", msg)); _stm_collectable_safe_point(); + timing_become_inevitable(); _validate_and_turn_inevitable(); STM_PSEGMENT->transaction_state = TS_INEVITABLE; + stm_rewind_jmp_forget(STM_SEGMENT->running_thread); invoke_and_clear_user_callbacks(0); /* for commit */ } diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -84,9 +84,12 @@ track the STM status: these are old objects that where written to and that will need to be recorded in the commit log. The list contains three entries for every such object, in the same - format as 'struct stm_undo_s' below. + format as 'struct stm_undo_s' below. It can also represent a + position marker, like 'struct stm_undo_s'. */ struct list_s *modified_old_objects; + uintptr_t position_markers_last; /* index of most recent pos marker */ + uintptr_t position_markers_len_old; /* length of list at last minor col */ struct list_s *objects_pointing_to_nursery; struct list_s *old_objects_with_cards_set; @@ -174,18 +177,27 @@ /* Commit Log things */ struct stm_undo_s { - object_t *object; /* the object that is modified */ - char *backup; /* some backup data (a slice of the original obj) */ - uint64_t slice; /* location and size of this slice (cannot cross - pages). The size is in the lower 2 bytes, and - the offset in the remaining 6 bytes. */ + union { + struct { + object_t *object; /* the object that is modified */ + char *backup; /* some backup data (a slice of the original obj) */ + uint64_t slice; /* location and size of this slice (cannot cross + pages). The size is in the lower 2 bytes, and + the offset in the remaining 6 bytes. */ + }; + struct { + intptr_t type; /* TYPE_POSITION_MARKER */ + uintptr_t marker_odd_number; /* the odd number part of the marker */ + object_t *marker_object; /* the object part of the marker */ + }; + }; }; +#define TYPE_POSITION_MARKER (-1) #define SLICE_OFFSET(slice) ((slice) >> 16) #define SLICE_SIZE(slice) ((int)((slice) & 0xFFFF)) #define NEW_SLICE(offset, size) (((uint64_t)(offset)) << 16 | (size)) - /* The model is: we have a global chained list, from 'commit_log_root', of 'struct stm_commit_log_entry_s' entries. Every one is fully read-only apart from the 'next' field. Every one stands for one diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -131,6 +131,19 @@ /************************************************************/ +static void major_collection_with_mutex(void) +{ + timing_event(STM_SEGMENT->running_thread, STM_GC_MAJOR_START); + + synchronize_all_threads(STOP_OTHERS_UNTIL_MUTEX_UNLOCK); + + if (is_major_collection_requested()) { /* if *still* true */ + major_collection_now_at_safe_point(); + } + + timing_event(STM_SEGMENT->running_thread, STM_GC_MAJOR_DONE); +} + static void major_collection_if_requested(void) { assert(!_has_mutex()); @@ -140,13 +153,7 @@ s_mutex_lock(); if (is_major_collection_requested()) { /* if still true */ - - synchronize_all_threads(STOP_OTHERS_UNTIL_MUTEX_UNLOCK); - - if (is_major_collection_requested()) { /* if *still* true */ - major_collection_now_at_safe_point(); - } - + major_collection_with_mutex(); } s_mutex_unlock(); @@ -348,6 +355,8 @@ struct stm_undo_s *modified = (struct stm_undo_s *)lst->items; struct stm_undo_s *end = (struct stm_undo_s *)(lst->items + lst->count); for (; modified < end; modified++) { + if (modified->type == TYPE_POSITION_MARKER) + continue; object_t *obj = modified->object; struct object_s *dst = (struct object_s*)REAL_ADDRESS(base, obj); @@ -386,6 +395,22 @@ LIST_FREE(uniques); } +static void mark_visit_from_markers(void) +{ + long i; + for (i = 1; i < NB_SEGMENTS; i++) { + struct stm_priv_segment_info_s *pseg = get_priv_segment(i); + struct list_s *lst = get_priv_segment(i)->modified_old_objects; + + struct stm_undo_s *modified = (struct stm_undo_s *)lst->items; + struct stm_undo_s *end = (struct stm_undo_s *)(lst->items + lst->count); + for (; modified < end; modified++) { + if (modified->type == TYPE_POSITION_MARKER) + mark_visit_possibly_new_object(modified->marker_object, pseg); + } + } +} + static void mark_visit_from_roots(void) { if (testing_prebuilt_objs != NULL) { @@ -605,7 +630,8 @@ /* free bk copies of entries: */ long count = cl->written_count; while (count-->0) { - free_bk(&cl->written[count]); + if (cl->written[count].type != TYPE_POSITION_MARKER) + free_bk(&cl->written[count]); } next = cl->next; @@ -683,6 +709,7 @@ /* marking */ LIST_CREATE(marked_objects_to_trace); mark_visit_from_modified_objects(); + mark_visit_from_markers(); mark_visit_from_roots(); mark_visit_from_finalizer_pending(); diff --git a/c8/stm/gcpage.h b/c8/stm/gcpage.h --- a/c8/stm/gcpage.h +++ b/c8/stm/gcpage.h @@ -19,5 +19,6 @@ static void major_collection_if_requested(void); static void major_collection_now_at_safe_point(void); +static void major_collection_with_mutex(void); static bool largemalloc_keep_object_at(char *data); /* for largemalloc.c */ static bool smallmalloc_keep_object_at(char *data); /* for smallmalloc.c */ diff --git a/c7/stm/marker.c b/c8/stm/marker.c copy from c7/stm/marker.c copy to c8/stm/marker.c --- a/c7/stm/marker.c +++ b/c8/stm/marker.c @@ -3,11 +3,10 @@ #endif -static void marker_fetch(stm_loc_marker_t *out_marker) +static bool marker_fetch(stm_thread_local_t *tl, stm_loc_marker_t *out_marker) { - /* Fetch the current marker from the 'out_marker->tl's shadow stack, + /* Fetch the current marker from tl's shadow stack, and return it in 'out_marker->odd_number' and 'out_marker->object'. */ - stm_thread_local_t *tl = out_marker->tl; struct stm_shadowentry_s *current = tl->shadowstack - 1; struct stm_shadowentry_s *base = tl->shadowstack_base; @@ -23,121 +22,82 @@ /* found the odd marker */ out_marker->odd_number = (uintptr_t)current[0].ss; out_marker->object = current[1].ss; + return true; } else { /* no marker found */ out_marker->odd_number = 0; out_marker->object = NULL; + return false; } } -static void _timing_fetch_inev(void) +static void marker_fetch_obj_write(struct stm_undo_s *start, + struct stm_undo_s *contention, + stm_loc_marker_t *out_marker) { - stm_loc_marker_t marker; - marker.tl = STM_SEGMENT->running_thread; - marker_fetch(&marker); - STM_PSEGMENT->marker_inev.odd_number = marker.odd_number; - STM_PSEGMENT->marker_inev.object = marker.object; -} - -static void marker_fetch_obj_write(object_t *obj, stm_loc_marker_t *out_marker) -{ - /* From 'out_marker->tl', fill in 'out_marker->segment_base' and - 'out_marker->odd_number' and 'out_marker->object' from the - marker associated with writing the 'obj'. + /* Fill out 'out_marker->odd_number' and 'out_marker->object' from + the marker just before 'contention' in the list starting at + 'start'. */ - assert(_has_mutex()); - - /* here, we acquired the other thread's marker_lock, which means that: - - (1) it has finished filling 'modified_old_objects' after it sets - up the write_locks[] value that we're conflicting with - - (2) it is not mutating 'modified_old_objects' right now (we have - the global mutex_lock at this point too). - */ - long i; - int in_segment_num = out_marker->tl->associated_segment_num; - assert(in_segment_num >= 1); - struct stm_priv_segment_info_s *pseg = get_priv_segment(in_segment_num); - struct list_s *mlst = pseg->modified_old_objects; - struct list_s *mlstm = pseg->modified_old_objects_markers; - assert(list_count(mlstm) <= 2 * list_count(mlst)); - for (i = list_count(mlstm) / 2; --i >= 0; ) { - if (list_item(mlst, i) == (uintptr_t)obj) { - out_marker->odd_number = list_item(mlstm, i * 2 + 0); - out_marker->object = (object_t *)list_item(mlstm, i * 2 + 1); + while (contention != start) { + --contention; + if (contention->type == TYPE_POSITION_MARKER) { + out_marker->odd_number = contention->marker_odd_number; + out_marker->object = contention->marker_object; return; } } + /* no position marker found... */ out_marker->odd_number = 0; out_marker->object = NULL; } -static void _timing_record_write(void) +static void _timing_record_write_position(void) { stm_loc_marker_t marker; - marker.tl = STM_SEGMENT->running_thread; - marker_fetch(&marker); + if (!marker_fetch(STM_SEGMENT->running_thread, &marker)) + return; - long base_count = list_count(STM_PSEGMENT->modified_old_objects); - struct list_s *mlstm = STM_PSEGMENT->modified_old_objects_markers; - while (list_count(mlstm) < 2 * base_count) { - mlstm = list_append2(mlstm, 0, 0); + struct list_s *list = STM_PSEGMENT->modified_old_objects; + uintptr_t i = STM_PSEGMENT->position_markers_last; + if (i < list_count(list)) { + struct stm_undo_s *undo = (struct stm_undo_s *)(list->items + i); + if (undo->type == TYPE_POSITION_MARKER && + undo->marker_odd_number == marker.odd_number && + undo->marker_object == marker.object) + return; /* already up-to-date */ } - mlstm = list_append2(mlstm, marker.odd_number, (uintptr_t)marker.object); - STM_PSEGMENT->modified_old_objects_markers = mlstm; + + STM_PSEGMENT->position_markers_last = list_count(list); + STM_PSEGMENT->modified_old_objects = list_append3( + list, + TYPE_POSITION_MARKER, /* type */ + marker.odd_number, /* marker_odd_number */ + (uintptr_t)marker.object); /* marker_object */ } -static void _timing_contention(enum stm_event_e kind, - uint8_t other_segment_num, object_t *obj) +static void timing_write_read_contention(struct stm_undo_s *start, + struct stm_undo_s *contention) { - struct stm_priv_segment_info_s *other_pseg; - other_pseg = get_priv_segment(other_segment_num); + if (stmcb_timing_event == NULL) + return; - char *other_segment_base = other_pseg->pub.segment_base; - acquire_marker_lock(other_segment_base); + stm_loc_marker_t marker; + marker_fetch_obj_write(start, contention, &marker); + stmcb_timing_event(STM_SEGMENT->running_thread, + STM_CONTENTION_WRITE_READ, &marker); +} - stm_loc_marker_t markers[2]; - - /* Collect the location for myself. It's usually the current - location, except in a write-read abort, in which case it's the - older location of the write. */ - markers[0].tl = STM_SEGMENT->running_thread; - markers[0].segment_base = STM_SEGMENT->segment_base; - - if (kind == STM_CONTENTION_WRITE_READ) - marker_fetch_obj_write(obj, &markers[0]); - else - marker_fetch(&markers[0]); - - /* For some categories, we can also collect the relevant information - for the other segment. */ - markers[1].tl = other_pseg->pub.running_thread; - markers[1].segment_base = other_pseg->pub.segment_base; - - switch (kind) { - case STM_CONTENTION_WRITE_WRITE: - marker_fetch_obj_write(obj, &markers[1]); - break; - case STM_CONTENTION_INEVITABLE: - markers[1].odd_number = other_pseg->marker_inev.odd_number; - markers[1].object = other_pseg->marker_inev.object; - break; - default: - markers[1].odd_number = 0; - markers[1].object = NULL; - break; - } - - stmcb_timing_event(markers[0].tl, kind, markers); - - /* only release the lock after stmcb_timing_event(), otherwise it could - run into race conditions trying to interpret 'markers[1].object' */ - release_marker_lock(other_segment_base); +static void _timing_become_inevitable(void) +{ + stm_loc_marker_t marker; + marker_fetch(STM_SEGMENT->running_thread, &marker); + stmcb_timing_event(STM_SEGMENT->running_thread, + STM_BECOME_INEVITABLE, &marker); } void (*stmcb_timing_event)(stm_thread_local_t *tl, /* the local thread */ enum stm_event_e event, - stm_loc_marker_t *markers); + stm_loc_marker_t *marker); diff --git a/c7/stm/marker.h b/c8/stm/marker.h copy from c7/stm/marker.h copy to c8/stm/marker.h --- a/c7/stm/marker.h +++ b/c8/stm/marker.h @@ -1,19 +1,17 @@ -static void _timing_record_write(void); -static void _timing_fetch_inev(void); -static void _timing_contention(enum stm_event_e kind, - uint8_t other_segment_num, object_t *obj); +static void _timing_record_write_position(void); +static void timing_write_read_contention(struct stm_undo_s *start, + struct stm_undo_s *contention); +static void _timing_become_inevitable(void); +#define timing_enabled() (stmcb_timing_event != NULL) + #define timing_event(tl, event) \ - (stmcb_timing_event != NULL ? stmcb_timing_event(tl, event, NULL) : (void)0) + (timing_enabled() ? stmcb_timing_event(tl, event, NULL) : (void)0) -#define timing_record_write() \ - (stmcb_timing_event != NULL ? _timing_record_write() : (void)0) +#define timing_record_write_position() \ + (timing_enabled() ? _timing_record_write_position() : (void)0) -#define timing_fetch_inev() \ - (stmcb_timing_event != NULL ? _timing_fetch_inev() : (void)0) - -#define timing_contention(kind, other_segnum, obj) \ - (stmcb_timing_event != NULL ? \ - _timing_contention(kind, other_segnum, obj) : (void)0) +#define timing_become_inevitable() \ + (timing_enabled() ? _timing_become_inevitable() : (void)0) diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c --- a/c8/stm/nursery.c +++ b/c8/stm/nursery.c @@ -185,6 +185,8 @@ struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count); for (; undo < end; undo++) { + if (undo->type == TYPE_POSITION_MARKER) + continue; _cards_cleared_in_object(pseg, undo->object, false); } LIST_FOREACH_R( @@ -410,6 +412,20 @@ } } +static void collect_roots_from_markers(uintptr_t len_old) +{ + dprintf(("collect_roots_from_markers\n")); + + /* visit the marker objects */ + struct list_s *list = STM_PSEGMENT->modified_old_objects; + struct stm_undo_s *undo = (struct stm_undo_s *)(list->items + len_old); + struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count); + + for (; undo < end; undo++) { + if (undo->type == TYPE_POSITION_MARKER) + minor_trace_if_young(&undo->marker_object); + } +} static void collect_objs_still_young_but_with_finalizers(void) { @@ -494,14 +510,25 @@ dprintf(("minor_collection commit=%d\n", (int)commit)); STM_PSEGMENT->minor_collect_will_commit_now = commit; + + uintptr_t len_old; + if (STM_PSEGMENT->overflow_number_has_been_used) + len_old = STM_PSEGMENT->position_markers_len_old; + else + len_old = 0; + if (!commit) { /* 'STM_PSEGMENT->overflow_number' is used now by this collection, in the sense that it's copied to the overflow objects */ STM_PSEGMENT->overflow_number_has_been_used = true; + STM_PSEGMENT->position_markers_len_old = + list_count(STM_PSEGMENT->modified_old_objects); } collect_cardrefs_to_nursery(); + collect_roots_from_markers(len_old); + collect_roots_in_nursery(); if (STM_PSEGMENT->finalizers != NULL) @@ -529,7 +556,11 @@ stm_safe_point(); + timing_event(STM_SEGMENT->running_thread, STM_GC_MINOR_START); + _do_minor_collection(commit); + + timing_event(STM_SEGMENT->running_thread, STM_GC_MINOR_DONE); } void stm_collect(long level) @@ -652,14 +683,13 @@ assert(get_priv_segment(i)->last_commit_log_entry->next == NULL || get_priv_segment(i)->last_commit_log_entry->next == INEV_RUNNING); if (!ok) { - assert(i != 0); /* sharing seg0 should never need an abort */ - if (STM_PSEGMENT->transaction_state == TS_NONE) { /* we found a segment that has stale read-marker data and thus is in conflict with committed objs. Since it is not running currently, it's fine to ignore it. */ continue; } + assert(i != 0); /* sharing seg0 should never need an abort */ /* tell it to abort when continuing */ STM_PSEGMENT->pub.nursery_end = NSE_SIGABORT; @@ -671,7 +701,7 @@ } - if (MINOR_NOTHING_TO_DO(STM_PSEGMENT)) /*TS_NONE segments have NOTHING_TO_DO*/ + if (MINOR_NOTHING_TO_DO(STM_PSEGMENT)) continue; assert(STM_PSEGMENT->transaction_state != TS_NONE); diff --git a/c8/stm/prof.c b/c8/stm/prof.c new file mode 100644 --- /dev/null +++ b/c8/stm/prof.c @@ -0,0 +1,122 @@ +#include +#include + + +static FILE *profiling_file; +static char *profiling_basefn = NULL; +static stm_expand_marker_fn profiling_expand_marker; + +#define MARKER_LEN_MAX 160 + + +static bool close_timing_log(void); /* forward */ + +static void _stm_profiling_event(stm_thread_local_t *tl, + enum stm_event_e event, + stm_loc_marker_t *marker) +{ + struct buf_s { + uint32_t tv_sec; + uint32_t tv_nsec; + uint32_t thread_num; + uint8_t event; + uint8_t marker_length; + char extra[MARKER_LEN_MAX+1]; + } __attribute__((packed)); + + struct buf_s buf; + struct timespec t; + clock_gettime(CLOCK_MONOTONIC, &t); + buf.tv_sec = t.tv_sec; + buf.tv_nsec = t.tv_nsec; + buf.thread_num = tl->thread_local_counter; + buf.event = event; + buf.marker_length = 0; + + if (marker != NULL && marker->odd_number != 0) { + buf.marker_length = profiling_expand_marker(get_segment_base(0), + marker, + buf.extra, MARKER_LEN_MAX); + } + + if (fwrite(&buf, offsetof(struct buf_s, extra) + buf.marker_length, + 1, profiling_file) != 1) { + fprintf(stderr, "stmgc: profiling log file closed unexpectedly: %m\n"); + close_timing_log(); + } +} + +static int default_expand_marker(char *b, stm_loc_marker_t *m, char *p, int s) +{ + *(uintptr_t *)p = m->odd_number; + return sizeof(uintptr_t); +} + +static bool open_timing_log(const char *filename) +{ + profiling_file = fopen(filename, "w"); + if (profiling_file == NULL) + return false; + + fwrite("STMGC-C8-PROF01\n", 16, 1, profiling_file); + stmcb_timing_event = _stm_profiling_event; + return true; +} + +static bool close_timing_log(void) +{ + if (stmcb_timing_event == &_stm_profiling_event) { + stmcb_timing_event = NULL; + fclose(profiling_file); + profiling_file = NULL; + return true; + } + return false; +} + +static void prof_forksupport_prepare(void) +{ + if (profiling_file != NULL) + fflush(profiling_file); +} + +static void prof_forksupport_child(void) +{ + if (close_timing_log() && profiling_basefn != NULL) { + char filename[1024]; + snprintf(filename, sizeof(filename), + "%s.fork%ld", profiling_basefn, (long)getpid()); + open_timing_log(filename); + } +} + +int stm_set_timing_log(const char *profiling_file_name, int fork_mode, + stm_expand_marker_fn expand_marker) +{ + close_timing_log(); + free(profiling_basefn); + profiling_basefn = NULL; + + if (profiling_file_name == NULL) + return 0; + + if (!expand_marker) + expand_marker = default_expand_marker; + profiling_expand_marker = expand_marker; + + static bool fork_support_ready = false; + if (!fork_support_ready) { + int res = pthread_atfork(prof_forksupport_prepare, + NULL, prof_forksupport_child); + if (res != 0) + stm_fatalerror("pthread_atfork() failed: %m"); + fork_support_ready = true; + } + + if (!open_timing_log(profiling_file_name)) + return -1; + + if (fork_mode != 0) + profiling_basefn = strdup(profiling_file_name); + return 0; +} diff --git a/c8/stm/setup.c b/c8/stm/setup.c --- a/c8/stm/setup.c +++ b/c8/stm/setup.c @@ -221,6 +221,8 @@ return (pthread_t *)(tl->creating_pthread); } +static int thread_local_counters = 0; + void stm_register_thread_local(stm_thread_local_t *tl) { int num; @@ -242,6 +244,7 @@ numbers automatically. */ tl->associated_segment_num = -1; tl->last_associated_segment_num = num + 1; + tl->thread_local_counter = ++thread_local_counters; *_get_cpth(tl) = pthread_self(); _init_shadow_stack(tl); set_gs_register(get_segment_base(num + 1)); diff --git a/c8/stm/sync.c b/c8/stm/sync.c --- a/c8/stm/sync.c +++ b/c8/stm/sync.c @@ -139,10 +139,13 @@ } /* No segment available. Wait until release_thread_segment() signals that one segment has been freed. */ + timing_event(tl, STM_WAIT_FREE_SEGMENT); cond_wait(C_SEGMENT_FREE); + timing_event(tl, STM_WAIT_DONE); /* Return false to the caller, which will call us again */ return false; + got_num: OPT_ASSERT(num >= 0 && num < NB_SEGMENTS-1); sync_ctl.in_use1[num+1] = 1; @@ -296,10 +299,12 @@ #ifdef STM_TESTS abort_with_mutex(); #endif + timing_event(STM_SEGMENT->running_thread, STM_WAIT_SYNC_PAUSE); cond_signal(C_AT_SAFE_POINT); STM_PSEGMENT->safe_point = SP_WAIT_FOR_C_REQUEST_REMOVED; cond_wait(C_REQUEST_REMOVED); STM_PSEGMENT->safe_point = SP_RUNNING; + timing_event(STM_SEGMENT->running_thread, STM_WAIT_DONE); } } diff --git a/c8/stm/sync.h b/c8/stm/sync.h --- a/c8/stm/sync.h +++ b/c8/stm/sync.h @@ -33,3 +33,5 @@ static bool pause_signalled, globally_unique_transaction; static void enter_safe_point_if_requested(void); + +#define safe_point_requested() (STM_SEGMENT->nursery_end != NURSERY_END) diff --git a/c8/stmgc.c b/c8/stmgc.c --- a/c8/stmgc.c +++ b/c8/stmgc.c @@ -14,6 +14,7 @@ #include "stm/gcpage.h" #include "stm/extra.h" #include "stm/fprintcolor.h" +#include "stm/marker.h" #include "stm/rewind_setjmp.h" #include "stm/finalizer.h" @@ -34,5 +35,7 @@ #include "stm/core.c" #include "stm/extra.c" #include "stm/fprintcolor.c" +#include "stm/marker.c" +#include "stm/prof.c" #include "stm/rewind_setjmp.c" #include "stm/finalizer.c" diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -68,6 +68,7 @@ /* the next fields are handled internally by the library */ int associated_segment_num; int last_associated_segment_num; + int thread_local_counter; struct stm_thread_local_s *prev, *next; void *creating_pthread[2]; } stm_thread_local_t; @@ -342,6 +343,77 @@ void stm_resume_all_other_threads(void); +/* Profiling events. In the comments: content of the markers, if any */ +enum stm_event_e { + /* always STM_TRANSACTION_START followed later by one of COMMIT or ABORT */ + STM_TRANSACTION_START, + STM_TRANSACTION_COMMIT, + STM_TRANSACTION_ABORT, + + /* write-read contention: a "marker" is included in the PYPYSTM file + saying where the write was done. Followed by STM_TRANSACTION_ABORT. */ + STM_CONTENTION_WRITE_READ, + + /* inevitable contention: all threads that try to become inevitable + have a STM_BECOME_INEVITABLE event with a position marker. Then, + if it waits it gets a STM_WAIT_OTHER_INEVITABLE. It is possible + that a thread gets STM_BECOME_INEVITABLE followed by + STM_TRANSACTION_ABORT if it fails to become inevitable. */ + STM_BECOME_INEVITABLE, + + /* always one STM_WAIT_xxx followed later by STM_WAIT_DONE */ + STM_WAIT_FREE_SEGMENT, + STM_WAIT_SYNC_PAUSE, + STM_WAIT_OTHER_INEVITABLE, + STM_WAIT_DONE, + + /* start and end of GC cycles */ + STM_GC_MINOR_START, + STM_GC_MINOR_DONE, + STM_GC_MAJOR_START, + STM_GC_MAJOR_DONE, + + _STM_EVENT_N +}; + +#define STM_EVENT_NAMES \ + "transaction start", \ + "transaction commit", \ + "transaction abort", \ + "contention write read", \ + "wait free segment", \ + "wait other inevitable", \ + "wait done", \ + "gc minor start", \ + "gc minor done", \ + "gc major start", \ + "gc major done" + +/* The markers pushed in the shadowstack are an odd number followed by a + regular object pointer. */ +typedef struct { + uintptr_t odd_number; + object_t *object; +} stm_loc_marker_t; +extern void (*stmcb_timing_event)(stm_thread_local_t *tl, /* the local thread */ + enum stm_event_e event, + stm_loc_marker_t *marker); + +/* Calling this sets up a stmcb_timing_event callback that will produce + a binary file called 'profiling_file_name'. Call it with + 'fork_mode == 0' for only the main process, and with + 'fork_mode == 1' to also write files called + 'profiling_file_name.fork' after a fork(). Call it with NULL to + stop profiling. Returns -1 in case of error (see errno then). + The optional 'expand_marker' function pointer is called to expand + the marker's odd_number and object into printable data, starting at + the given position and with the given maximum length. */ +typedef int (*stm_expand_marker_fn)(char *seg_base, stm_loc_marker_t *marker, + char *output, int output_size); +int stm_set_timing_log(const char *profiling_file_name, int fork_mode, + stm_expand_marker_fn expand_marker); + + /* Convenience macros to push the markers into the shadowstack */ #define STM_PUSH_MARKER(tl, odd_num, p) do { \ uintptr_t _odd_num = (odd_num); \ diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -124,6 +124,56 @@ long stm_call_on_abort(stm_thread_local_t *, void *key, void callback(void *)); long stm_call_on_commit(stm_thread_local_t *, void *key, void callback(void *)); +/* Profiling events. In the comments: content of the markers, if any */ +enum stm_event_e { + /* always STM_TRANSACTION_START followed later by one of COMMIT or ABORT */ + STM_TRANSACTION_START, + STM_TRANSACTION_COMMIT, + STM_TRANSACTION_ABORT, + + /* write-read contention: a "marker" is included in the PYPYSTM file + saying where the write was done. Followed by STM_TRANSACTION_ABORT. */ + STM_CONTENTION_WRITE_READ, + + /* inevitable contention: all threads that try to become inevitable + have a STM_BECOME_INEVITABLE event with a position marker. Then, + if it waits it gets a STM_WAIT_OTHER_INEVITABLE. It is possible + that a thread gets STM_BECOME_INEVITABLE followed by + STM_TRANSACTION_ABORT if it fails to become inevitable. */ + STM_BECOME_INEVITABLE, + + /* always one STM_WAIT_xxx followed later by STM_WAIT_DONE */ + STM_WAIT_FREE_SEGMENT, + STM_WAIT_SYNC_PAUSE, + STM_WAIT_OTHER_INEVITABLE, + STM_WAIT_DONE, + + /* start and end of GC cycles */ + STM_GC_MINOR_START, + STM_GC_MINOR_DONE, + STM_GC_MAJOR_START, + STM_GC_MAJOR_DONE, + ... +}; + +typedef struct { + uintptr_t odd_number; + object_t *object; +} stm_loc_marker_t; + +typedef void (*stmcb_timing_event_fn)(stm_thread_local_t *tl, + enum stm_event_e event, + stm_loc_marker_t *markers); +stmcb_timing_event_fn stmcb_timing_event; + +typedef int (*stm_expand_marker_fn)(char *seg_base, stm_loc_marker_t *marker, + char *output, int output_size); +int stm_set_timing_log(const char *profiling_file_name, int fork_mode, + stm_expand_marker_fn expand_marker); + +void stm_push_marker(stm_thread_local_t *, uintptr_t, object_t *); +void stm_update_marker_num(stm_thread_local_t *, uintptr_t); +void stm_pop_marker(stm_thread_local_t *); long _stm_count_modified_old_objects(void); long _stm_count_objects_pointing_to_nursery(void); @@ -147,7 +197,6 @@ void stm_enable_light_finalizer(object_t *); void (*stmcb_finalizer)(object_t *); - """) @@ -374,6 +423,21 @@ } } +void stm_push_marker(stm_thread_local_t *tl, uintptr_t onum, object_t *ob) +{ + STM_PUSH_MARKER(*tl, onum, ob); +} + +void stm_update_marker_num(stm_thread_local_t *tl, uintptr_t onum) +{ + STM_UPDATE_MARKER_NUM(*tl, onum); +} + +void stm_pop_marker(stm_thread_local_t *tl) +{ + STM_POP_MARKER(*tl); +} + long current_segment_num(void) { return STM_SEGMENT->segment_num; diff --git a/c7/test/test_marker.py b/c8/test/test_marker.py copy from c7/test/test_marker.py copy to c8/test/test_marker.py --- a/c7/test/test_marker.py +++ b/c8/test/test_marker.py @@ -7,16 +7,11 @@ def recording(self, *kinds): seen = [] @ffi.callback("stmcb_timing_event_fn") - def timing_event(tl, event, markers): + def timing_event(tl, event, marker): if len(kinds) > 0 and event not in kinds: return - if markers: - expanded = [] - for i in range(2): - expanded.append((markers[i].tl, - markers[i].segment_base, - markers[i].odd_number, - markers[i].object)) + if marker: + expanded = (marker.odd_number, marker.object) else: expanded = None seen.append((tl, event, expanded)) @@ -24,18 +19,13 @@ self.timing_event_keepalive = timing_event self.seen = seen - def check_recording(self, i1, o1, i2, o2, extra=None): + def check_recording(self, i1, o1, generating_thread_num=1): seen = self.seen - tl, event, markers = seen[0] - assert tl == self.tls[1] - segbase = lib._stm_get_segment_base - assert markers[0] == (self.tls[1], segbase(2), i1, o1) - assert markers[1] == (self.tls[0], segbase(1), i2, o2) - if extra is None: - assert len(seen) == 1 - else: - assert seen[1] == (self.tls[1], extra, None) - assert len(seen) == 2 + tl, event, marker = seen[0] + assert tl == self.tls[generating_thread_num] + assert marker == (i1, o1) + assert len(seen) == 1 + del self.seen[:] def test_marker_odd_simple(self): self.start_transaction() @@ -47,9 +37,7 @@ assert int(ffi.cast("uintptr_t", x)) == 29 def test_abort_marker_no_shadowstack(self): - self.recording(lib.STM_CONTENTION_WRITE_WRITE, - lib.STM_WAIT_CONTENTION, - lib.STM_ABORTING_OTHER_CONTENTION) + self.recording(lib.STM_CONTENTION_WRITE_READ) p = stm_allocate_old(16) # self.start_transaction() @@ -57,9 +45,13 @@ # self.switch(1) self.start_transaction() - py.test.raises(Conflict, stm_set_char, p, 'B') + stm_set_char(p, 'B') # - self.check_recording(0, ffi.NULL, 0, ffi.NULL) + self.switch(0) + self.commit_transaction() + # + py.test.raises(Conflict, self.switch, 1) + self.check_recording(0, ffi.NULL) def test_macros(self): self.start_transaction() @@ -96,8 +88,9 @@ py.test.raises(EmptyStack, self.pop_root) def test_double_abort_markers_cb_write_write(self): - self.recording(lib.STM_CONTENTION_WRITE_WRITE) + self.recording(lib.STM_CONTENTION_WRITE_READ) p = stm_allocate_old(16) + p2 = stm_allocate_old(16) # self.start_transaction() self.push_root(ffi.cast("object_t *", 19)) @@ -107,30 +100,39 @@ self.pop_root() self.push_root(ffi.cast("object_t *", 17)) self.push_root(ffi.cast("object_t *", ffi.NULL)) + stm_set_char(p, 'B') + stm_set_char(p2, 'C') stm_minor_collect() # self.switch(1) self.start_transaction() self.push_root(ffi.cast("object_t *", 21)) self.push_root(ffi.cast("object_t *", ffi.NULL)) - py.test.raises(Conflict, stm_set_char, p, 'B') + stm_set_char(p, 'B') # - self.check_recording(21, ffi.NULL, 19, ffi.NULL) + self.switch(0) + self.commit_transaction() + # + py.test.raises(Conflict, self.switch, 1) + self.check_recording(19, ffi.NULL) - def test_double_abort_markers_cb_inevitable(self): - self.recording(lib.STM_CONTENTION_INEVITABLE) + def test_become_inevitable_marker(self): + self.recording(lib.STM_BECOME_INEVITABLE) + # + self.switch(1) + self.start_transaction() + self.push_root(ffi.cast("object_t *", 19)) + self.push_root(ffi.cast("object_t *", ffi.NULL)) + self.become_inevitable() + # + self.check_recording(19, ffi.NULL) + + def test_abort_markers_cb_inevitable(self): + self.recording(lib.STM_BECOME_INEVITABLE) # self.start_transaction() - p = stm_allocate(16) - stm_set_char(p, 'A') - self.push_root(ffi.cast("object_t *", 19)) - self.push_root(ffi.cast("object_t *", p)) self.become_inevitable() - self.pop_root() - self.pop_root() - self.push_root(ffi.cast("object_t *", 17)) - self.push_root(ffi.cast("object_t *", ffi.NULL)) - stm_minor_collect() + self.check_recording(0, ffi.NULL, generating_thread_num=0) # self.switch(1) self.start_transaction() @@ -140,34 +142,16 @@ self.push_root(ffi.cast("object_t *", p)) py.test.raises(Conflict, self.become_inevitable) # - self.check_recording(21, p, 19, p) + # only during tests does become_inevitable() abort because + # another thread is already inevitable; but it should have + # recorded the marker first + self.check_recording(21, p, generating_thread_num=1) def test_read_write_contention(self): self.recording(lib.STM_CONTENTION_WRITE_READ) p = stm_allocate_old(16) # self.start_transaction() - assert stm_get_char(p) == '\x00' - # - self.switch(1) - self.start_transaction() - self.push_root(ffi.cast("object_t *", 19)) - self.push_root(ffi.cast("object_t *", ffi.NULL)) - stm_set_char(p, 'A') - self.pop_root() - self.pop_root() - self.push_root(ffi.cast("object_t *", 17)) - self.push_root(ffi.cast("object_t *", ffi.NULL)) - py.test.raises(Conflict, self.commit_transaction) - # - self.check_recording(19, ffi.NULL, 0, ffi.NULL) - - def test_double_remote_markers_cb_write_write(self): - self.recording(lib.STM_CONTENTION_WRITE_WRITE, - lib.STM_ABORTING_OTHER_CONTENTION) - p = stm_allocate_old(16) - # - self.start_transaction() self.push_root(ffi.cast("object_t *", 19)) self.push_root(ffi.cast("object_t *", ffi.NULL)) stm_set_char(p, 'A') @@ -175,49 +159,16 @@ self.pop_root() self.push_root(ffi.cast("object_t *", 17)) self.push_root(ffi.cast("object_t *", ffi.NULL)) - tl0 = self.get_stm_thread_local() # self.switch(1) self.start_transaction() - self.become_inevitable() - self.push_root(ffi.cast("object_t *", 21)) - self.push_root(ffi.cast("object_t *", ffi.NULL)) - stm_set_char(p, 'B') # aborts in #0 - self.pop_root() - self.pop_root() - self.push_root(ffi.cast("object_t *", 23)) - self.push_root(ffi.cast("object_t *", ffi.NULL)) + assert stm_get_char(p) == '\x00' # - py.test.raises(Conflict, self.switch, 0) - # - self.check_recording(21, ffi.NULL, 19, ffi.NULL, - extra=lib.STM_ABORTING_OTHER_CONTENTION) - - def test_double_remote_markers_cb_write_read(self): - self.recording(lib.STM_CONTENTION_WRITE_READ, - lib.STM_ABORTING_OTHER_CONTENTION) - p = stm_allocate_old(16) - # - self.start_transaction() - assert stm_get_char(p) == '\x00' # read - tl0 = self.get_stm_thread_local() - # - self.switch(1) - self.start_transaction() - self.become_inevitable() - self.push_root(ffi.cast("object_t *", 21)) - self.push_root(ffi.cast("object_t *", ffi.NULL)) - stm_set_char(p, 'B') # write, will abort #0 - self.pop_root() - self.pop_root() - self.push_root(ffi.cast("object_t *", 23)) - self.push_root(ffi.cast("object_t *", ffi.NULL)) + self.switch(0) self.commit_transaction() # - py.test.raises(Conflict, self.switch, 0) - # - self.check_recording(21, ffi.NULL, 0, ffi.NULL, - extra=lib.STM_ABORTING_OTHER_CONTENTION) + py.test.raises(Conflict, self.switch, 1) + self.check_recording(19, ffi.NULL) def test_all(self): self.recording() # all events From noreply at buildbot.pypy.org Tue Mar 10 17:48:55 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 10 Mar 2015 17:48:55 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c8: Report the minor collection times Message-ID: <20150310164855.875B21C04BE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c8 Changeset: r76297:c5bdbc0ead82 Date: 2015-03-10 17:47 +0100 http://bitbucket.org/pypy/pypy/changeset/c5bdbc0ead82/ Log: Report the minor collection times diff --git a/pypy/stm/print_stm_log.py b/pypy/stm/print_stm_log.py --- a/pypy/stm/print_stm_log.py +++ b/pypy/stm/print_stm_log.py @@ -91,6 +91,7 @@ self.cpu_time_committed = 0.0 self.cpu_time_aborted = 0.0 self.cpu_time_paused = 0.0 + self.cpu_time_gc_minor = 0.0 self._prev = (0.0, "stop") self.reset_counters() @@ -99,6 +100,7 @@ self._transaction_pause_time = 0.0 self._transaction_aborting = False self._transaction_inev = None + self._in_minor_coll = None def transaction_start(self, entry): self.reset_counters() @@ -167,9 +169,19 @@ def wait_for_other_inev(self, wait_time, out_conflicts): c = self.get_conflict(self._transaction_inev[0], out_conflicts) - assert wait_time >= 0 + assert wait_time >= 0.0 c.paused_time += wait_time + def gc_minor_start(self, event): + self._in_minor_coll = event.timestamp + + def gc_minor_done(self, event): + if self._in_minor_coll is not None: + gc_time = event.timestamp - self._in_minor_coll + assert gc_time >= 0.0 + self.cpu_time_gc_minor += gc_time + self._in_minor_coll = None + class ConflictSummary(object): def __init__(self, event, marker): @@ -250,6 +262,10 @@ t.transaction_pause(entry) elif entry.event == STM_WAIT_DONE: t.transaction_unpause(entry, conflicts) + elif entry.event == STM_GC_MINOR_START: + t.gc_minor_start(entry) + elif entry.event == STM_GC_MINOR_DONE: + t.gc_minor_done(entry) # if cnt == 0: raise Exception("empty file") @@ -272,6 +288,7 @@ total_cpu_time_total = (total_cpu_time_committed + total_cpu_time_aborted + total_cpu_time_paused) + total_cpu_time_gc_minor = stmlog.get_total_cpu_time_gc_minor() print 'CPU time in STM mode: %.3fs (%s) committed' % ( total_cpu_time_committed, percent(total_cpu_time_committed, total_time)) print ' %.3fs (%s) aborted' % ( @@ -280,6 +297,8 @@ total_cpu_time_paused, percent(total_cpu_time_paused, total_time)) print ' %.3fs (%s) total' % ( total_cpu_time_total, percent(total_cpu_time_total, total_time)) + print ' including %.3fs (%s) minor GC collections' % ( + total_cpu_time_gc_minor, percent(total_cpu_time_gc_minor, total_time)) print # values = stmlog.get_conflicts() @@ -308,6 +327,9 @@ def get_total_cpu_time_paused(self): return sum([v.cpu_time_paused for v in self.threads.values()]) + def get_total_cpu_time_gc_minor(self): + return sum([v.cpu_time_gc_minor for v in self.threads.values()]) + def get_conflicts(self): values = self.conflicts.values() values.sort(key=ConflictSummary.sortkey) From noreply at buildbot.pypy.org Tue Mar 10 17:48:56 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 10 Mar 2015 17:48:56 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c8: Report major collections Message-ID: <20150310164856.B54261C04BE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c8 Changeset: r76298:9f955c2fa6c8 Date: 2015-03-10 17:48 +0100 http://bitbucket.org/pypy/pypy/changeset/9f955c2fa6c8/ Log: Report major collections diff --git a/pypy/stm/print_stm_log.py b/pypy/stm/print_stm_log.py --- a/pypy/stm/print_stm_log.py +++ b/pypy/stm/print_stm_log.py @@ -92,7 +92,9 @@ self.cpu_time_aborted = 0.0 self.cpu_time_paused = 0.0 self.cpu_time_gc_minor = 0.0 + self.cpu_time_gc_major = 0.0 self._prev = (0.0, "stop") + self._in_major_coll = None self.reset_counters() def reset_counters(self): @@ -182,6 +184,16 @@ self.cpu_time_gc_minor += gc_time self._in_minor_coll = None + def gc_major_start(self, event): + self._in_major_coll = event.timestamp + + def gc_major_done(self, event): + if self._in_major_coll is not None: + gc_time = event.timestamp - self._in_major_coll + assert gc_time >= 0.0 + self.cpu_time_gc_major += gc_time + self._in_major_coll = None + class ConflictSummary(object): def __init__(self, event, marker): @@ -266,6 +278,10 @@ t.gc_minor_start(entry) elif entry.event == STM_GC_MINOR_DONE: t.gc_minor_done(entry) + elif entry.event == STM_GC_MAJOR_START: + t.gc_major_start(entry) + elif entry.event == STM_GC_MAJOR_DONE: + t.gc_major_done(entry) # if cnt == 0: raise Exception("empty file") @@ -289,6 +305,7 @@ total_cpu_time_aborted + total_cpu_time_paused) total_cpu_time_gc_minor = stmlog.get_total_cpu_time_gc_minor() + total_cpu_time_gc_major = stmlog.get_total_cpu_time_gc_major() print 'CPU time in STM mode: %.3fs (%s) committed' % ( total_cpu_time_committed, percent(total_cpu_time_committed, total_time)) print ' %.3fs (%s) aborted' % ( @@ -299,6 +316,8 @@ total_cpu_time_total, percent(total_cpu_time_total, total_time)) print ' including %.3fs (%s) minor GC collections' % ( total_cpu_time_gc_minor, percent(total_cpu_time_gc_minor, total_time)) + print ' and %.3fs (%s) major GC collections' % ( + total_cpu_time_gc_major, percent(total_cpu_time_gc_major, total_time)) print # values = stmlog.get_conflicts() @@ -330,6 +349,9 @@ def get_total_cpu_time_gc_minor(self): return sum([v.cpu_time_gc_minor for v in self.threads.values()]) + def get_total_cpu_time_gc_major(self): + return sum([v.cpu_time_gc_major for v in self.threads.values()]) + def get_conflicts(self): values = self.conflicts.values() values.sort(key=ConflictSummary.sortkey) From noreply at buildbot.pypy.org Tue Mar 10 17:57:33 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 10 Mar 2015 17:57:33 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c8: alignment Message-ID: <20150310165733.7CD011C1193@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c8 Changeset: r76299:ec4da66ee4d0 Date: 2015-03-10 17:57 +0100 http://bitbucket.org/pypy/pypy/changeset/ec4da66ee4d0/ Log: alignment diff --git a/pypy/stm/print_stm_log.py b/pypy/stm/print_stm_log.py --- a/pypy/stm/print_stm_log.py +++ b/pypy/stm/print_stm_log.py @@ -296,7 +296,7 @@ start_time = stmlog.start_time total_time = stmlog.total_time print - print 'Total real time: %.3fs' % (total_time,) + print 'Total real time: %9.3fs' % (total_time,) # total_cpu_time_committed = stmlog.get_total_cpu_time_committed() total_cpu_time_aborted = stmlog.get_total_cpu_time_aborted() @@ -306,17 +306,17 @@ total_cpu_time_paused) total_cpu_time_gc_minor = stmlog.get_total_cpu_time_gc_minor() total_cpu_time_gc_major = stmlog.get_total_cpu_time_gc_major() - print 'CPU time in STM mode: %.3fs (%s) committed' % ( + print 'CPU time in STM mode:%9.3fs (%4s) committed' % ( total_cpu_time_committed, percent(total_cpu_time_committed, total_time)) - print ' %.3fs (%s) aborted' % ( + print ' %9.3fs (%4s) aborted' % ( total_cpu_time_aborted, percent(total_cpu_time_aborted, total_time)) - print ' %.3fs (%s) paused' % ( + print ' %9.3fs (%4s) paused' % ( total_cpu_time_paused, percent(total_cpu_time_paused, total_time)) - print ' %.3fs (%s) total' % ( + print ' %9.3fs (%4s) TOTAL' % ( total_cpu_time_total, percent(total_cpu_time_total, total_time)) - print ' including %.3fs (%s) minor GC collections' % ( + print ' including %9.3fs (%4s) minor GC collections' % ( total_cpu_time_gc_minor, percent(total_cpu_time_gc_minor, total_time)) - print ' and %.3fs (%s) major GC collections' % ( + print ' and %9.3fs (%4s) major GC collections' % ( total_cpu_time_gc_major, percent(total_cpu_time_gc_major, total_time)) print # From noreply at buildbot.pypy.org Tue Mar 10 18:17:03 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 10 Mar 2015 18:17:03 +0100 (CET) Subject: [pypy-commit] pypy default: Make some more virtualizable-array rules explicit Message-ID: <20150310171703.2A8341C015A@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76300:17681166180a Date: 2015-03-10 18:16 +0100 http://bitbucket.org/pypy/pypy/changeset/17681166180a/ Log: Make some more virtualizable-array rules explicit diff --git a/rpython/doc/jit/virtualizable.rst b/rpython/doc/jit/virtualizable.rst --- a/rpython/doc/jit/virtualizable.rst +++ b/rpython/doc/jit/virtualizable.rst @@ -44,8 +44,14 @@ virtualizable arrays that can be very confusing. Those will usually end up with a compile-time error (as opposed to strange behavior). The rules are: +* A virtualizable array must be a fixed-size list. After it is + initialized (e.g. in ``Frame.__init__``) you cannot resize it at all. + You cannot assign a different list to the field, or even pass around the + list. You can only access ``frame.array[index]`` directly. + * Each array access must be with a known positive index that cannot raise - an ``IndexError``. Using ``no = jit.hint(no, promote=True)`` might be useful + an ``IndexError``. + Using ``index = jit.hint(index, promote=True)`` might be useful to get a constant-number access. This is only safe if the index is actually constant or changing rarely within the context of the user's code. From noreply at buildbot.pypy.org Tue Mar 10 20:01:11 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 10 Mar 2015 20:01:11 +0100 (CET) Subject: [pypy-commit] stmgc c8-hashtable: Redo the hashtable work Message-ID: <20150310190111.293AC1C04BE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-hashtable Changeset: r1712:3baa8368a958 Date: 2015-03-10 19:07 +0100 http://bitbucket.org/pypy/stmgc/changeset/3baa8368a958/ Log: Redo the hashtable work From noreply at buildbot.pypy.org Tue Mar 10 20:01:12 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 10 Mar 2015 20:01:12 +0100 (CET) Subject: [pypy-commit] stmgc c8-hashtable: Add the code from c7 Message-ID: <20150310190112.53ED21C04BE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-hashtable Changeset: r1713:397c4402e270 Date: 2015-03-10 20:01 +0100 http://bitbucket.org/pypy/stmgc/changeset/397c4402e270/ Log: Add the code from c7 diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -284,6 +284,17 @@ static void _signal_handler(int sig, siginfo_t *siginfo, void *context); static bool _stm_validate(); +static inline bool was_read_remote(char *base, object_t *obj) +{ + uint8_t other_transaction_read_version = + ((struct stm_segment_info_s *)REAL_ADDRESS(base, STM_PSEGMENT)) + ->transaction_read_version; + uint8_t rm = ((struct stm_read_marker_s *) + (base + (((uintptr_t)obj) >> 4)))->rm; + assert(rm <= other_transaction_read_version); + return rm == other_transaction_read_version; +} + static inline void _duck(void) { /* put a call to _duck() between two instructions that set 0 into a %gs-prefixed address and that may otherwise be replaced with diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -127,6 +127,56 @@ return o; } +static void _fill_preexisting_slice(long segnum, char *dest, + const char *src, uintptr_t size) +{ + uintptr_t np = dest - get_segment_base(segnum); + if (get_page_status_in(segnum, np / 4096) != PAGE_NO_ACCESS) + memcpy(dest, src, size); +} + +object_t *stm_allocate_preexisting(ssize_t size_rounded_up, + const char *initial_data) +{ + stm_char *np = allocate_outside_nursery_large(size_rounded_up); + uintptr_t nobj = (uintptr_t)np; + dprintf(("allocate_preexisting: %p\n", (object_t *)nobj)); + + char *dest = stm_object_pages + nobj; + memcpy(dest, initial_data, size_rounded_up); + ((struct object_s *)dest)->stm_flags = GCFLAG_WRITE_BARRIER; + + acquire_all_privatization_locks(); + + long j; + for (j = 1; j <= NB_SEGMENTS; j++) { + const char *src = initial_data; + char *dest = get_segment_base(j) + nobj; + char *end = dest + size_rounded_up; + + while (((uintptr_t)dest) / 4096 != ((uintptr_t)end - 1) / 4096) { + uintptr_t count = 4096 - ((uintptr_t)dest) / 4096; + _fill_preexisting_slice(j, dest, src, count); + src += count; + dest += count; + } + _fill_preexisting_slice(j, dest, src, end - dest); + +#ifdef STM_TESTS + /* can't really enable this check outside tests, because there is + a change that the transaction_state changes in parallel */ + if (get_priv_segment(j)->transaction_state != TS_NONE) { + assert(!was_read_remote(get_segment_base(j), (object_t *)nobj)); + } +#endif + } + + release_all_privatization_locks(); + + write_fence(); /* make sure 'nobj' is fully initialized from + all threads here */ + return (object_t *)nobj; +} /************************************************************/ diff --git a/c7/stm/hashtable.c b/c8/stm/hashtable.c copy from c7/stm/hashtable.c copy to c8/stm/hashtable.c --- a/c7/stm/hashtable.c +++ b/c8/stm/hashtable.c @@ -110,7 +110,7 @@ /* can only be safely called during major GC, when all other threads are suspended */ long i; - for (i = 1; i <= NB_SEGMENTS; i++) { + for (i = 1; i < NB_SEGMENTS; i++) { if (get_priv_segment(i)->transaction_state == TS_NONE) continue; if (was_read_remote(get_segment_base(i), obj)) diff --git a/c8/stm/misc.c b/c8/stm/misc.c --- a/c8/stm/misc.c +++ b/c8/stm/misc.c @@ -31,10 +31,7 @@ bool _stm_was_read(object_t *obj) { - uint8_t rm = ((struct stm_read_marker_s *) - (STM_SEGMENT->segment_base + (((uintptr_t)obj) >> 4)))->rm; - assert(rm <= STM_SEGMENT->transaction_read_version); - return rm == STM_SEGMENT->transaction_read_version; + return was_read_remote(STM_SEGMENT->segment_base, obj); } bool _stm_was_written(object_t *obj) diff --git a/c8/stmgc.c b/c8/stmgc.c --- a/c8/stmgc.c +++ b/c8/stmgc.c @@ -39,3 +39,4 @@ #include "stm/prof.c" #include "stm/rewind_setjmp.c" #include "stm/finalizer.c" +#include "stm/hashtable.c" diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -467,6 +467,37 @@ /* dummies for now: */ static inline void stm_flush_timing(stm_thread_local_t *tl, int verbose) {} + +/* Hashtables. Keys are 64-bit unsigned integers, values are + 'object_t *'. Note that the type 'stm_hashtable_t' is not an + object type at all; you need to allocate and free it explicitly. + If you want to embed the hashtable inside an 'object_t' you + probably need a light finalizer to do the freeing. */ +typedef struct stm_hashtable_s stm_hashtable_t; +typedef TLPREFIX struct stm_hashtable_entry_s stm_hashtable_entry_t; + +stm_hashtable_t *stm_hashtable_create(void); +void stm_hashtable_free(stm_hashtable_t *); +stm_hashtable_entry_t *stm_hashtable_lookup(object_t *, stm_hashtable_t *, + uintptr_t key); +object_t *stm_hashtable_read(object_t *, stm_hashtable_t *, uintptr_t key); +void stm_hashtable_write(object_t *, stm_hashtable_t *, uintptr_t key, + object_t *nvalue, stm_thread_local_t *); +void stm_hashtable_write_entry(object_t *hobj, stm_hashtable_entry_t *entry, + object_t *nvalue); +long stm_hashtable_length_upper_bound(stm_hashtable_t *); +long stm_hashtable_list(object_t *, stm_hashtable_t *, + stm_hashtable_entry_t **results); +extern uint32_t stm_hashtable_entry_userdata; +void stm_hashtable_tracefn(stm_hashtable_t *, void (object_t **)); + +struct stm_hashtable_entry_s { + struct object_s header; + uint32_t userdata; + uintptr_t index; + object_t *object; +}; + /* ==================== END ==================== */ static void (*stmcb_expand_marker)(char *segment_base, uintptr_t odd_number, diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -197,6 +197,25 @@ void stm_enable_light_finalizer(object_t *); void (*stmcb_finalizer)(object_t *); + +typedef struct stm_hashtable_s stm_hashtable_t; +typedef ... stm_hashtable_entry_t; +stm_hashtable_t *stm_hashtable_create(void); +void stm_hashtable_free(stm_hashtable_t *); +bool _check_hashtable_read(object_t *, stm_hashtable_t *, uintptr_t key); +object_t *hashtable_read_result; +bool _check_hashtable_write(object_t *, stm_hashtable_t *, uintptr_t key, + object_t *nvalue, stm_thread_local_t *tl); +long stm_hashtable_length_upper_bound(stm_hashtable_t *); +long stm_hashtable_list(object_t *, stm_hashtable_t *, + stm_hashtable_entry_t **results); +uint32_t stm_hashtable_entry_userdata; +void stm_hashtable_tracefn(stm_hashtable_t *, void (object_t **)); + +void _set_hashtable(object_t *obj, stm_hashtable_t *h); +stm_hashtable_t *_get_hashtable(object_t *obj); +uintptr_t _get_entry_index(stm_hashtable_entry_t *entry); +object_t *_get_entry_object(stm_hashtable_entry_t *entry); """) @@ -299,6 +318,19 @@ CHECKED(stm_validate()); } +object_t *hashtable_read_result; + +bool _check_hashtable_read(object_t *hobj, stm_hashtable_t *h, uintptr_t key) +{ + CHECKED(hashtable_read_result = stm_hashtable_read(hobj, h, key)); +} + +bool _check_hashtable_write(object_t *hobj, stm_hashtable_t *h, uintptr_t key, + object_t *nvalue, stm_thread_local_t *tl) +{ + CHECKED(stm_hashtable_write(hobj, h, key, nvalue, tl)); +} + #undef CHECKED @@ -326,6 +358,32 @@ return *WEAKREF_PTR(obj, size); } +void _set_hashtable(object_t *obj, stm_hashtable_t *h) +{ + stm_char *field_addr = ((stm_char*)obj); + field_addr += SIZEOF_MYOBJ; /* header */ + *(stm_hashtable_t *TLPREFIX *)field_addr = h; +} + +stm_hashtable_t *_get_hashtable(object_t *obj) +{ + stm_char *field_addr = ((stm_char*)obj); + field_addr += SIZEOF_MYOBJ; /* header */ + return *(stm_hashtable_t *TLPREFIX *)field_addr; +} + +uintptr_t _get_entry_index(stm_hashtable_entry_t *entry) +{ + stm_read((object_t *)entry); + return entry->index; +} + +object_t *_get_entry_object(stm_hashtable_entry_t *entry) +{ + stm_read((object_t *)entry); + return entry->object; +} + void _set_ptr(object_t *obj, int n, object_t *v) { long nrefs = (long)((myobj_t*)obj)->type_id - 421420; @@ -351,11 +409,17 @@ } - ssize_t stmcb_size_rounded_up(struct object_s *obj) { struct myobj_s *myobj = (struct myobj_s*)obj; + assert(myobj->type_id != 0); if (myobj->type_id < 421420) { + if (myobj->type_id == 421419) { /* hashtable */ + return sizeof(struct myobj_s) + 1 * sizeof(void*); + } + if (myobj->type_id == 421418) { /* hashtable entry */ + return sizeof(struct stm_hashtable_entry_s); + } /* basic case: tid equals 42 plus the size of the object */ assert(myobj->type_id >= 42 + sizeof(struct myobj_s)); assert((myobj->type_id - 42) >= 16); @@ -371,11 +435,21 @@ } } - void stmcb_trace(struct object_s *obj, void visit(object_t **)) { int i; struct myobj_s *myobj = (struct myobj_s*)obj; + if (myobj->type_id == 421419) { + /* hashtable */ + stm_hashtable_t *h = *((stm_hashtable_t **)(myobj + 1)); + stm_hashtable_tracefn(h, visit); + return; + } + if (myobj->type_id == 421418) { + /* hashtable entry */ + object_t **ref = &((struct stm_hashtable_entry_s *)myobj)->object; + visit(ref); + } if (myobj->type_id < 421420) { /* basic case: no references */ return; @@ -396,6 +470,7 @@ { int i; struct myobj_s *myobj = (struct myobj_s*)obj; + assert(myobj->type_id != 0); assert(myobj->type_id != 421419); assert(myobj->type_id != 421418); if (myobj->type_id < 421420) { @@ -413,6 +488,9 @@ uintptr_t offset_itemsize[2]) { struct myobj_s *myobj = (struct myobj_s*)obj; + assert(myobj->type_id != 0); + assert(myobj->type_id != 421419); + assert(myobj->type_id != 421418); if (myobj->type_id < 421420) { offset_itemsize[0] = SIZEOF_MYOBJ; offset_itemsize[1] = 1; @@ -467,6 +545,7 @@ CARD_CLEAR = 0 CARD_MARKED = lib._STM_CARD_MARKED CARD_MARKED_OLD = lib._stm_get_transaction_read_version +lib.stm_hashtable_entry_userdata = 421418 class Conflict(Exception): @@ -529,6 +608,18 @@ lib._set_type_id(o, tid) return o +def stm_allocate_hashtable(): + o = lib.stm_allocate(16) + tid = 421419 + lib._set_type_id(o, tid) + h = lib.stm_hashtable_create() + lib._set_hashtable(o, h) + return o + +def get_hashtable(o): + assert lib._get_type_id(o) == 421419 + return lib._get_hashtable(o) + def stm_get_weakref(o): return lib._get_weakref(o) diff --git a/c7/test/test_hashtable.py b/c8/test/test_hashtable.py copy from c7/test/test_hashtable.py copy to c8/test/test_hashtable.py From noreply at buildbot.pypy.org Tue Mar 10 21:31:04 2015 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 10 Mar 2015 21:31:04 +0100 (CET) Subject: [pypy-commit] cffi win32-zintegration: skip zintegration on win32, adjust zdistutils for setuptools Message-ID: <20150310203104.100681C04BE@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: win32-zintegration Changeset: r1662:caa37e313aa3 Date: 2015-03-10 22:31 +0200 http://bitbucket.org/cffi/cffi/changeset/caa37e313aa3/ Log: skip zintegration on win32, adjust zdistutils for setuptools diff --git a/testing/test_zdistutils.py b/testing/test_zdistutils.py --- a/testing/test_zdistutils.py +++ b/testing/test_zdistutils.py @@ -164,7 +164,8 @@ assert lib.sin(12.3) == math.sin(12.3) v = ffi.verifier ext = v.get_extension() - assert 'distutils.extension.Extension' in str(ext.__class__) + assert 'distutils.extension.Extension' in str(ext.__class__) or \ + 'setuptools.extension.Extension' in str(ext.__class__) assert ext.sources == [maybe_relative_path(v.sourcefilename)] assert ext.name == v.get_module_name() assert ext.define_macros == [('TEST_EXTENSION_OBJECT', '1')] @@ -193,7 +194,8 @@ assert lib.test1eoes(7.0) == 42.0 v = ffi.verifier ext = v.get_extension() - assert 'distutils.extension.Extension' in str(ext.__class__) + assert 'distutils.extension.Extension' in str(ext.__class__) or \ + 'setuptools.extension.Extension' in str(ext.__class__) assert ext.sources == [maybe_relative_path(v.sourcefilename), extra_source] assert ext.name == v.get_module_name() diff --git a/testing/test_zintegration.py b/testing/test_zintegration.py --- a/testing/test_zintegration.py +++ b/testing/test_zintegration.py @@ -3,6 +3,9 @@ import subprocess from testing.udir import udir +if sys.platform == 'win32': + py.test.skip('snippets do not run on win32') + def create_venv(name): tmpdir = udir.join(name) try: From noreply at buildbot.pypy.org Tue Mar 10 21:46:42 2015 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 10 Mar 2015 21:46:42 +0100 (CET) Subject: [pypy-commit] cffi win32-zintegration: close branch to be merged Message-ID: <20150310204642.A798F1C0196@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: win32-zintegration Changeset: r1663:e212575e19a6 Date: 2015-03-10 22:47 +0200 http://bitbucket.org/cffi/cffi/changeset/e212575e19a6/ Log: close branch to be merged From noreply at buildbot.pypy.org Tue Mar 10 21:46:43 2015 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 10 Mar 2015 21:46:43 +0100 (CET) Subject: [pypy-commit] cffi default: merge with win32-zintegration which skips/fixes failures on win32 w/setuptools Message-ID: <20150310204643.AA3721C0196@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r1664:edfbf69377b0 Date: 2015-03-10 22:48 +0200 http://bitbucket.org/cffi/cffi/changeset/edfbf69377b0/ Log: merge with win32-zintegration which skips/fixes failures on win32 w/setuptools diff --git a/testing/test_zdistutils.py b/testing/test_zdistutils.py --- a/testing/test_zdistutils.py +++ b/testing/test_zdistutils.py @@ -164,7 +164,8 @@ assert lib.sin(12.3) == math.sin(12.3) v = ffi.verifier ext = v.get_extension() - assert 'distutils.extension.Extension' in str(ext.__class__) + assert 'distutils.extension.Extension' in str(ext.__class__) or \ + 'setuptools.extension.Extension' in str(ext.__class__) assert ext.sources == [maybe_relative_path(v.sourcefilename)] assert ext.name == v.get_module_name() assert ext.define_macros == [('TEST_EXTENSION_OBJECT', '1')] @@ -193,7 +194,8 @@ assert lib.test1eoes(7.0) == 42.0 v = ffi.verifier ext = v.get_extension() - assert 'distutils.extension.Extension' in str(ext.__class__) + assert 'distutils.extension.Extension' in str(ext.__class__) or \ + 'setuptools.extension.Extension' in str(ext.__class__) assert ext.sources == [maybe_relative_path(v.sourcefilename), extra_source] assert ext.name == v.get_module_name() diff --git a/testing/test_zintegration.py b/testing/test_zintegration.py --- a/testing/test_zintegration.py +++ b/testing/test_zintegration.py @@ -3,6 +3,9 @@ import subprocess from testing.udir import udir +if sys.platform == 'win32': + py.test.skip('snippets do not run on win32') + def create_venv(name): tmpdir = udir.join(name) try: @@ -12,6 +15,20 @@ except OSError as e: py.test.skip("Cannot execute virtualenv: %s" % (e,)) + try: + deepcopy = os.symlink + except: + import shutil, errno + def deepcopy(src, dst): + try: + shutil.copytree(src, dst) + except OSError as e: + if e.errno in (errno.ENOTDIR, errno.EINVAL): + shutil.copy(src, dst) + else: + print 'got errno',e.errno,'not',errno.ENOTDIR + raise + site_packages = None for dirpath, dirnames, filenames in os.walk(str(tmpdir)): if os.path.basename(dirpath) == 'site-packages': @@ -31,7 +48,7 @@ modules += ('ply',) # needed for older versions of pycparser for module in modules: target = imp.find_module(module)[1] - os.symlink(target, os.path.join(site_packages, + deepcopy(target, os.path.join(site_packages, os.path.basename(target))) return tmpdir @@ -50,7 +67,11 @@ python_f.write(py.code.Source(python_snippet)) try: os.chdir(str(SNIPPET_DIR.join(dirname))) - vp = str(venv_dir.join('bin/python')) + if os.name == 'nt': + bindir = 'Scripts' + else: + bindir = 'bin' + vp = str(venv_dir.join(bindir).join('python')) subprocess.check_call((vp, 'setup.py', 'clean')) subprocess.check_call((vp, 'setup.py', 'install')) subprocess.check_call((vp, str(python_f))) From noreply at buildbot.pypy.org Tue Mar 10 21:56:44 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Tue, 10 Mar 2015 21:56:44 +0100 (CET) Subject: [pypy-commit] pypy py3.3: Add support for pickling set iterator Message-ID: <20150310205644.D73571C04BE@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3.3 Changeset: r76301:91188a5ae517 Date: 2015-03-06 16:10 +0100 http://bitbucket.org/pypy/pypy/changeset/91188a5ae517/ Log: Add support for pickling set iterator diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py --- a/pypy/objspace/std/setobject.py +++ b/pypy/objspace/std/setobject.py @@ -1520,10 +1520,23 @@ return w_key raise OperationError(space.w_StopIteration, space.w_None) + def descr_reduce(self, space): + # copy the iterator state + w_set = self.iterimplementation.setimplementation + w_clone = W_SetIterObject(space, w_set.iter()) + # spool until we have the same pos + for x in xrange(self.iterimplementation.pos): + w_clone.descr_next(space) + w_res = space.call_function(space.w_list, w_clone) + w_iter = space.builtin.get('iter') + return space.newtuple([w_iter, space.newtuple([w_res])]) + + W_SetIterObject.typedef = TypeDef("setiterator", __length_hint__ = gateway.interp2app(W_SetIterObject.descr_length_hint), __iter__ = gateway.interp2app(W_SetIterObject.descr_iter), - __next__ = gateway.interp2app(W_SetIterObject.descr_next) + __next__ = gateway.interp2app(W_SetIterObject.descr_next), + __reduce__ = gateway.interp2app(W_SetIterObject.descr_reduce), ) setiter_typedef = W_SetIterObject.typedef diff --git a/pypy/objspace/std/test/test_setobject.py b/pypy/objspace/std/test/test_setobject.py --- a/pypy/objspace/std/test/test_setobject.py +++ b/pypy/objspace/std/test/test_setobject.py @@ -1062,3 +1062,17 @@ s = set([1, 2, 3]) s.intersection_update(set()) assert strategy(s) == "EmptySetStrategy" + + def test_pickle(self): + d = {1, 2, 3} + it = iter(d) + first = next(it) + reduced = it.__reduce__() + rebuild, args = reduced + assert rebuild is iter + new = rebuild(*args) + items = set(new) + assert len(items) == 2 + items.add(first) + assert items == set(d) + From noreply at buildbot.pypy.org Tue Mar 10 21:56:46 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Tue, 10 Mar 2015 21:56:46 +0100 (CET) Subject: [pypy-commit] pypy py3.3: Add a failing test that shows that interned strings are not always preserved. Message-ID: <20150310205646.2CC8D1C04BE@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3.3 Changeset: r76302:a0960c3a822c Date: 2015-03-10 14:03 +0100 http://bitbucket.org/pypy/pypy/changeset/a0960c3a822c/ Log: Add a failing test that shows that interned strings are not always preserved. diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py --- a/pypy/objspace/std/test/test_dictmultiobject.py +++ b/pypy/objspace/std/test/test_dictmultiobject.py @@ -618,6 +618,10 @@ def test_bytes_keys(self): assert isinstance(list({b'a': 1})[0], bytes) + def test_interned_keywords(self): + assert list(dict(abcdef=1))[0] is 'abcdef' + + class AppTest_DictMultiObject(AppTest_DictObject): def test_emptydict_unhashable(self): From noreply at buildbot.pypy.org Tue Mar 10 21:56:47 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Tue, 10 Mar 2015 21:56:47 +0100 (CET) Subject: [pypy-commit] pypy py3.3: Add a cffi implementation of the _decimal module. Message-ID: <20150310205647.955781C04BE@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3.3 Changeset: r76303:99b65d047f8c Date: 2015-03-10 14:47 +0100 http://bitbucket.org/pypy/pypy/changeset/99b65d047f8c/ Log: Add a cffi implementation of the _decimal module. diff too long, truncating to 2000 out of 18186 lines diff --git a/lib-python/3/test/test_decimal.py b/lib-python/3/test/test_decimal.py --- a/lib-python/3/test/test_decimal.py +++ b/lib-python/3/test/test_decimal.py @@ -4011,8 +4011,8 @@ self.assertRaises(KeyError, Context, traps=["Q"]) # Type error in conversion - self.assertRaises(TypeError, Context, flags=(0,1)) - self.assertRaises(TypeError, Context, traps=(1,0)) + self.assertRaises(TypeError, Context, flags=0) + self.assertRaises(TypeError, Context, traps=1) class CContextInputValidation(ContextInputValidation): decimal = C @@ -4150,7 +4150,7 @@ self.assertEqual(C.__version__, P.__version__) self.assertEqual(C.__libmpdec_version__, P.__libmpdec_version__) - x = dir(C) + x = [s for s in dir(C) if '__' in s or not s.startswith('_')] y = [s for s in dir(P) if '__' in s or not s.startswith('_')] self.assertEqual(set(x) - set(y), set()) @@ -4158,6 +4158,7 @@ x = [s for s in dir(C.Context()) if '__' in s or not s.startswith('_')] y = [s for s in dir(P.Context()) if '__' in s or not s.startswith('_')] + y.append('__slots__') self.assertEqual(set(x) - set(y), set()) def test_decimal_attributes(self): @@ -4793,15 +4794,17 @@ self.assertRaises(OverflowError, Context, Emax=int_max+1) self.assertRaises(OverflowError, Context, Emin=-int_max-2) self.assertRaises(OverflowError, Context, clamp=int_max+1) - self.assertRaises(OverflowError, Context, capitals=int_max+1) + self.assertRaises((OverflowError, ValueError), Context, capitals=int_max+1) # OverflowError, general ValueError for attr in ('prec', 'Emin', 'Emax', 'capitals', 'clamp'): - self.assertRaises(OverflowError, setattr, c, attr, int_max+1) - self.assertRaises(OverflowError, setattr, c, attr, -int_max-2) + self.assertRaises((OverflowError, ValueError), setattr, c, attr, + int_max+1) + self.assertRaises((OverflowError, ValueError), setattr, c, attr, + -int_max-2) if sys.platform != 'win32': - self.assertRaises(ValueError, setattr, c, attr, int_max) - self.assertRaises(ValueError, setattr, c, attr, -int_max-1) + self.assertRaises((OverflowError, ValueError), setattr, c, attr, int_max) + self.assertRaises((OverflowError, ValueError), setattr, c, attr, -int_max-1) # OverflowError: _unsafe_setprec, _unsafe_setemin, _unsafe_setemax if C.MAX_PREC == 425000000: @@ -4830,8 +4833,8 @@ self.assertRaises(ValueError, setattr, c, attr, 2) self.assertRaises(TypeError, setattr, c, attr, [1,2,3]) if HAVE_CONFIG_64: - self.assertRaises(ValueError, setattr, c, attr, 2**32) - self.assertRaises(ValueError, setattr, c, attr, 2**32+1) + self.assertRaises((ValueError, OverflowError), setattr, c, attr, 2**32) + self.assertRaises((ValueError, OverflowError), setattr, c, attr, 2**32+1) # Invalid local context self.assertRaises(TypeError, exec, 'with localcontext("xyz"): pass', @@ -4845,6 +4848,8 @@ self.assertRaises(TypeError, setcontext, "xyz") setcontext(saved_context) + # pypy does not keep interned strings + @cpython_only def test_rounding_strings_interned(self): self.assertIs(C.ROUND_UP, P.ROUND_UP) @@ -5371,6 +5376,7 @@ x = (1, (0, 1), "N") self.assertEqual(str(Decimal(x)), '-sNaN1') + @cpython_only def test_sizeof(self): Decimal = C.Decimal HAVE_CONFIG_64 = (C.MAX_PREC > 425000000) diff --git a/lib_pypy/_decimal.py b/lib_pypy/_decimal.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_decimal.py @@ -0,0 +1,1900 @@ +# Implementation of the "decimal" module, based on libmpdec library. + +from cffi import FFI as _FFI +import collections as _collections +import math as _math +import numbers as _numbers +import sys as _sys + +# Compatibility with the C version +HAVE_THREADS = True +if _sys.maxsize == 2**63-1: + MAX_PREC = 999999999999999999 + MAX_EMAX = 999999999999999999 + MIN_EMIN = -999999999999999999 +else: + MAX_PREC = 425000000 + MAX_EMAX = 425000000 + MIN_EMIN = -425000000 + +MIN_ETINY = MIN_EMIN - (MAX_PREC-1) + +# Errors + +class DecimalException(ArithmeticError): + __module__ = 'decimal' + def handle(self, context, *args): + pass + +class Clamped(DecimalException): + __module__ = 'decimal' + +class InvalidOperation(DecimalException): + __module__ = 'decimal' + def handle(self, context, *args): + if args: + ans = _dec_from_triple(args[0]._sign, args[0]._int, 'n', True) + return ans._fix_nan(context) + return _NaN + +class ConversionSyntax(InvalidOperation): + __module__ = 'decimal' + def handle(self, context, *args): + return _NaN + +class DivisionByZero(DecimalException, ZeroDivisionError): + __module__ = 'decimal' + def handle(self, context, sign, *args): + return _SignedInfinity[sign] + +class DivisionImpossible(InvalidOperation): + __module__ = 'decimal' + def handle(self, context, *args): + return _NaN + +class DivisionUndefined(InvalidOperation, ZeroDivisionError): + __module__ = 'decimal' + def handle(self, context, *args): + return _NaN + +class Inexact(DecimalException): + __module__ = 'decimal' + +class InvalidContext(InvalidOperation): + __module__ = 'decimal' + def handle(self, context, *args): + return _NaN + +class Rounded(DecimalException): + __module__ = 'decimal' + +class Subnormal(DecimalException): + __module__ = 'decimal' + +class Overflow(Inexact, Rounded): + __module__ = 'decimal' + def handle(self, context, sign, *args): + if context.rounding in (ROUND_HALF_UP, ROUND_HALF_EVEN, + ROUND_HALF_DOWN, ROUND_UP): + return _SignedInfinity[sign] + if sign == 0: + if context.rounding == ROUND_CEILING: + return _SignedInfinity[sign] + return _dec_from_triple(sign, '9'*context.prec, + context.Emax-context.prec+1) + if sign == 1: + if context.rounding == ROUND_FLOOR: + return _SignedInfinity[sign] + return _dec_from_triple(sign, '9'*context.prec, + context.Emax-context.prec+1) + +class Underflow(Inexact, Rounded, Subnormal): + __module__ = 'decimal' + +class FloatOperation(DecimalException, TypeError): + __module__ = 'decimal' + +# Bindings to the libmpdec library + +_ffi = _FFI() +_ffi.cdef(""" +typedef size_t mpd_size_t; /* unsigned size type */ +typedef ssize_t mpd_ssize_t; /* signed size type */ +typedef size_t mpd_uint_t; +#define MPD_SIZE_MAX ... +#define MPD_SSIZE_MIN ... +#define MPD_SSIZE_MAX ... + +const char *mpd_version(void); +void mpd_free(void *ptr); + +typedef struct mpd_context_t { + mpd_ssize_t prec; /* precision */ + mpd_ssize_t emax; /* max positive exp */ + mpd_ssize_t emin; /* min negative exp */ + uint32_t traps; /* status events that should be trapped */ + uint32_t status; /* status flags */ + uint32_t newtrap; /* set by mpd_addstatus_raise() */ + int round; /* rounding mode */ + int clamp; /* clamp mode */ + int allcr; /* all functions correctly rounded */ +} mpd_context_t; + +enum { + MPD_ROUND_UP, /* round away from 0 */ + MPD_ROUND_DOWN, /* round toward 0 (truncate) */ + MPD_ROUND_CEILING, /* round toward +infinity */ + MPD_ROUND_FLOOR, /* round toward -infinity */ + MPD_ROUND_HALF_UP, /* 0.5 is rounded up */ + MPD_ROUND_HALF_DOWN, /* 0.5 is rounded down */ + MPD_ROUND_HALF_EVEN, /* 0.5 is rounded to even */ + MPD_ROUND_05UP, /* round zero or five away from 0 */ + MPD_ROUND_TRUNC, /* truncate, but set infinity */ + MPD_ROUND_GUARD +}; + +#define MPD_Clamped ... +#define MPD_Conversion_syntax ... +#define MPD_Division_by_zero ... +#define MPD_Division_impossible ... +#define MPD_Division_undefined ... +#define MPD_Fpu_error ... +#define MPD_Inexact ... +#define MPD_Invalid_context ... +#define MPD_Invalid_operation ... +#define MPD_Malloc_error ... +#define MPD_Not_implemented ... +#define MPD_Overflow ... +#define MPD_Rounded ... +#define MPD_Subnormal ... +#define MPD_Underflow ... +#define MPD_Max_status ... +/* Conditions that result in an IEEE 754 exception */ +#define MPD_IEEE_Invalid_operation ... +/* Errors that require the result of an operation to be set to NaN */ +#define MPD_Errors ... + + + +void mpd_maxcontext(mpd_context_t *ctx); +int mpd_qsetprec(mpd_context_t *ctx, mpd_ssize_t prec); +int mpd_qsetemax(mpd_context_t *ctx, mpd_ssize_t emax); +int mpd_qsetemin(mpd_context_t *ctx, mpd_ssize_t emin); +int mpd_qsetround(mpd_context_t *ctx, int newround); +int mpd_qsettraps(mpd_context_t *ctx, uint32_t flags); +int mpd_qsetstatus(mpd_context_t *ctx, uint32_t flags); +int mpd_qsetclamp(mpd_context_t *ctx, int c); + + + + +typedef struct mpd_t { + uint8_t flags; + mpd_ssize_t exp; + mpd_ssize_t digits; + mpd_ssize_t len; + mpd_ssize_t alloc; + mpd_uint_t *data; +} mpd_t; + +#define MPD_POS ... +#define MPD_NEG ... +#define MPD_INF ... +#define MPD_NAN ... +#define MPD_SNAN ... +#define MPD_SPECIAL ... +#define MPD_STATIC ... +#define MPD_STATIC_DATA ... +#define MPD_SHARED_DATA ... +#define MPD_CONST_DATA ... +#define MPD_DATAFLAGS ... + + +mpd_t *mpd_qnew(void); +void mpd_del(mpd_t *dec); + + +/* Operations */ +void mpd_qabs(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); +void mpd_qplus(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); +void mpd_qminus(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); +void mpd_qsqrt(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); +void mpd_qexp(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); +void mpd_qln(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); +void mpd_qlog10(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); +void mpd_qlogb(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); +void mpd_qinvert(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); + +void mpd_qmax(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); +void mpd_qmax_mag(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); +void mpd_qmin(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); +void mpd_qmin_mag(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); + +void mpd_qadd(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); +void mpd_qsub(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); +void mpd_qmul(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); +void mpd_qdiv(mpd_t *q, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); +void mpd_qdivint(mpd_t *q, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); +void mpd_qfma(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_t *c, const mpd_context_t *ctx, uint32_t *status); +void mpd_qrem(mpd_t *r, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); +void mpd_qrem_near(mpd_t *r, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); +void mpd_qpow(mpd_t *result, const mpd_t *base, const mpd_t *exp, const mpd_context_t *ctx, uint32_t *status); +void mpd_qpowmod(mpd_t *result, const mpd_t *base, const mpd_t *exp, const mpd_t *mod, const mpd_context_t *ctx, uint32_t *status); +int mpd_qcopy_sign(mpd_t *result, const mpd_t *a, const mpd_t *b, uint32_t *status); +int mpd_qcopy_abs(mpd_t *result, const mpd_t *a, uint32_t *status); +int mpd_qcopy_negate(mpd_t *result, const mpd_t *a, uint32_t *status); +void mpd_qdivmod(mpd_t *q, mpd_t *r, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); +void mpd_qand(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); +void mpd_qor(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); +void mpd_qxor(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); +int mpd_same_quantum(const mpd_t *a, const mpd_t *b); + +void mpd_qround_to_intx(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); +void mpd_qround_to_int(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); +int mpd_qcopy(mpd_t *result, const mpd_t *a, uint32_t *status); + +int mpd_qcmp(const mpd_t *a, const mpd_t *b, uint32_t *status); +int mpd_qcompare(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); +int mpd_qcompare_signal(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); +int mpd_compare_total(mpd_t *result, const mpd_t *a, const mpd_t *b); +int mpd_compare_total_mag(mpd_t *result, const mpd_t *a, const mpd_t *b); +void mpd_qnext_toward(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); +void mpd_qnext_minus(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); +void mpd_qnext_plus(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); +void mpd_qquantize(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); + +void mpd_qrotate(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); +void mpd_qscaleb(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); +void mpd_qshift(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); +void mpd_qreduce(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); + +/* Get attributes */ +uint8_t mpd_sign(const mpd_t *dec); +int mpd_isnegative(const mpd_t *dec); +int mpd_ispositive(const mpd_t *dec); +int mpd_iszero(const mpd_t *dec); +int mpd_isfinite(const mpd_t *dec); +int mpd_isinfinite(const mpd_t *dec); +int mpd_issigned(const mpd_t *dec); +int mpd_isnan(const mpd_t *dec); +int mpd_issnan(const mpd_t *dec); +int mpd_isspecial(const mpd_t *dec); +int mpd_isqnan(const mpd_t *dec); +int mpd_isnormal(const mpd_t *dec, const mpd_context_t *ctx); +int mpd_issubnormal(const mpd_t *dec, const mpd_context_t *ctx); +mpd_ssize_t mpd_adjexp(const mpd_t *dec); +mpd_ssize_t mpd_etiny(const mpd_context_t *ctx); +mpd_ssize_t mpd_etop(const mpd_context_t *ctx); + +mpd_t *mpd_qncopy(const mpd_t *a); + +/* Set attributes */ +void mpd_set_sign(mpd_t *result, uint8_t sign); +void mpd_set_positive(mpd_t *result); +void mpd_clear_flags(mpd_t *result); +void mpd_seterror(mpd_t *result, uint32_t flags, uint32_t *status); +void mpd_setspecial(mpd_t *dec, uint8_t sign, uint8_t type); + +/* I/O */ +void mpd_qimport_u16(mpd_t *result, const uint16_t *srcdata, size_t srclen, + uint8_t srcsign, uint32_t srcbase, + const mpd_context_t *ctx, uint32_t *status); +size_t mpd_qexport_u16(uint16_t **rdata, size_t rlen, uint32_t base, + const mpd_t *src, uint32_t *status); +void mpd_qset_string(mpd_t *dec, const char *s, const mpd_context_t *ctx, uint32_t *status); +void mpd_qset_uint(mpd_t *result, mpd_uint_t a, const mpd_context_t *ctx, uint32_t *status); +void mpd_qset_ssize(mpd_t *result, mpd_ssize_t a, const mpd_context_t *ctx, uint32_t *status); +void mpd_qsset_ssize(mpd_t *result, mpd_ssize_t a, const mpd_context_t *ctx, uint32_t *status); +mpd_ssize_t mpd_qget_ssize(const mpd_t *dec, uint32_t *status); +int mpd_lsnprint_signals(char *dest, int nmemb, uint32_t flags, const char *signal_string[]); +#define MPD_MAX_SIGNAL_LIST ... +const char *dec_signal_string[]; + +void mpd_qfinalize(mpd_t *result, const mpd_context_t *ctx, uint32_t *status); +const char *mpd_class(const mpd_t *a, const mpd_context_t *ctx); + +/* format specification */ +typedef struct mpd_spec_t { + mpd_ssize_t min_width; /* minimum field width */ + mpd_ssize_t prec; /* fraction digits or significant digits */ + char type; /* conversion specifier */ + char align; /* alignment */ + char sign; /* sign printing/alignment */ + char fill[5]; /* fill character */ + const char *dot; /* decimal point */ + const char *sep; /* thousands separator */ + const char *grouping; /* grouping of digits */ +} mpd_spec_t; + +char *mpd_to_sci(const mpd_t *dec, int fmt); +char *mpd_to_eng(const mpd_t *dec, int fmt); +int mpd_parse_fmt_str(mpd_spec_t *spec, const char *fmt, int caps); +int mpd_validate_lconv(mpd_spec_t *spec); +char *mpd_qformat_spec(const mpd_t *dec, const mpd_spec_t *spec, const mpd_context_t *ctx, uint32_t *status); + +""") + +import os + +_libdir = os.path.join(os.path.dirname(__file__), '_libmpdec') +_mpdec = _ffi.verify( + """ +#include "mpdecimal.h" + +const char *dec_signal_string[MPD_NUM_FLAGS] = { + "Clamped", + "InvalidOperation", + "DivisionByZero", + "InvalidOperation", + "InvalidOperation", + "InvalidOperation", + "Inexact", + "InvalidOperation", + "InvalidOperation", + "InvalidOperation", + "FloatOperation", + "Overflow", + "Rounded", + "Subnormal", + "Underflow", +}; +""", + sources=[os.path.join(_libdir, 'mpdecimal.c'), + os.path.join(_libdir, 'basearith.c'), + os.path.join(_libdir, 'convolute.c'), + os.path.join(_libdir, 'constants.c'), + os.path.join(_libdir, 'context.c'), + os.path.join(_libdir, 'io.c'), + os.path.join(_libdir, 'fourstep.c'), + os.path.join(_libdir, 'sixstep.c'), + os.path.join(_libdir, 'transpose.c'), + os.path.join(_libdir, 'difradix2.c'), + os.path.join(_libdir, 'numbertheory.c'), + os.path.join(_libdir, 'fnt.c'), + os.path.join(_libdir, 'crt.c'), + os.path.join(_libdir, 'memory.c'), + ], + include_dirs=[_libdir], + extra_compile_args=[ + "-DANSI", + "-DHAVE_STDINT_H", + "-DHAVE_INTTYPES_H", + "-DCONFIG_64" if _sys.maxsize > 1 << 32 else "-DCONFIG_32", + ], +) + +del os + +_mpdec.MPD_Float_operation = _mpdec.MPD_Not_implemented + +__version__ = "1.70" +__libmpdec_version__ = _ffi.string(_mpdec.mpd_version()) + +# Default context + +import threading +local = threading.local() + +def getcontext(*, _local=local): + """Returns this thread's context. + + If this thread does not yet have a context, returns + a new context and sets this thread's context. + New contexts are copies of DefaultContext. + """ + try: + return _local.__decimal_context__ + except AttributeError: + context = Context() + _local.__decimal_context__ = context + return context + +def _getcontext(context=None): + if context is None: + return getcontext() + if not isinstance(context, Context): + raise TypeError + return context + +def setcontext(context, *, _local=local): + """Set this thread's context to context.""" + if context in (DefaultContext, BasicContext, ExtendedContext): + context = context.copy() + context.clear_flags() + if not isinstance(context, Context): + raise TypeError + _local.__decimal_context__ = context + + +del local, threading + +def localcontext(ctx=None): + """Return a context manager for a copy of the supplied context. + """ + return _ContextManager(_getcontext(ctx)) + + +from collections import namedtuple as _namedtuple +DecimalTuple = _namedtuple('DecimalTuple', 'sign digits exponent') + + +# A codecs error handler to handle unicode digits +import codecs as _codecs +import unicodedata as _unicodedata +def _handle_decimaldigits(exc): + res = "" + for c in exc.object[exc.start:exc.end]: + if c.isspace(): + res += ' ' + else: + res += str(_unicodedata.digit(c)) + return res, exc.end +_codecs.register_error('_decimal_encode', _handle_decimaldigits) + + +# Decimal class + +_DEC_MINALLOC = 4 + +class Decimal(object): + __module__ = 'decimal' + + __slots__ = ('_mpd', '_data') + + def __new__(cls, value="0", context=None): + return cls._from_object(value, context, exact=True) + + @classmethod + def _new_empty(cls): + self = object.__new__(cls) + self._mpd = mpd = _ffi.new("struct mpd_t*") + self._data = _ffi.new("mpd_uint_t[]", _DEC_MINALLOC) + mpd.flags = _mpdec.MPD_STATIC | _mpdec.MPD_STATIC_DATA + mpd.alloc = _DEC_MINALLOC + mpd.exp = 0 + mpd.digits = 0 + mpd.len = 0 + mpd.data = self._data + return self + + def __del__(self): + _mpdec.mpd_del(self._mpd) + + @classmethod + def _from_object(cls, value, context, exact=True): + if isinstance(value, Decimal): + return cls._from_decimal(value, context, exact=exact) + if isinstance(value, str): + return cls._from_str(value, context, exact=exact, strip=exact) + if isinstance(value, int): + return cls._from_int(value, context, exact=exact) + if isinstance(value, (list, tuple)): + return cls._from_tuple(value, context, exact=exact) + if isinstance(value, float): + context = _getcontext(context) + context._add_status(_mpdec.MPD_Float_operation) + return cls._from_float(value, context, exact=exact) + raise TypeError("conversion from %s to Decimal is not supported" % + value.__class__.__name__) + + @classmethod + def _from_decimal(cls, value, context, exact=True): + if exact: + if cls is Decimal and type(value) is Decimal: + return value + self = cls._new_empty() + with _CatchConversions(self._mpd, context, exact) as ( + ctx, status_ptr): + _mpdec.mpd_qcopy(self._mpd, value._mpd, status_ptr) + return self + else: + if (_mpdec.mpd_isnan(value._mpd) and + value._mpd.digits > (context._ctx.prec - context._ctx.clamp)): + # Special case: too many NaN payload digits + context._add_status(_mpdec.MPD_Conversion_syntax) + self = cls._new_empty() + _mpdec.mpd_setspecial(self._mpd, _mpdec.MPD_POS, _mpdec.MPD_NAN) + return self + else: + self = cls._new_empty() + with _CatchStatus(context) as (ctx, status_ptr): + _mpdec.mpd_qcopy(self._mpd, value._mpd, status_ptr) + _mpdec.mpd_qfinalize(self._mpd, ctx, status_ptr) + return self + + @classmethod + def _from_str(cls, value, context, exact=True, strip=True): + s = value.encode('ascii', '_decimal_encode') + if b'\0' in s: + s = b'' # empty string triggers ConversionSyntax. + if strip: + s = s.strip() + return cls._from_bytes(s, context, exact=exact) + + @classmethod + def _from_bytes(cls, value, context, exact=True): + self = cls._new_empty() + with _CatchConversions(self._mpd, context, exact) as (ctx, status_ptr): + _mpdec.mpd_qset_string(self._mpd, value, ctx, status_ptr) + return self + + @classmethod + def _from_int(cls, value, context, exact=True): + self = cls._new_empty() + with _CatchConversions(self._mpd, context, exact) as (ctx, status_ptr): + size = (((value|1).bit_length() + 15) // 16) + 5 + if value < 0: + value = -value + sign = _mpdec.MPD_NEG + else: + sign = _mpdec.MPD_POS + array = value.to_bytes(2*size, byteorder='little', signed=False) + digits = _ffi.new("uint8_t[]", array) + _mpdec.mpd_qimport_u16( + self._mpd, _ffi.cast("uint16_t*", digits), + size, sign, 0x10000, ctx, status_ptr) + return self + + @classmethod + def _from_tuple(cls, value, context, exact=True): + sign, digits, exponent = value + + # Make a bytes string representation of a DecimalTuple + builder = [] + + # sign + if not isinstance(sign, int) or sign not in (0, 1): + raise ValueError("sign must be an integer with the value 0 or 1") + builder.append(b'-' if sign else b'+') + + # exponent or encoding for a special number + is_infinite = False + is_special = False + if isinstance(exponent, str): + # special + is_special = True + if exponent == 'F': + builder.append(b'Inf') + is_infinite = True + elif exponent == 'n': + builder.append(b'Nan') + elif exponent == 'N': + builder.append(b'sNan') + else: + raise ValueError("string argument in the third position " + "must be 'F', 'n' or 'N'") + exponent = 0 + else: + if not isinstance(exponent, int): + raise ValueError("exponent must be an integer") + if not -_sys.maxsize-1 <= exponent <= _sys.maxsize: + # Compatibility with CPython + raise OverflowError() + + # coefficients + if not digits and not is_special: + # empty tuple: zero coefficient, except for special numbers + builder.append(b'0') + for digit in digits: + if not isinstance(digit, int) or not 0 <= digit <= 9: + raise ValueError("coefficient must be a tuple of digits") + if is_infinite: + # accept but ignore any well-formed coefficient for + # compatibility with decimal.py + continue + builder.append(bytes([ord('0') + digit])) + + if not is_special: + builder.append(b'E') + builder.append(str(exponent).encode()) + + return cls._from_bytes(b''.join(builder), context, exact=exact) + + @classmethod + def from_float(cls, value): + return cls._from_float(value, getcontext(), exact=True) + + @classmethod + def _from_float(cls, value, context, exact=True): + if isinstance(value, int): + return cls._from_int(value, context, exact=exact) + sign = 0 if _math.copysign(1.0, value) == 1.0 else 1 + + if _math.isnan(value): + self = cls._new_empty() + # decimal.py calls repr(float(+-nan)), which always gives a + # positive result. + _mpdec.mpd_setspecial(self._mpd, _mpdec.MPD_POS, _mpdec.MPD_NAN) + return self + if _math.isinf(value): + self = cls._new_empty() + _mpdec.mpd_setspecial(self._mpd, sign, _mpdec.MPD_INF) + return self + + # float as integer ratio: numerator/denominator + num, den = abs(value).as_integer_ratio() + k = den.bit_length() - 1 + + self = cls._from_int(num, context, exact=True) + + # Compute num * 5**k + d1 = _mpdec.mpd_qnew() + if not d1: + raise MemoryError() + try: + d2 = _mpdec.mpd_qnew() + if not d2: + raise MemoryError() + try: + with _CatchConversions(self._mpd, context, exact=True) as ( + ctx, status_ptr): + _mpdec.mpd_qset_uint(d1, 5, ctx, status_ptr) + _mpdec.mpd_qset_ssize(d2, k, ctx, status_ptr) + _mpdec.mpd_qpow(d1, d1, d2, ctx, status_ptr) + finally: + _mpdec.mpd_del(d2) + with _CatchConversions(self._mpd, context, exact=True) as ( + ctx, status_ptr): + _mpdec.mpd_qmul(self._mpd, self._mpd, d1, ctx, status_ptr) + finally: + _mpdec.mpd_del(d1) + + # result = +- n * 5**k * 10**-k + _mpdec.mpd_set_sign(self._mpd, sign) + self._mpd.exp = - k + + if not exact: + with _CatchStatus(context) as (ctx, status_ptr): + _mpdec.mpd_qfinalize(self._mpd, ctx, status_ptr) + return self + + def __str__(self): + return getcontext().to_sci_string(self) + + def __repr__(self): + context = getcontext() + output = _mpdec.mpd_to_sci(self._mpd, context._capitals) + if not output: + raise MemoryError + try: + result = _ffi.string(output) + finally: + _mpdec.mpd_free(output) + return "Decimal('%s')" % result.decode() + + def as_tuple(self): + "Return the DecimalTuple representation of a Decimal" + mpd = self._mpd + sign = _mpdec.mpd_sign(mpd) + if _mpdec.mpd_isinfinite(mpd): + expt = "F" + # decimal.py has non-compliant infinity payloads. + coeff = (0,) + else: + if _mpdec.mpd_isnan(mpd): + if _mpdec.mpd_issnan(mpd): + expt = "N" + else: + expt = "n" + else: + expt = mpd.exp + + if mpd.len > 0: + # coefficient is defined + + # make an integer + # XXX this should be done in C... + x = _mpdec.mpd_qncopy(mpd) + if not x: + raise MemoryError + try: + x.exp = 0 + # clear NaN and sign + _mpdec.mpd_clear_flags(x) + intstring = _mpdec.mpd_to_sci(x, 1) + finally: + _mpdec.mpd_del(x) + if not intstring: + raise MemoryError + try: + digits = _ffi.string(intstring) + finally: + _mpdec.mpd_free(intstring) + coeff = tuple(d - ord('0') for d in digits) + else: + coeff = () + + return DecimalTuple(sign, coeff, expt) + + def _convert_for_comparison(self, other, op): + if isinstance(other, Decimal): + return self, other + + context = getcontext() + if isinstance(other, int): + other = Decimal._from_int(other, context) + elif isinstance(other, float): + if op not in ('eq', 'ne'): + # Add status, and maybe raise + context._add_status(_mpdec.MPD_Float_operation) + else: + # Add status, but don't raise + context._ctx.status |= _mpdec.MPD_Float_operation + other = Decimal._from_float(other, context) + elif isinstance(other, complex): + if op not in ('eq', 'ne'): + return NotImplemented, NotImplemented + if other.imag != 0.0: + return NotImplemented, NotImplemented + # Add status, but don't raise + context._ctx.status |= _mpdec.MPD_Float_operation + other = Decimal._from_float(other.real, context) + elif isinstance(other, _numbers.Rational): + numerator = Decimal._from_int(other.numerator, context) + if not _mpdec.mpd_isspecial(self._mpd): + # multiplied = self * other.denominator + # + # Prevent Overflow in the following multiplication. + # The result of the multiplication is + # only used in mpd_qcmp, which can handle values that + # are technically out of bounds, like (for 32-bit) + # 99999999999999999999...99999999e+425000000. + vv = _mpdec.mpd_qncopy(self._mpd) + if not vv: + raise MemoryError + try: + exp = vv.exp + vv.exp = 0 + multiplied = Decimal._new_empty() + denom = Decimal(other.denominator) + with _CatchStatus(context) as (ctx, status_ptr): + _mpdec.mpd_qmul(multiplied._mpd, vv, denom._mpd, + ctx, status_ptr) + multiplied._mpd.exp = exp + finally: + _mpdec.mpd_del(vv) + + return multiplied, numerator + else: + return self, numerator + else: + return NotImplemented, NotImplemented + return self, other + + # _PyHASH_10INV is the inverse of 10 modulo the prime _PyHASH_MODULUS + _PyHASH_MODULUS = _sys.hash_info.modulus + _PyHASH_10INV = pow(10, _PyHASH_MODULUS - 2, _PyHASH_MODULUS) + + def __bool__(self): + return not _mpdec.mpd_iszero(self._mpd) + + def __hash__(self): + # In order to make sure that the hash of a Decimal instance + # agrees with the hash of a numerically equal integer, float + # or Fraction, we follow the rules for numeric hashes outlined + # in the documentation. (See library docs, 'Built-in Types'). + mpd = self._mpd + if _mpdec.mpd_isspecial(mpd): + if _mpdec.mpd_issnan(mpd): + raise TypeError("cannot hash a signaling NaN value") + elif _mpdec.mpd_isnan(mpd): + return _sys.hash_info.nan + elif _mpdec.mpd_isnegative(mpd): + return -_sys.hash_info.inf + else: + return _sys.hash_info.inf + + maxctx = _ffi.new("struct mpd_context_t*") + _mpdec.mpd_maxcontext(maxctx) + status_ptr = _ffi.new("uint32_t*") + + # XXX cache these + p = self._new_empty() + _mpdec.mpd_qset_ssize(p._mpd, self._PyHASH_MODULUS, + maxctx, status_ptr) + ten = self._new_empty() + _mpdec.mpd_qset_ssize(ten._mpd, 10, + maxctx, status_ptr) + inv10_p = self._new_empty() + _mpdec.mpd_qset_ssize(inv10_p._mpd, self._PyHASH_10INV, + maxctx, status_ptr) + + tmp = self._new_empty() + exp_hash = self._new_empty() + + if mpd.exp >= 0: + # 10**exp(v) % p + _mpdec.mpd_qsset_ssize(tmp._mpd, mpd.exp, maxctx, status_ptr) + _mpdec.mpd_qpowmod(exp_hash._mpd, ten._mpd, tmp._mpd, p._mpd, + maxctx, status_ptr) + else: + # inv10_p**(-exp(v)) % p + _mpdec.mpd_qsset_ssize(tmp._mpd, -mpd.exp, maxctx, status_ptr) + _mpdec.mpd_qpowmod(exp_hash._mpd, inv10_p._mpd, tmp._mpd, p._mpd, + maxctx, status_ptr) + + # hash = (int(v) * exp_hash) % p + if not _mpdec.mpd_qcopy(tmp._mpd, mpd, status_ptr): + raise MemoryError + + tmp._mpd.exp = 0 + _mpdec.mpd_set_positive(tmp._mpd) + + maxctx.prec = MAX_PREC + 21 + maxctx.emax = MAX_EMAX + 21 + maxctx.emin = MIN_EMIN - 21 + + _mpdec.mpd_qmul(tmp._mpd, tmp._mpd, exp_hash._mpd, maxctx, status_ptr) + _mpdec.mpd_qrem(tmp._mpd, tmp._mpd, p._mpd, maxctx, status_ptr) + + result = _mpdec.mpd_qget_ssize(tmp._mpd, status_ptr) + result = result if _mpdec.mpd_ispositive(mpd) else -result + result = result if result != -1 else -2 + + if status_ptr[0]: + if status_ptr[0] & _mpdec.MPD_Malloc_error: + raise MemoryError + else: + raise SystemError("Decimal.__hash__") + + return result + + def _cmp(self, other, op): + a, b = self._convert_for_comparison(other, op) + if a is NotImplemented: + return NotImplemented + status_ptr = _ffi.new("uint32_t*") + r = _mpdec.mpd_qcmp(a._mpd, b._mpd, status_ptr) + if r > 1: # INT_MAX + # sNaNs or op={le,ge,lt,gt} always signal + if (_mpdec.mpd_issnan(a._mpd) or + _mpdec.mpd_issnan(b._mpd) or + op not in ('eq', 'ne')): + getcontext()._add_status(status_ptr[0]) + # qNaN comparison with op={eq,ne} or comparison with + # InvalidOperation disabled. + # Arrange to return False. + if op in ('gt', 'ge'): + return -1 + else: + return 1 + return r + + def __eq__(self, other): + r = self._cmp(other, 'eq') + if r is NotImplemented: + return NotImplemented + return r == 0 + + def __ne__(self, other): + r = self._cmp(other, 'ne') + if r is NotImplemented: + return NotImplemented + return r != 0 + + def __lt__(self, other): + r = self._cmp(other, 'lt') + if r is NotImplemented: + return NotImplemented + return r < 0 + + def __le__(self, other): + r = self._cmp(other, 'le') + if r is NotImplemented: + return NotImplemented + return r <= 0 + + def __gt__(self, other): + r = self._cmp(other, 'gt') + if r is NotImplemented: + return NotImplemented + return r > 0 + + def __ge__(self, other): + r = self._cmp(other, 'ge') + if r is NotImplemented: + return NotImplemented + return r >= 0 + + # operations + def _make_unary_operation(name, ctxop_name=None): + ctxop_name = ctxop_name or name + if name.startswith('__'): + def method(self): + return getattr(getcontext(), ctxop_name)(self) + else: + # Allow optional context + def method(self, context=None): + context = _getcontext(context) + return getattr(context, ctxop_name)(self) + method.__name__ = name + return method + + def _make_unary_operation_noctx(name, ctxop_name=None): + ctxop_name = ctxop_name or name + def method(self): + return getattr(getcontext(), ctxop_name)(self) + method.__name__ = name + return method + + def _make_binary_operation(name, ctxop_name=None): + ctxop_name = ctxop_name or name + if name.startswith('__'): + def method(self, other): + return getattr(getcontext(), ctxop_name)( + self, other, strict=False) + else: + def method(self, other, context=None): + context = _getcontext(context) + return getattr(context, ctxop_name)( + self, other) + method.__name__ = name + return method + + def _make_binary_roperation(name, ctxop_name): + def method(self, other): + return getattr(getcontext(), ctxop_name)(other, self, strict=False) + method.__name__ = name + return method + + __abs__ = _make_unary_operation('__abs__', 'abs') + __pos__ = _make_unary_operation('__pos__', 'plus') + __neg__ = _make_unary_operation('__neg__', 'minus') + + __add__ = _make_binary_operation('__add__', 'add') + __sub__ = _make_binary_operation('__sub__', 'subtract') + __mul__ = _make_binary_operation('__mul__', 'multiply') + __floordiv__ = _make_binary_operation('__floordiv__', 'divide_int') + __truediv__ = _make_binary_operation('__truediv__', 'divide') + __mod__ = _make_binary_operation('__mod__', 'remainder') + __divmod__ = _make_binary_operation('__divmod__', 'divmod') + + __radd__ = _make_binary_roperation('__radd__', 'add') + __rsub__ = _make_binary_roperation('__rsub__', 'subtract') + __rmul__ = _make_binary_roperation('__rmul__', 'multiply') + __rfloordiv__ = _make_binary_roperation('__rfloordiv__', 'divide_int') + __rtruediv__ = _make_binary_roperation('__rtruediv__', 'divide') + __rmod__ = _make_binary_roperation('__rmod__', 'remainder') + __rdivmod__ = _make_binary_roperation('__rdivmod__', 'divmod') + + def __pow__(self, other, modulo=None): + return getcontext().power(self, other, modulo, strict=False) + def __rpow__(self, other): + return getcontext().power(other, self, strict=False) + + copy_sign = _make_binary_operation('copy_sign') + copy_abs = _make_unary_operation_noctx('copy_abs') + copy_negate = _make_unary_operation_noctx('copy_negate') + + sqrt = _make_unary_operation('sqrt') + exp = _make_unary_operation('exp') + ln = _make_unary_operation('ln') + log10 = _make_unary_operation('log10') + logb = _make_unary_operation('logb') + logical_invert = _make_unary_operation('logical_invert') + normalize = _make_unary_operation('normalize') + + compare = _make_binary_operation('compare') + compare_signal = _make_binary_operation('compare_signal') + compare_total = _make_binary_operation('compare') + compare_total_mag = _make_binary_operation('compare') + logical_and = _make_binary_operation('logical_and') + logical_or = _make_binary_operation('logical_or') + logical_xor = _make_binary_operation('logical_xor') + max = _make_binary_operation('max') + max_mag = _make_binary_operation('max_mag') + min = _make_binary_operation('min') + min_mag = _make_binary_operation('min_mag') + next_minus = _make_unary_operation('next_minus') + next_plus = _make_unary_operation('next_plus') + next_toward = _make_binary_operation('next_toward') + remainder_near = _make_binary_operation('remainder_near') + rotate = _make_binary_operation('rotate') + same_quantum = _make_binary_operation('same_quantum') + scaleb = _make_binary_operation('scaleb') + shift = _make_binary_operation('shift') + + is_normal = _make_unary_operation('is_normal') + is_subnormal = _make_unary_operation('is_subnormal') + is_signed = _make_unary_operation_noctx('is_signed') + is_zero = _make_unary_operation_noctx('is_zero') + is_nan = _make_unary_operation_noctx('is_nan') + is_snan = _make_unary_operation_noctx('is_snan') + is_qnan = _make_unary_operation_noctx('is_qnan') + is_finite = _make_unary_operation_noctx('is_finite') + is_infinite = _make_unary_operation_noctx('is_infinite') + number_class = _make_unary_operation('number_class') + + to_eng_string = _make_unary_operation('to_eng_string') + + def fma(self, other, third, context=None): + context = _getcontext(context) + return context.fma(self, other, third) + + def _to_int(self, rounding): + mpd = self._mpd + if _mpdec.mpd_isspecial(mpd): + if _mpdec.mpd_isnan(mpd): + raise ValueError("cannot convert NaN to integer") + else: + raise OverflowError("cannot convert Infinity to integer") + + x = Decimal._new_empty() + context = getcontext() + tempctx = context.copy() + tempctx._ctx.round = rounding + with _CatchStatus(context) as (ctx, status_ptr): + # We round with the temporary context, but set status and + # raise errors on the global one. + _mpdec.mpd_qround_to_int(x._mpd, mpd, tempctx._ctx, status_ptr) + + # XXX mpd_qexport_u64 would be faster... + digits_ptr = _ffi.new("uint16_t**") + n = _mpdec.mpd_qexport_u16(digits_ptr, 0, 0x10000, + x._mpd, status_ptr) + if n == _mpdec.MPD_SIZE_MAX: + raise MemoryError + try: + s = _ffi.buffer(digits_ptr[0], n * 2)[:] + finally: + _mpdec.mpd_free(digits_ptr[0]) + result = int.from_bytes(s, 'little', signed=False) + if _mpdec.mpd_isnegative(x._mpd) and not _mpdec.mpd_iszero(x._mpd): + result = -result + return result + + def __int__(self): + return self._to_int(_mpdec.MPD_ROUND_DOWN) + + __trunc__ = __int__ + + def __floor__(self): + return self._to_int(_mpdec.MPD_ROUND_FLOOR) + + def __ceil__(self): + return self._to_int(_mpdec.MPD_ROUND_CEILING) + + def to_integral(self, rounding=None, context=None): + context = _getcontext(context) + workctx = context.copy() + if rounding is not None: + workctx.rounding = rounding + result = Decimal._new_empty() + with _CatchStatus(context) as (ctx, status_ptr): + # We round with the temporary context, but set status and + # raise errors on the global one. + _mpdec.mpd_qround_to_int(result._mpd, self._mpd, + workctx._ctx, status_ptr) + return result + + to_integral_value = to_integral + + def to_integral_exact(self, rounding=None, context=None): + context = _getcontext(context) + workctx = context.copy() + if rounding is not None: + workctx.rounding = rounding + result = Decimal._new_empty() + with _CatchStatus(context) as (ctx, status_ptr): + # We round with the temporary context, but set status and + # raise errors on the global one. + _mpdec.mpd_qround_to_intx(result._mpd, self._mpd, + workctx._ctx, status_ptr) + return result + + def quantize(self, exp, rounding=None, context=None): + context = _getcontext(context) + exp = context._convert_unaryop(exp) + workctx = context.copy() + if rounding is not None: + workctx.rounding = rounding + result = Decimal._new_empty() + with _CatchStatus(context) as (ctx, status_ptr): + # We round with the temporary context, but set status and + # raise errors on the global one. + _mpdec.mpd_qquantize(result._mpd, self._mpd, exp._mpd, + workctx._ctx, status_ptr) + return result + + def __round__(self, x=None): + if x is None: + return self._to_int(_mpdec.MPD_ROUND_HALF_EVEN) + result = Decimal._new_empty() + context = getcontext() + q = Decimal._from_int(1, context) + if x == _mpdec.MPD_SSIZE_MIN: + q._mpd.exp = _mpdec.MPD_SSIZE_MAX + elif x == -_mpdec.MPD_SSIZE_MIN: + raise OverflowError # For compatibility with CPython. + else: + q._mpd.exp = -x + with _CatchStatus(context) as (ctx, status_ptr): + _mpdec.mpd_qquantize(result._mpd, self._mpd, q._mpd, + ctx, status_ptr) + return result + + def __float__(self): + if _mpdec.mpd_isnan(self._mpd): + if _mpdec.mpd_issnan(self._mpd): + raise ValueError("cannot convert signaling NaN to float") + if _mpdec.mpd_isnegative(self._mpd): + return float("-nan") + else: + return float("nan") + else: + return float(str(self)) + + def radix(self): + return Decimal(10) + + def canonical(self): + return self + + def is_canonical(self): + return True + + def adjusted(self): + if _mpdec.mpd_isspecial(self._mpd): + return 0 + return _mpdec.mpd_adjexp(self._mpd) + + @property + def real(self): + return self + + @property + def imag(self): + return Decimal(0) + + def conjugate(self): + return self + + def __complex__(self): + return complex(float(self)) + + def __copy__(self): + return self + + def __deepcopy__(self, memo=None): + return self + + def __reduce__(self): + return (type(self), (str(self),)) + + def __format__(self, specifier, override=None): + if not isinstance(specifier, str): + raise TypeError + fmt = specifier.encode('utf-8') + context = getcontext() + + replace_fillchar = False + if fmt and fmt[0] == 0: + # NUL fill character: must be replaced with a valid UTF-8 char + # before calling mpd_parse_fmt_str(). + replace_fillchar = True + fmt = b'_' + fmt[1:] + + spec = _ffi.new("mpd_spec_t*") + if not _mpdec.mpd_parse_fmt_str(spec, fmt, context._capitals): + raise ValueError("invalid format string") + if replace_fillchar: + # In order to avoid clobbering parts of UTF-8 thousands + # separators or decimal points when the substitution is + # reversed later, the actual placeholder must be an invalid + # UTF-8 byte. + spec.fill = b'\xff\x00' + + if override: + # Values for decimal_point, thousands_sep and grouping can + # be explicitly specified in the override dict. These values + # take precedence over the values obtained from localeconv() + # in mpd_parse_fmt_str(). The feature is not documented and + # is only used in test_decimal. + try: + dot = _ffi.new("char[]", override['decimal_point'].encode()) + except KeyError: + pass + else: + spec.dot = dot + try: + sep = _ffi.new("char[]", override['thousands_sep'].encode()) + except KeyError: + pass + else: + spec.sep = sep + try: + grouping = _ffi.new("char[]", override['grouping'].encode()) + except KeyError: + pass + else: + spec.grouping = grouping + if _mpdec.mpd_validate_lconv(spec) < 0: + raise ValueError("invalid override dict") + + with _CatchStatus(context) as (ctx, status_ptr): + decstring = _mpdec.mpd_qformat_spec( + self._mpd, spec, ctx, status_ptr) + status = status_ptr[0] + if not decstring: + if status & _mpdec.MPD_Malloc_error: + raise MemoryError + else: + raise ValueError("format specification exceeds " + "internal limits of _decimal") + result = _ffi.string(decstring) + if replace_fillchar: + result = result.replace(b'\xff', b'\0') + return result.decode('utf-8') + + +# Register Decimal as a kind of Number (an abstract base class). +# However, do not register it as Real (because Decimals are not +# interoperable with floats). +_numbers.Number.register(Decimal) + +# Context class + +_DEC_DFLT_EMAX = 999999 +_DEC_DFLT_EMIN = -999999 + +# Rounding +_ROUNDINGS = { + 'ROUND_DOWN': _mpdec.MPD_ROUND_DOWN, + 'ROUND_HALF_UP': _mpdec.MPD_ROUND_HALF_UP, + 'ROUND_HALF_EVEN': _mpdec.MPD_ROUND_HALF_EVEN, + 'ROUND_CEILING': _mpdec.MPD_ROUND_CEILING, + 'ROUND_FLOOR': _mpdec.MPD_ROUND_FLOOR, + 'ROUND_UP': _mpdec.MPD_ROUND_UP, + 'ROUND_HALF_DOWN': _mpdec.MPD_ROUND_HALF_DOWN, + 'ROUND_05UP': _mpdec.MPD_ROUND_05UP, +} +for _rounding in _ROUNDINGS: + globals()[_rounding] = _rounding + +_SIGNALS = { + InvalidOperation: _mpdec.MPD_IEEE_Invalid_operation, + FloatOperation: _mpdec.MPD_Float_operation, + DivisionByZero: _mpdec.MPD_Division_by_zero , + Overflow: _mpdec.MPD_Overflow , + Underflow: _mpdec.MPD_Underflow , + Subnormal: _mpdec.MPD_Subnormal , + Inexact: _mpdec.MPD_Inexact , + Rounded: _mpdec.MPD_Rounded, + Clamped: _mpdec.MPD_Clamped, +} + +class _ContextManager(object): + """Context manager class to support localcontext(). + + Sets a copy of the supplied context in __enter__() and restores + the previous decimal context in __exit__() + """ + def __init__(self, new_context): + self.new_context = new_context.copy() + def __enter__(self): + self.saved_context = getcontext() + setcontext(self.new_context) + return self.new_context + def __exit__(self, t, v, tb): + setcontext(self.saved_context) + + +class Context(object): + """Contains the context for a Decimal instance. + + Contains: + prec - precision (for use in rounding, division, square roots..) + rounding - rounding type (how you round) + traps - If traps[exception] = 1, then the exception is + raised when it is caused. Otherwise, a value is + substituted in. + flags - When an exception is caused, flags[exception] is set. + (Whether or not the trap_enabler is set) + Should be reset by user of Decimal instance. + Emin - Minimum exponent + Emax - Maximum exponent + capitals - If 1, 1*10^1 is printed as 1E+1. + If 0, printed as 1e1 + clamp - If 1, change exponents if too high (Default 0) + """ + + __module__ = 'decimal' + + __slots__ = ('_ctx', '_capitals') + + def __new__(cls, *args, **kwargs): + self = object.__new__(cls) + self._ctx = ctx = _ffi.new("struct mpd_context_t*") + # Default context + ctx.prec = 28 + ctx.emax = _DEC_DFLT_EMAX + ctx.emin = _DEC_DFLT_EMIN + ctx.traps = (_mpdec.MPD_IEEE_Invalid_operation| + _mpdec.MPD_Division_by_zero| + _mpdec.MPD_Overflow) + ctx.status = 0 + ctx.newtrap = 0 + ctx.round = _mpdec.MPD_ROUND_HALF_EVEN + ctx.clamp = 0 + ctx.allcr = 1 + + self._capitals = 1 + return self + + def __init__(self, prec=None, rounding=None, Emin=None, Emax=None, + capitals=None, clamp=None, flags=None, traps=None): + ctx = self._ctx + + try: + dc = DefaultContext._ctx + except NameError: + pass + else: + ctx[0] = dc[0] + if prec is not None: + self.prec = prec + if rounding is not None: + self.rounding = rounding + if Emin is not None: + self.Emin = Emin + if Emax is not None: + self.Emax = Emax + if clamp is not None: + self.clamp = clamp + if capitals is not None: + self.capitals = capitals + + if traps is None: + ctx.traps = dc.traps + elif not isinstance(traps, dict): + ctx.traps = 0 + for signal in traps: + ctx.traps |= _SIGNALS[signal] + else: + ctx.traps = 0 + for signal, value in traps.items(): + if value: + ctx.traps |= _SIGNALS[signal] + + if flags is None: + ctx.status = 0 + elif not isinstance(flags, dict): + ctx.status = 0 + for signal in flags: + ctx.status |= _SIGNALS[signal] + else: + for signal, value in flags.items(): + if value: + ctx.status |= _SIGNALS[signal] + + def clear_flags(self): + self._ctx.status = 0 + + def clear_traps(self): + self._ctx.traps = 0 + + @property + def prec(self): + return self._ctx.prec + @prec.setter + def prec(self, value): + if not _mpdec.mpd_qsetprec(self._ctx, value): + raise ValueError("valid range for prec is [1, MAX_PREC]") + + @property + def clamp(self): + return self._ctx.clamp + @clamp.setter + def clamp(self, value): + if not _mpdec.mpd_qsetclamp(self._ctx, value): + raise ValueError("valid values for clamp are 0 or 1") + + @property + def rounding(self): + return next(name + for (name, value) in _ROUNDINGS.items() + if value==self._ctx.round) + @rounding.setter + def rounding(self, value): + if value not in _ROUNDINGS: + raise TypeError( + "valid values for rounding are:\n" + "[ROUND_CEILING, ROUND_FLOOR, ROUND_UP, ROUND_DOWN,\n" + "ROUND_HALF_UP, ROUND_HALF_DOWN, ROUND_HALF_EVEN,\n" + "ROUND_05UP]") + if not _mpdec.mpd_qsetround(self._ctx, _ROUNDINGS[value]): + raise RuntimeError("internal error while setting rounding") + + @property + def Emin(self): + return self._ctx.emin + @Emin.setter + def Emin(self, value): + if not _mpdec.mpd_qsetemin(self._ctx, value): + raise ValueError("valid range for Emin is [MIN_EMIN, 0]") + + @property + def Emax(self): + return self._ctx.emax + @Emax.setter + def Emax(self, value): + if not _mpdec.mpd_qsetemax(self._ctx, value): + raise ValueError("valid range for Emax is [0, MAX_EMAX]") + + @property + def flags(self): + return _SignalDict(self._ctx, 'status') + @flags.setter + def flags(self, value): + if not isinstance(value, _collections.abc.Mapping): + raise TypeError + if len(value) != len(_SIGNALS): + raise KeyError("Invalid signal dict") + for signal, value in value.items(): + if value: + self._ctx.status |= _SIGNALS[signal] + + @property + def traps(self): + return _SignalDict(self._ctx, 'traps') + @traps.setter + def traps(self, value): + if not isinstance(value, _collections.abc.Mapping): + raise TypeError + if len(value) != len(_SIGNALS): + raise KeyError("Invalid signal dict") + for signal, value in value.items(): + if value: + self._ctx.traps |= _SIGNALS[signal] + + @property + def capitals(self): + return self._capitals + @capitals.setter + def capitals(self, value): + if not isinstance(value, int): + raise TypeError + if value not in (0, 1): + raise ValueError("valid values for capitals are 0 or 1") + self._capitals = value + + def __repr__(self): + ctx = self._ctx + return ("Context(prec=%s, rounding=%s, Emin=%s, Emax=%s, " + "capitals=%s, clamp=%s, flags=%s, traps=%s)" % ( + ctx.prec, self.rounding, + ctx.emin, ctx.emax, + self._capitals, ctx.clamp, + self.flags, self.traps)) + + def radix(self): + return Decimal(10) + + def Etiny(self): + return _mpdec.mpd_etiny(self._ctx) + + def Etop(self): + return _mpdec.mpd_etop(self._ctx) + + def is_canonical(self, a): + if not isinstance(a, Decimal): + raise TypeError("is_canonical requires a Decimal as an argument.") + return a.is_canonical() + + def canonical(self, a): + if not isinstance(a, Decimal): + raise TypeError("argument must be a Decimal") + return a + + def copy(self): + other = Context() + other._ctx[0] = self._ctx[0] + other._capitals = self._capitals + return other + + def __copy__(self): + return self.copy() + + def __reduce__(self): + return (type(self), ( + self.prec, self.rounding, self.Emin, self.Emax, + self._capitals, self.clamp, + self.flags._as_list(), + self.traps._as_list())) + + def _add_status(self, status): + self._ctx.status |= status + if self._ctx.status & _mpdec.MPD_Malloc_error: + raise MemoryError() + trapped = self._ctx.traps & status + if trapped: + for exception, flag in _SIGNALS.items(): + if trapped & flag: + raise exception + raise RuntimeError("Invalid error flag", trapped) + + def create_decimal(self, num="0"): + return Decimal._from_object(num, self, exact=False) + + def create_decimal_from_float(self, value): + return Decimal._from_float(value, self, exact=False) + + # operations + def _convert_unaryop(self, a, strict=True): + if isinstance(a, Decimal): + return a + elif isinstance(a, int): + return Decimal._from_int(a, self) + if strict: + raise TypeError("Unable to convert %s to Decimal" % (a,)) + else: + return NotImplemented + + def _convert_binop(self, a, b, strict=True): + a = self._convert_unaryop(a, strict=strict) + b = self._convert_unaryop(b, strict=strict) + if b is NotImplemented: + return b, b + return a, b + + def _make_unary_method(name, mpd_func_name): + mpd_func = getattr(_mpdec, mpd_func_name) + + def method(self, a, *, strict=True): + a = self._convert_unaryop(a, strict=strict) + if a is NotImplemented: + return NotImplemented + res = Decimal._new_empty() + with _CatchStatus(self) as (ctx, status_ptr): + mpd_func(res._mpd, a._mpd, ctx, status_ptr) + return res + method.__name__ = name + return method + + def _make_unary_method_noctx(name, mpd_func_name): + mpd_func = getattr(_mpdec, mpd_func_name) + + def method(self, a, *, strict=True): + a = self._convert_unaryop(a, strict=strict) + if a is NotImplemented: + return NotImplemented + res = Decimal._new_empty() + with _CatchStatus(self) as (ctx, status_ptr): + mpd_func(res._mpd, a._mpd, status_ptr) + return res + method.__name__ = name + return method + + def _make_bool_method(name, mpd_func_name): + mpd_func = getattr(_mpdec, mpd_func_name) + + def method(self, a): + a = self._convert_unaryop(a) + return bool(mpd_func(a._mpd, self._ctx)) + method.__name__ = name + return method + + def _make_bool_method_noctx(name, mpd_func_name): + mpd_func = getattr(_mpdec, mpd_func_name) + + def method(self, a): + a = self._convert_unaryop(a) + return bool(mpd_func(a._mpd)) + method.__name__ = name + return method + + def _make_binary_method(name, mpd_func_name): + mpd_func = getattr(_mpdec, mpd_func_name) + + def method(self, a, b, *, strict=True): + a, b = self._convert_binop(a, b, strict=strict) + if a is NotImplemented: + return NotImplemented + res = Decimal._new_empty() + with _CatchStatus(self) as (ctx, status_ptr): + mpd_func(res._mpd, a._mpd, b._mpd, ctx, status_ptr) + return res + method.__name__ = name + return method + + def _make_binary_bool_method(name, mpd_func_name): + mpd_func = getattr(_mpdec, mpd_func_name) + + def method(self, a, b): + a, b = self._convert_binop(a, b) + return bool(mpd_func(a._mpd, b._mpd)) + method.__name__ = name + return method + + def _make_binary_method_noctx(name, mpd_func_name): + mpd_func = getattr(_mpdec, mpd_func_name) + + def method(self, a, b, *, strict=True): + a, b = self._convert_binop(a, b, strict=strict) + if a is NotImplemented: + return NotImplemented + res = Decimal._new_empty() + with _CatchStatus(self) as (ctx, status_ptr): + mpd_func(res._mpd, a._mpd, b._mpd, status_ptr) + return res + method.__name__ = name + return method + + def _make_binary_method_nostatus(name, mpd_func_name): + mpd_func = getattr(_mpdec, mpd_func_name) + + def method(self, a, b, *, strict=True): + a, b = self._convert_binop(a, b, strict=strict) + if a is NotImplemented: + return NotImplemented + res = Decimal._new_empty() + mpd_func(res._mpd, a._mpd, b._mpd) + return res + method.__name__ = name + return method + + abs = _make_unary_method('abs', 'mpd_qabs') + plus = _make_unary_method('plus', 'mpd_qplus') + minus = _make_unary_method('minus', 'mpd_qminus') + sqrt = _make_unary_method('sqrt', 'mpd_qsqrt') + exp = _make_unary_method('exp', 'mpd_qexp') + ln = _make_unary_method('ln', 'mpd_qln') + log10 = _make_unary_method('log10', 'mpd_qlog10') + logb = _make_unary_method('logb', 'mpd_qlogb') + logical_invert = _make_unary_method('logical_invert', 'mpd_qinvert') + normalize = _make_unary_method('normalize', 'mpd_qreduce') + + add = _make_binary_method('add', 'mpd_qadd') + subtract = _make_binary_method('add', 'mpd_qsub') + multiply = _make_binary_method('multiply', 'mpd_qmul') + divide = _make_binary_method('divide', 'mpd_qdiv') + divide_int = _make_binary_method('divide_int', 'mpd_qdivint') + remainder = _make_binary_method('remainder', 'mpd_qrem') + remainder_near = _make_binary_method('remainder_near', 'mpd_qrem_near') + copy_sign = _make_binary_method_noctx('copy_sign', 'mpd_qcopy_sign') + copy_abs = _make_unary_method_noctx('copy_abs', 'mpd_qcopy_abs') + copy_negate = _make_unary_method_noctx('copy_negate', 'mpd_qcopy_negate') + + compare = _make_binary_method('compare', 'mpd_qcompare') + compare_signal = _make_binary_method('compare_signal', + 'mpd_qcompare_signal') + compare_total = _make_binary_method_nostatus('compare_total', + 'mpd_compare_total') + compare_total_mag = _make_binary_method_nostatus('compare_total_mag', + 'mpd_compare_total_mag') + logical_and = _make_binary_method('logical_and', 'mpd_qand') + logical_or = _make_binary_method('logical_or', 'mpd_qor') + logical_xor = _make_binary_method('logical_xor', 'mpd_qxor') + max = _make_binary_method('max', 'mpd_qmax') + max_mag = _make_binary_method('max_mag', 'mpd_qmax_mag') + min = _make_binary_method('min', 'mpd_qmin') + min_mag = _make_binary_method('min_mag', 'mpd_qmin_mag') + next_minus = _make_unary_method('next_minus', 'mpd_qnext_minus') + next_plus = _make_unary_method('next_plus', 'mpd_qnext_plus') + next_toward = _make_binary_method('next_toward', 'mpd_qnext_toward') + rotate = _make_binary_method('rotate', 'mpd_qrotate') + same_quantum = _make_binary_bool_method('same_quantum', 'mpd_same_quantum') + scaleb = _make_binary_method('scaleb', 'mpd_qscaleb') + shift = _make_binary_method('shift', 'mpd_qshift') + quantize = _make_binary_method('quantize', 'mpd_qquantize') + + is_normal = _make_bool_method('is_normal', 'mpd_isnormal') + is_signed = _make_bool_method_noctx('is_signed', 'mpd_issigned') + is_zero = _make_bool_method_noctx('is_signed', 'mpd_iszero') + is_subnormal = _make_bool_method('is_subnormal', 'mpd_issubnormal') + is_nan = _make_bool_method_noctx('is_qnan', 'mpd_isnan') + is_snan = _make_bool_method_noctx('is_qnan', 'mpd_issnan') + is_qnan = _make_bool_method_noctx('is_qnan', 'mpd_isqnan') + is_finite = _make_bool_method_noctx('is_finite', 'mpd_isfinite') + is_infinite = _make_bool_method_noctx('is_infinite', 'mpd_isinfinite') + + def _apply(self, a): + # Apply the context to the input operand. + a = self._convert_unaryop(a) + result = Decimal._new_empty() + with _CatchStatus(self) as (ctx, status_ptr): + _mpdec.mpd_qcopy(result._mpd, a._mpd, status_ptr) + _mpdec.mpd_qfinalize(result._mpd, ctx, status_ptr) + return result + + def divmod(self, a, b, strict=True): + a, b = self._convert_binop(a, b, strict=strict) + if a is NotImplemented: + return NotImplemented + q = Decimal._new_empty() + r = Decimal._new_empty() + with _CatchStatus(self) as (ctx, status_ptr): + _mpdec.mpd_qdivmod(q._mpd, r._mpd, a._mpd, b._mpd, + ctx, status_ptr) + return q, r + + def power(self, a, b, modulo=None, strict=True): + a, b = self._convert_binop(a, b, strict=strict) + if a is NotImplemented: + return NotImplemented + if modulo is not None: + modulo = self._convert_unaryop(modulo, strict=strict) + if modulo is NotImplemented: + return NotImplemented + res = Decimal._new_empty() + with _CatchStatus(self) as (ctx, status_ptr): + if modulo is not None: + _mpdec.mpd_qpowmod(res._mpd, a._mpd, b._mpd, modulo._mpd, + ctx, status_ptr) + else: + _mpdec.mpd_qpow(res._mpd, a._mpd, b._mpd, + ctx, status_ptr) + return res + + to_integral = _make_unary_method('to_integral', 'mpd_qround_to_int') + to_integral_value = to_integral + to_integral_exact = _make_unary_method('to_integral_exact', + 'mpd_qround_to_intx') + + def fma(self, a, b, c): + a = self._convert_unaryop(a) + b = self._convert_unaryop(b) + c = self._convert_unaryop(c) + res = Decimal._new_empty() + with _CatchStatus(self) as (ctx, status_ptr): + _mpdec.mpd_qfma(res._mpd, a._mpd, b._mpd, c._mpd, + ctx, status_ptr) + return res + + def copy_decimal(self, a): + return self._convert_unaryop(a) + + def number_class(self, a): + a = self._convert_unaryop(a) + cp = _mpdec.mpd_class(a._mpd, self._ctx) + return _ffi.string(cp).decode() + + def to_eng_string(self, a): + a = self._convert_unaryop(a) + output = _mpdec.mpd_to_eng(a._mpd, self._capitals) + if not output: + raise MemoryError + try: + result = _ffi.string(output) + finally: + _mpdec.mpd_free(output) + return result.decode() + + def to_sci_string(self, a): + a = self._convert_unaryop(a) + output = _mpdec.mpd_to_sci(a._mpd, self._capitals) + if not output: + raise MemoryError + try: + result = _ffi.string(output) + finally: + _mpdec.mpd_free(output) + return result.decode() + + +class _SignalDict(_collections.abc.MutableMapping): + + def __init__(self, ctx, attrname): + self.ctx = ctx + self.attrname = attrname + + def __repr__(self): + value = getattr(self.ctx, self.attrname) + buf = _ffi.new("char[]", _mpdec.MPD_MAX_SIGNAL_LIST) + n = _mpdec.mpd_lsnprint_signals(buf, len(buf), value, + _mpdec.dec_signal_string) + if not 0 <= n < len(buf): + raise SystemError("flags repr") + return _ffi.buffer(buf, n)[:].decode() + + def _as_list(self): + value = getattr(self.ctx, self.attrname) + names = [] + for name, flag in _SIGNALS.items(): + if value & flag: + names.append(name) + return names + + def _as_dict(self): + value = getattr(self.ctx, self.attrname) + return {name: bool(value & flag) + for (name, flag) in _SIGNALS.items()} + + def copy(self): + return self._as_dict() + + def __len__(self): + return len(_SIGNALS) + + def __iter__(self): + return iter(_SIGNALS) + + def __getitem__(self, key): + return bool(getattr(self.ctx, self.attrname) & _SIGNALS[key]) + + def __setitem__(self, key, value): + if value: + setattr(self.ctx, self.attrname, + getattr(self.ctx, self.attrname) | _SIGNALS[key]) + else: + setattr(self.ctx, self.attrname, + getattr(self.ctx, self.attrname) & ~_SIGNALS[key]) + + def __delitem__(self, key): + raise ValueError("signal keys cannot be deleted") + + +class _CatchConversions: + def __init__(self, mpd, context, exact): + self.mpd = mpd + self.context = _getcontext(context) + self.exact = exact + + def __enter__(self): + if self.exact: + self.ctx = _ffi.new("struct mpd_context_t*") + _mpdec.mpd_maxcontext(self.ctx) + else: + self.ctx = self.context._ctx + self.status_ptr = _ffi.new("uint32_t*") + return self.ctx, self.status_ptr + + def __exit__(self, *args): + if self.exact: + # we want exact results + status = self.status_ptr[0] + if status & (_mpdec.MPD_Inexact | + _mpdec.MPD_Rounded | + _mpdec.MPD_Clamped): + _mpdec.mpd_seterror( + self.mpd, _mpdec.MPD_Invalid_operation, self.status_ptr) + status = self.status_ptr[0] + if self.exact: + status &= _mpdec.MPD_Errors + # May raise a DecimalException + self.context._add_status(status) + +class _CatchStatus: + def __init__(self, context): + self.context = context + + def __enter__(self): + self.status_ptr = _ffi.new("uint32_t*") + return self.context._ctx, self.status_ptr + + def __exit__(self, *args): + status = self.status_ptr[0] + # May raise a DecimalException + self.context._add_status(status) + +##### Setup Specific Contexts ############################################ + +# The default context prototype used by Context() +# Is mutable, so that new contexts can have different default values + +DefaultContext = Context( + prec=28, rounding=ROUND_HALF_EVEN, + traps=[DivisionByZero, Overflow, InvalidOperation], + flags=[], + Emax=999999, + Emin=-999999, + capitals=1, + clamp=0 +) + +# Pre-made alternate contexts offered by the specification +# Don't change these; the user should be able to select these +# contexts and be able to reproduce results from other implementations +# of the spec. + +BasicContext = Context( + prec=9, rounding=ROUND_HALF_UP, + traps=[DivisionByZero, Overflow, InvalidOperation, Clamped, Underflow], + flags=[], +) + +ExtendedContext = Context( + prec=9, rounding=ROUND_HALF_EVEN, + traps=[], + flags=[], +) diff --git a/lib_pypy/_libmpdec/basearith.c b/lib_pypy/_libmpdec/basearith.c new file mode 100644 --- /dev/null +++ b/lib_pypy/_libmpdec/basearith.c @@ -0,0 +1,658 @@ +/* + * Copyright (c) 2008-2016 Stefan Krah. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright From noreply at buildbot.pypy.org Tue Mar 10 21:56:48 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Tue, 10 Mar 2015 21:56:48 +0100 (CET) Subject: [pypy-commit] pypy py3.3: Precompile _decimal cffi libraries Message-ID: <20150310205648.C00C51C04BE@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3.3 Changeset: r76304:3f222cd95c63 Date: 2015-03-10 14:49 +0100 http://bitbucket.org/pypy/pypy/changeset/3f222cd95c63/ Log: Precompile _decimal cffi libraries diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -54,7 +54,7 @@ def create_cffi_import_libraries(pypy_c, options): - modules = ['_sqlite3', 'lzma', 'audioop'] + modules = ['_sqlite3', 'lzma', 'audioop', '_decimal'] if not sys.platform == 'win32': modules += ['_curses', 'syslog', '_gdbm',] if not options.no_tk: From noreply at buildbot.pypy.org Tue Mar 10 21:56:50 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Tue, 10 Mar 2015 21:56:50 +0100 (CET) Subject: [pypy-commit] pypy py3.3: Fix a RPython crash in test_threading Message-ID: <20150310205650.02EC21C04BE@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3.3 Changeset: r76305:26d190840ef4 Date: 2015-03-10 15:10 +0100 http://bitbucket.org/pypy/pypy/changeset/26d190840ef4/ Log: Fix a RPython crash in test_threading diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py --- a/pypy/module/thread/os_lock.py +++ b/pypy/module/thread/os_lock.py @@ -244,6 +244,9 @@ def release_save_w(self, space): """For internal use by `threading.Condition`.""" + if self.rlock_count == 0: + raise OperationError(space.w_RuntimeError, space.wrap( + "cannot release un-acquired lock")) count, self.rlock_count = self.rlock_count, 0 owner, self.rlock_owner = self.rlock_owner, 0 self.lock.release() diff --git a/pypy/module/thread/test/test_lock.py b/pypy/module/thread/test/test_lock.py --- a/pypy/module/thread/test/test_lock.py +++ b/pypy/module/thread/test/test_lock.py @@ -140,6 +140,7 @@ def test_release_save(self): import _thread lock = _thread.RLock() + raises(RuntimeError, lock._release_save) lock.acquire() state = lock._release_save() lock._acquire_restore(state) From noreply at buildbot.pypy.org Tue Mar 10 21:56:51 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Tue, 10 Mar 2015 21:56:51 +0100 (CET) Subject: [pypy-commit] pypy py3.3: Be consistent and raise the same RuntimeError for unacquired locks Message-ID: <20150310205651.29F0F1C04BE@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3.3 Changeset: r76306:c4010a0d5561 Date: 2015-03-10 15:20 +0100 http://bitbucket.org/pypy/pypy/changeset/c4010a0d5561/ Log: Be consistent and raise the same RuntimeError for unacquired locks diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py --- a/pypy/module/thread/os_lock.py +++ b/pypy/module/thread/os_lock.py @@ -89,7 +89,8 @@ try: self.lock.release() except rthread.error: - raise wrap_thread_error(space, "release unlocked lock") + raise OperationError(space.w_RuntimeError, space.wrap( + "cannot release un-acquired lock")) def descr_lock_locked(self, space): """Return whether the lock is in the locked state.""" diff --git a/pypy/module/thread/test/test_lock.py b/pypy/module/thread/test/test_lock.py --- a/pypy/module/thread/test/test_lock.py +++ b/pypy/module/thread/test/test_lock.py @@ -12,7 +12,7 @@ lock = _thread.allocate_lock() assert type(lock) is _thread.LockType assert lock.locked() is False - raises(_thread.error, lock.release) + raises(RuntimeError, lock.release) assert lock.locked() is False r = lock.acquire() assert r is True @@ -21,7 +21,7 @@ assert lock.locked() is True lock.release() assert lock.locked() is False - raises(_thread.error, lock.release) + raises(RuntimeError, lock.release) assert lock.locked() is False feedback = [] lock.acquire() From noreply at buildbot.pypy.org Tue Mar 10 21:56:52 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Tue, 10 Mar 2015 21:56:52 +0100 (CET) Subject: [pypy-commit] pypy py3.3: Adjust test: pypy use __qualname__ even for builtin methods. Message-ID: <20150310205652.4AFD31C04BE@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3.3 Changeset: r76307:890f849d67aa Date: 2015-03-10 15:23 +0100 http://bitbucket.org/pypy/pypy/changeset/890f849d67aa/ Log: Adjust test: pypy use __qualname__ even for builtin methods. diff --git a/lib-python/3/test/test_reprlib.py b/lib-python/3/test/test_reprlib.py --- a/lib-python/3/test/test_reprlib.py +++ b/lib-python/3/test/test_reprlib.py @@ -187,7 +187,7 @@ # cpython repr(dict.items) == "", # pypy - repr(dict.items).startswith(" Author: Amaury Forgeot d'Arc Branch: py3.3 Changeset: r76308:1a0b1ff00ed7 Date: 2015-03-10 17:05 +0100 http://bitbucket.org/pypy/pypy/changeset/1a0b1ff00ed7/ Log: use fsdecode() instead of utf8 to encode unix socket names. diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -106,7 +106,10 @@ flowinfo = make_unsigned_flowinfo(space, flowinfo) return rsocket.INET6Address(host, port, flowinfo, scope_id) if rsocket.HAS_AF_UNIX and family == rsocket.AF_UNIX: - return rsocket.UNIXAddress(space.str_w(w_address)) + # Not using space.fsencode_w since Linux allows embedded NULs. + if space.isinstance_w(w_address, space, w_unicode): + w_address = self.fsencode(w_address) + return rsocket.UNIXAddress(space.bytes_w(w_address)) if rsocket.HAS_AF_NETLINK and family == rsocket.AF_NETLINK: w_pid, w_groups = space.unpackiterable(w_address, 2) return rsocket.NETLINKAddress(space.uint_w(w_pid), space.uint_w(w_groups)) From noreply at buildbot.pypy.org Tue Mar 10 21:56:54 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Tue, 10 Mar 2015 21:56:54 +0100 (CET) Subject: [pypy-commit] pypy py3.3: Record __context__ exception when close() fails after a failure in flush() Message-ID: <20150310205654.8E7E61C04BE@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3.3 Changeset: r76309:76586cc5f6f7 Date: 2015-03-10 17:33 +0100 http://bitbucket.org/pypy/pypy/changeset/76586cc5f6f7/ Log: Record __context__ exception when close() fails after a failure in flush() diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py --- a/pypy/module/_io/interp_bufferedio.py +++ b/pypy/module/_io/interp_bufferedio.py @@ -306,11 +306,23 @@ with self.lock: if self._closed(space): return + flush_exception = None try: space.call_method(self, "flush") + except OperationError as flush_exception: + pass finally: with self.lock: - space.call_method(self.w_raw, "close") + try: + space.call_method(self.w_raw, "close") + except OperationError as e: + if flush_exception: + space.setattr(e.get_w_value(space), + space.wrap('__context__'), + flush_exception.get_w_value(space)) + raise + if flush_exception: + raise def _dealloc_warn_w(self, space, w_source): space.call_method(self.w_raw, "_dealloc_warn", w_source) diff --git a/pypy/module/_io/test/test_bufferedio.py b/pypy/module/_io/test/test_bufferedio.py --- a/pypy/module/_io/test/test_bufferedio.py +++ b/pypy/module/_io/test/test_bufferedio.py @@ -560,6 +560,7 @@ b.flush = bad_flush err = raises(IOError, b.close) # exception not swallowed assert err.value.args == ('close',) + assert err.value.__context__.args == ('flush',) assert not b.closed class AppTestBufferedRWPair: From noreply at buildbot.pypy.org Tue Mar 10 22:08:34 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Tue, 10 Mar 2015 22:08:34 +0100 (CET) Subject: [pypy-commit] pypy py3.3: oops, don't we have tests for this? Message-ID: <20150310210834.0C2C31C00EF@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3.3 Changeset: r76310:d58705c3328c Date: 2015-03-10 22:07 +0100 http://bitbucket.org/pypy/pypy/changeset/d58705c3328c/ Log: oops, don't we have tests for this? diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -107,7 +107,7 @@ return rsocket.INET6Address(host, port, flowinfo, scope_id) if rsocket.HAS_AF_UNIX and family == rsocket.AF_UNIX: # Not using space.fsencode_w since Linux allows embedded NULs. - if space.isinstance_w(w_address, space, w_unicode): + if space.isinstance_w(w_address, space.w_unicode): w_address = self.fsencode(w_address) return rsocket.UNIXAddress(space.bytes_w(w_address)) if rsocket.HAS_AF_NETLINK and family == rsocket.AF_NETLINK: From noreply at buildbot.pypy.org Tue Mar 10 22:39:02 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Tue, 10 Mar 2015 22:39:02 +0100 (CET) Subject: [pypy-commit] pypy py3.3: Actually tun the test, and fix. Message-ID: <20150310213902.6C19D1C015A@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3.3 Changeset: r76311:6eb66f20eeb6 Date: 2015-03-10 22:38 +0100 http://bitbucket.org/pypy/pypy/changeset/6eb66f20eeb6/ Log: Actually tun the test, and fix. diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -108,7 +108,7 @@ if rsocket.HAS_AF_UNIX and family == rsocket.AF_UNIX: # Not using space.fsencode_w since Linux allows embedded NULs. if space.isinstance_w(w_address, space.w_unicode): - w_address = self.fsencode(w_address) + w_address = space.fsencode(w_address) return rsocket.UNIXAddress(space.bytes_w(w_address)) if rsocket.HAS_AF_NETLINK and family == rsocket.AF_NETLINK: w_pid, w_groups = space.unpackiterable(w_address, 2) From noreply at buildbot.pypy.org Tue Mar 10 22:52:17 2015 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 10 Mar 2015 22:52:17 +0100 (CET) Subject: [pypy-commit] pypy release-2.5.x: merge default into branch Message-ID: <20150310215217.BE71F1C119D@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: release-2.5.x Changeset: r76312:eae427987fb2 Date: 2015-03-10 21:28 +0200 http://bitbucket.org/pypy/pypy/changeset/eae427987fb2/ Log: merge default into branch diff too long, truncating to 2000 out of 40273 lines diff --git a/.gitignore b/.gitignore --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,10 @@ bin/pypy-c include/*.h +include/numpy/ lib_pypy/ctypes_config_cache/_[^_]*_*.py +libpypy-c.* +pypy-c pypy/_cache pypy/doc/*.html pypy/doc/config/*.html @@ -18,4 +21,5 @@ pypy/translator/c/src/dtoa.o pypy/translator/goal/pypy-c pypy/translator/goal/target*-c -release/ \ No newline at end of file +release/ +rpython/_cache/ diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -7,10 +7,7 @@ 9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm ab0dd631c22015ed88e583d9fdd4c43eebf0be21 pypy-2.1-beta1-arm 20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0 -20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0 -0000000000000000000000000000000000000000 release-2.3.0 394146e9bb673514c61f0150ab2013ccf78e8de7 release-2.3 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1 -32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1 -0000000000000000000000000000000000000000 release-2.2=3.1 +10f1b29a2bd21f837090286174a9ca030b8680b2 release-2.5.0 diff --git a/lib-python/2.7/CGIHTTPServer.py b/lib-python/2.7/CGIHTTPServer.py --- a/lib-python/2.7/CGIHTTPServer.py +++ b/lib-python/2.7/CGIHTTPServer.py @@ -106,16 +106,16 @@ def run_cgi(self): """Execute a CGI script.""" dir, rest = self.cgi_info - - i = rest.find('/') + path = dir + '/' + rest + i = path.find('/', len(dir)+1) while i >= 0: - nextdir = rest[:i] - nextrest = rest[i+1:] + nextdir = path[:i] + nextrest = path[i+1:] scriptdir = self.translate_path(nextdir) if os.path.isdir(scriptdir): dir, rest = nextdir, nextrest - i = rest.find('/') + i = path.find('/', len(dir)+1) else: break diff --git a/lib-python/2.7/Cookie.py b/lib-python/2.7/Cookie.py --- a/lib-python/2.7/Cookie.py +++ b/lib-python/2.7/Cookie.py @@ -56,7 +56,7 @@ >>> C = Cookie.SmartCookie() [Note: Long-time users of Cookie.py will remember using -Cookie.Cookie() to create an Cookie object. Although deprecated, it +Cookie.Cookie() to create a Cookie object. Although deprecated, it is still supported by the code. See the Backward Compatibility notes for more information.] @@ -426,6 +426,8 @@ "version" : "Version", } + _flags = {'secure', 'httponly'} + def __init__(self): # Set defaults self.key = self.value = self.coded_value = None @@ -529,9 +531,11 @@ _LegalCharsPatt = r"[\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=]" _CookiePattern = re.compile( r"(?x)" # This is a Verbose pattern + r"\s*" # Optional whitespace at start of cookie r"(?P" # Start of group 'key' ""+ _LegalCharsPatt +"+?" # Any word of at least one letter, nongreedy r")" # End of group 'key' + r"(" # Optional group: there may not be a value. r"\s*=\s*" # Equal Sign r"(?P" # Start of group 'val' r'"(?:[^\\"]|\\.)*"' # Any doublequoted string @@ -540,7 +544,9 @@ r"|" # or ""+ _LegalCharsPatt +"*" # Any word or empty string r")" # End of group 'val' - r"\s*;?" # Probably ending in a semi-colon + r")?" # End of optional value group + r"\s*" # Any number of spaces. + r"(\s+|;|$)" # Ending either at space, semicolon, or EOS. ) @@ -585,8 +591,12 @@ def __setitem__(self, key, value): """Dictionary style assignment.""" - rval, cval = self.value_encode(value) - self.__set(key, rval, cval) + if isinstance(value, Morsel): + # allow assignment of constructed Morsels (e.g. for pickling) + dict.__setitem__(self, key, value) + else: + rval, cval = self.value_encode(value) + self.__set(key, rval, cval) # end __setitem__ def output(self, attrs=None, header="Set-Cookie:", sep="\015\012"): @@ -641,7 +651,7 @@ while 0 <= i < n: # Start looking for a cookie - match = patt.search(str, i) + match = patt.match(str, i) if not match: break # No more cookies K,V = match.group("key"), match.group("val") @@ -656,8 +666,12 @@ M[ K[1:] ] = V elif K.lower() in Morsel._reserved: if M: - M[ K ] = _unquote(V) - else: + if V is None: + if K.lower() in Morsel._flags: + M[K] = True + else: + M[K] = _unquote(V) + elif V is not None: rval, cval = self.value_decode(V) self.__set(K, rval, cval) M = self[K] diff --git a/lib-python/2.7/SocketServer.py b/lib-python/2.7/SocketServer.py --- a/lib-python/2.7/SocketServer.py +++ b/lib-python/2.7/SocketServer.py @@ -416,8 +416,12 @@ self.socket = socket.socket(self.address_family, self.socket_type) if bind_and_activate: - self.server_bind() - self.server_activate() + try: + self.server_bind() + self.server_activate() + except: + self.server_close() + raise def server_bind(self): """Called by constructor to bind the socket. diff --git a/lib-python/2.7/_abcoll.py b/lib-python/2.7/_abcoll.py --- a/lib-python/2.7/_abcoll.py +++ b/lib-python/2.7/_abcoll.py @@ -143,7 +143,7 @@ methods except for __contains__, __iter__ and __len__. To override the comparisons (presumably for speed, as the - semantics are fixed), all you have to do is redefine __le__ and + semantics are fixed), redefine __le__ and __ge__, then the other operations will automatically follow suit. """ diff --git a/lib-python/2.7/argparse.py b/lib-python/2.7/argparse.py --- a/lib-python/2.7/argparse.py +++ b/lib-python/2.7/argparse.py @@ -1089,7 +1089,14 @@ # parse all the remaining options into the namespace # store any unrecognized options on the object, so that the top # level parser can decide what to do with them - namespace, arg_strings = parser.parse_known_args(arg_strings, namespace) + + # In case this subparser defines new defaults, we parse them + # in a new namespace object and then update the original + # namespace for the relevant parts. + subnamespace, arg_strings = parser.parse_known_args(arg_strings, None) + for key, value in vars(subnamespace).items(): + setattr(namespace, key, value) + if arg_strings: vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, []) getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings) diff --git a/lib-python/2.7/asynchat.py b/lib-python/2.7/asynchat.py --- a/lib-python/2.7/asynchat.py +++ b/lib-python/2.7/asynchat.py @@ -46,12 +46,17 @@ you - by calling your self.found_terminator() method. """ +import asyncore +import errno import socket -import asyncore from collections import deque from sys import py3kwarning from warnings import filterwarnings, catch_warnings +_BLOCKING_IO_ERRORS = (errno.EAGAIN, errno.EALREADY, errno.EINPROGRESS, + errno.EWOULDBLOCK) + + class async_chat (asyncore.dispatcher): """This is an abstract class. You must derive from this class, and add the two methods collect_incoming_data() and found_terminator()""" @@ -109,6 +114,8 @@ try: data = self.recv (self.ac_in_buffer_size) except socket.error, why: + if why.args[0] in _BLOCKING_IO_ERRORS: + return self.handle_error() return diff --git a/lib-python/2.7/bsddb/test/test_queue.py b/lib-python/2.7/bsddb/test/test_queue.py --- a/lib-python/2.7/bsddb/test/test_queue.py +++ b/lib-python/2.7/bsddb/test/test_queue.py @@ -10,6 +10,7 @@ #---------------------------------------------------------------------- + at unittest.skip("fails on Windows; see issue 22943") class SimpleQueueTestCase(unittest.TestCase): def setUp(self): self.filename = get_new_database_path() diff --git a/lib-python/2.7/cookielib.py b/lib-python/2.7/cookielib.py --- a/lib-python/2.7/cookielib.py +++ b/lib-python/2.7/cookielib.py @@ -1719,12 +1719,12 @@ def __repr__(self): r = [] for cookie in self: r.append(repr(cookie)) - return "<%s[%s]>" % (self.__class__, ", ".join(r)) + return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r)) def __str__(self): r = [] for cookie in self: r.append(str(cookie)) - return "<%s[%s]>" % (self.__class__, ", ".join(r)) + return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r)) # derives from IOError for backwards-compatibility with Python 2.4.0 diff --git a/lib-python/2.7/ctypes/test/test_pointers.py b/lib-python/2.7/ctypes/test/test_pointers.py --- a/lib-python/2.7/ctypes/test/test_pointers.py +++ b/lib-python/2.7/ctypes/test/test_pointers.py @@ -7,6 +7,8 @@ c_long, c_ulong, c_longlong, c_ulonglong, c_double, c_float] python_types = [int, int, int, int, int, long, int, long, long, long, float, float] +LargeNamedType = type('T' * 2 ** 25, (Structure,), {}) +large_string = 'T' * 2 ** 25 class PointersTestCase(unittest.TestCase): @@ -188,5 +190,11 @@ mth = WINFUNCTYPE(None)(42, "name", (), None) self.assertEqual(bool(mth), True) + def test_pointer_type_name(self): + self.assertTrue(POINTER(LargeNamedType)) + + def test_pointer_type_str_name(self): + self.assertTrue(POINTER(large_string)) + if __name__ == '__main__': unittest.main() diff --git a/lib-python/2.7/ctypes/test/test_python_api.py b/lib-python/2.7/ctypes/test/test_python_api.py --- a/lib-python/2.7/ctypes/test/test_python_api.py +++ b/lib-python/2.7/ctypes/test/test_python_api.py @@ -46,8 +46,8 @@ # This test is unreliable, because it is possible that code in # unittest changes the refcount of the '42' integer. So, it # is disabled by default. - @requires("refcount") def test_PyInt_Long(self): + requires("refcount") ref42 = grc(42) pythonapi.PyInt_FromLong.restype = py_object self.assertEqual(pythonapi.PyInt_FromLong(42), 42) diff --git a/lib-python/2.7/ctypes/test/test_win32.py b/lib-python/2.7/ctypes/test/test_win32.py --- a/lib-python/2.7/ctypes/test/test_win32.py +++ b/lib-python/2.7/ctypes/test/test_win32.py @@ -38,8 +38,11 @@ @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') class FunctionCallTestCase(unittest.TestCase): - @requires("SEH") + @unittest.skipUnless('MSC' in sys.version, "SEH only supported by MSC") + @unittest.skipIf(sys.executable.endswith('_d.exe'), + "SEH not enabled in debug builds") def test_SEH(self): + requires("SEH") # Call functions with invalid arguments, and make sure # that access violations are trapped and raise an # exception. @@ -87,9 +90,29 @@ dll = CDLL(_ctypes_test.__file__) - pt = POINT(10, 10) - rect = RECT(0, 0, 20, 20) - self.assertEqual(1, dll.PointInRect(byref(rect), pt)) + pt = POINT(15, 25) + left = c_long.in_dll(dll, 'left') + top = c_long.in_dll(dll, 'top') + right = c_long.in_dll(dll, 'right') + bottom = c_long.in_dll(dll, 'bottom') + rect = RECT(left, top, right, bottom) + PointInRect = dll.PointInRect + PointInRect.argtypes = [POINTER(RECT), POINT] + self.assertEqual(1, PointInRect(byref(rect), pt)) + + ReturnRect = dll.ReturnRect + ReturnRect.argtypes = [c_int, RECT, POINTER(RECT), POINT, RECT, + POINTER(RECT), POINT, RECT] + ReturnRect.restype = RECT + for i in range(4): + ret = ReturnRect(i, rect, pointer(rect), pt, rect, + byref(rect), pt, rect) + # the c function will check and modify ret if something is + # passed in improperly + self.assertEqual(ret.left, left.value) + self.assertEqual(ret.right, right.value) + self.assertEqual(ret.top, top.value) + self.assertEqual(ret.bottom, bottom.value) if __name__ == '__main__': unittest.main() diff --git a/lib-python/2.7/decimal.py b/lib-python/2.7/decimal.py --- a/lib-python/2.7/decimal.py +++ b/lib-python/2.7/decimal.py @@ -136,7 +136,6 @@ __version__ = '1.70' # Highest version of the spec this complies with -import copy as _copy import math as _math import numbers as _numbers @@ -3665,6 +3664,8 @@ if self._is_special: sign = _format_sign(self._sign, spec) body = str(self.copy_abs()) + if spec['type'] == '%': + body += '%' return _format_align(sign, body, spec) # a type of None defaults to 'g' or 'G', depending on context @@ -6033,7 +6034,10 @@ format_dict['decimal_point'] = '.' # record whether return type should be str or unicode - format_dict['unicode'] = isinstance(format_spec, unicode) + try: + format_dict['unicode'] = isinstance(format_spec, unicode) + except NameError: + format_dict['unicode'] = False return format_dict diff --git a/lib-python/2.7/distutils/__init__.py b/lib-python/2.7/distutils/__init__.py --- a/lib-python/2.7/distutils/__init__.py +++ b/lib-python/2.7/distutils/__init__.py @@ -15,5 +15,5 @@ # Updated automatically by the Python release process. # #--start constants-- -__version__ = "2.7.8" +__version__ = "2.7.9" #--end constants-- diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py --- a/lib-python/2.7/distutils/command/build_ext.py +++ b/lib-python/2.7/distutils/command/build_ext.py @@ -245,7 +245,7 @@ # Python's library directory must be appended to library_dirs # See Issues: #1600860, #4366 if (sysconfig.get_config_var('Py_ENABLE_SHARED')): - if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")): + if not sysconfig.python_build: # building third party extensions self.library_dirs.append(sysconfig.get_config_var('LIBDIR')) else: diff --git a/lib-python/2.7/distutils/command/upload.py b/lib-python/2.7/distutils/command/upload.py --- a/lib-python/2.7/distutils/command/upload.py +++ b/lib-python/2.7/distutils/command/upload.py @@ -136,8 +136,8 @@ # Build up the MIME payload for the POST data boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254' - sep_boundary = '\n--' + boundary - end_boundary = sep_boundary + '--' + sep_boundary = '\r\n--' + boundary + end_boundary = sep_boundary + '--\r\n' body = StringIO.StringIO() for key, value in data.items(): # handle multiple entries for the same name @@ -151,14 +151,13 @@ fn = "" body.write(sep_boundary) - body.write('\nContent-Disposition: form-data; name="%s"'%key) + body.write('\r\nContent-Disposition: form-data; name="%s"' % key) body.write(fn) - body.write("\n\n") + body.write("\r\n\r\n") body.write(value) if value and value[-1] == '\r': body.write('\n') # write an extra newline (lurve Macs) body.write(end_boundary) - body.write("\n") body = body.getvalue() self.announce("Submitting %s to %s" % (filename, self.repository), log.INFO) diff --git a/lib-python/2.7/distutils/file_util.py b/lib-python/2.7/distutils/file_util.py --- a/lib-python/2.7/distutils/file_util.py +++ b/lib-python/2.7/distutils/file_util.py @@ -85,7 +85,8 @@ (os.symlink) instead of copying: set it to "hard" or "sym"; if it is None (the default), files are copied. Don't set 'link' on systems that don't support it: 'copy_file()' doesn't check if hard or symbolic - linking is available. + linking is available. If hardlink fails, falls back to + _copy_file_contents(). Under Mac OS, uses the native file copy function in macostools; on other systems, uses '_copy_file_contents()' to copy file contents. @@ -137,24 +138,31 @@ # (Unix only, of course, but that's the caller's responsibility) if link == 'hard': if not (os.path.exists(dst) and os.path.samefile(src, dst)): - os.link(src, dst) + try: + os.link(src, dst) + return (dst, 1) + except OSError: + # If hard linking fails, fall back on copying file + # (some special filesystems don't support hard linking + # even under Unix, see issue #8876). + pass elif link == 'sym': if not (os.path.exists(dst) and os.path.samefile(src, dst)): os.symlink(src, dst) + return (dst, 1) # Otherwise (non-Mac, not linking), copy the file contents and # (optionally) copy the times and mode. - else: - _copy_file_contents(src, dst) - if preserve_mode or preserve_times: - st = os.stat(src) + _copy_file_contents(src, dst) + if preserve_mode or preserve_times: + st = os.stat(src) - # According to David Ascher , utime() should be done - # before chmod() (at least under NT). - if preserve_times: - os.utime(dst, (st[ST_ATIME], st[ST_MTIME])) - if preserve_mode: - os.chmod(dst, S_IMODE(st[ST_MODE])) + # According to David Ascher , utime() should be done + # before chmod() (at least under NT). + if preserve_times: + os.utime(dst, (st[ST_ATIME], st[ST_MTIME])) + if preserve_mode: + os.chmod(dst, S_IMODE(st[ST_MODE])) return (dst, 1) diff --git a/lib-python/2.7/distutils/sysconfig_cpython.py b/lib-python/2.7/distutils/sysconfig_cpython.py --- a/lib-python/2.7/distutils/sysconfig_cpython.py +++ b/lib-python/2.7/distutils/sysconfig_cpython.py @@ -165,7 +165,8 @@ # version and build tools may not support the same set # of CPU architectures for universal builds. global _config_vars - if not _config_vars.get('CUSTOMIZED_OSX_COMPILER', ''): + # Use get_config_var() to ensure _config_vars is initialized. + if not get_config_var('CUSTOMIZED_OSX_COMPILER'): import _osx_support _osx_support.customize_compiler(_config_vars) _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True' diff --git a/lib-python/2.7/distutils/tests/test_bdist_rpm.py b/lib-python/2.7/distutils/tests/test_bdist_rpm.py --- a/lib-python/2.7/distutils/tests/test_bdist_rpm.py +++ b/lib-python/2.7/distutils/tests/test_bdist_rpm.py @@ -25,6 +25,7 @@ """ class BuildRpmTestCase(support.TempdirManager, + support.EnvironGuard, support.LoggingSilencer, unittest.TestCase): @@ -50,6 +51,7 @@ def test_quiet(self): # let's create a package tmp_dir = self.mkdtemp() + os.environ['HOME'] = tmp_dir # to confine dir '.rpmdb' creation pkg_dir = os.path.join(tmp_dir, 'foo') os.mkdir(pkg_dir) self.write_file((pkg_dir, 'setup.py'), SETUP_PY) @@ -92,6 +94,7 @@ def test_no_optimize_flag(self): # let's create a package that brakes bdist_rpm tmp_dir = self.mkdtemp() + os.environ['HOME'] = tmp_dir # to confine dir '.rpmdb' creation pkg_dir = os.path.join(tmp_dir, 'foo') os.mkdir(pkg_dir) self.write_file((pkg_dir, 'setup.py'), SETUP_PY) diff --git a/lib-python/2.7/distutils/tests/test_dist.py b/lib-python/2.7/distutils/tests/test_dist.py --- a/lib-python/2.7/distutils/tests/test_dist.py +++ b/lib-python/2.7/distutils/tests/test_dist.py @@ -11,7 +11,7 @@ from distutils.dist import Distribution, fix_help_options from distutils.cmd import Command import distutils.dist -from test.test_support import TESTFN, captured_stdout, run_unittest +from test.test_support import TESTFN, captured_stdout, run_unittest, unlink from distutils.tests import support @@ -64,6 +64,7 @@ with open(TESTFN, "w") as f: f.write("[global]\n") f.write("command_packages = foo.bar, splat") + self.addCleanup(unlink, TESTFN) files = [TESTFN] sys.argv.append("build") diff --git a/lib-python/2.7/distutils/tests/test_file_util.py b/lib-python/2.7/distutils/tests/test_file_util.py --- a/lib-python/2.7/distutils/tests/test_file_util.py +++ b/lib-python/2.7/distutils/tests/test_file_util.py @@ -8,6 +8,11 @@ from distutils.tests import support from test.test_support import run_unittest + +requires_os_link = unittest.skipUnless(hasattr(os, "link"), + "test requires os.link()") + + class FileUtilTestCase(support.TempdirManager, unittest.TestCase): def _log(self, msg, *args): @@ -74,6 +79,44 @@ copy_file(foo, dst_dir) self.assertTrue(os.path.exists(os.path.join(dst_dir, 'foo'))) + @requires_os_link + def test_copy_file_hard_link(self): + with open(self.source, 'w') as f: + f.write('some content') + st = os.stat(self.source) + copy_file(self.source, self.target, link='hard') + st2 = os.stat(self.source) + st3 = os.stat(self.target) + self.assertTrue(os.path.samestat(st, st2), (st, st2)) + self.assertTrue(os.path.samestat(st2, st3), (st2, st3)) + with open(self.source, 'r') as f: + self.assertEqual(f.read(), 'some content') + + @requires_os_link + def test_copy_file_hard_link_failure(self): + # If hard linking fails, copy_file() falls back on copying file + # (some special filesystems don't support hard linking even under + # Unix, see issue #8876). + with open(self.source, 'w') as f: + f.write('some content') + st = os.stat(self.source) + def _os_link(*args): + raise OSError(0, "linking unsupported") + old_link = os.link + os.link = _os_link + try: + copy_file(self.source, self.target, link='hard') + finally: + os.link = old_link + st2 = os.stat(self.source) + st3 = os.stat(self.target) + self.assertTrue(os.path.samestat(st, st2), (st, st2)) + self.assertFalse(os.path.samestat(st2, st3), (st2, st3)) + for fn in (self.source, self.target): + with open(fn, 'r') as f: + self.assertEqual(f.read(), 'some content') + + def test_suite(): return unittest.makeSuite(FileUtilTestCase) diff --git a/lib-python/2.7/distutils/tests/test_sysconfig.py b/lib-python/2.7/distutils/tests/test_sysconfig.py --- a/lib-python/2.7/distutils/tests/test_sysconfig.py +++ b/lib-python/2.7/distutils/tests/test_sysconfig.py @@ -3,6 +3,9 @@ import test import unittest import shutil +import subprocess +import sys +import textwrap from distutils import sysconfig from distutils.tests import support @@ -99,6 +102,24 @@ self.assertEqual(global_sysconfig.get_config_var('LDSHARED'), sysconfig.get_config_var('LDSHARED')) self.assertEqual(global_sysconfig.get_config_var('CC'), sysconfig.get_config_var('CC')) + def test_customize_compiler_before_get_config_vars(self): + # Issue #21923: test that a Distribution compiler + # instance can be called without an explicit call to + # get_config_vars(). + with open(TESTFN, 'w') as f: + f.writelines(textwrap.dedent('''\ + from distutils.core import Distribution + config = Distribution().get_command_obj('config') + # try_compile may pass or it may fail if no compiler + # is found but it should not raise an exception. + rc = config.try_compile('int x;') + ''')) + p = subprocess.Popen([str(sys.executable), TESTFN], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True) + outs, errs = p.communicate() + self.assertEqual(0, p.returncode, "Subprocess failed: " + outs) def test_suite(): diff --git a/lib-python/2.7/distutils/tests/test_upload.py b/lib-python/2.7/distutils/tests/test_upload.py --- a/lib-python/2.7/distutils/tests/test_upload.py +++ b/lib-python/2.7/distutils/tests/test_upload.py @@ -119,7 +119,7 @@ # what did we send ? self.assertIn('dédé', self.last_open.req.data) headers = dict(self.last_open.req.headers) - self.assertEqual(headers['Content-length'], '2085') + self.assertEqual(headers['Content-length'], '2159') self.assertTrue(headers['Content-type'].startswith('multipart/form-data')) self.assertEqual(self.last_open.req.get_method(), 'POST') self.assertEqual(self.last_open.req.get_full_url(), diff --git a/lib-python/2.7/doctest.py b/lib-python/2.7/doctest.py --- a/lib-python/2.7/doctest.py +++ b/lib-python/2.7/doctest.py @@ -216,7 +216,7 @@ # get_data() opens files as 'rb', so one must do the equivalent # conversion as universal newlines would do. return file_contents.replace(os.linesep, '\n'), filename - with open(filename) as f: + with open(filename, 'U') as f: return f.read(), filename # Use sys.stdout encoding for ouput. diff --git a/lib-python/2.7/email/feedparser.py b/lib-python/2.7/email/feedparser.py --- a/lib-python/2.7/email/feedparser.py +++ b/lib-python/2.7/email/feedparser.py @@ -49,8 +49,8 @@ simple abstraction -- it parses until EOF closes the current message. """ def __init__(self): - # The last partial line pushed into this object. - self._partial = '' + # Chunks of the last partial line pushed into this object. + self._partial = [] # The list of full, pushed lines, in reverse order self._lines = [] # The stack of false-EOF checking predicates. @@ -66,8 +66,8 @@ def close(self): # Don't forget any trailing partial line. - self._lines.append(self._partial) - self._partial = '' + self.pushlines(''.join(self._partial).splitlines(True)) + self._partial = [] self._closed = True def readline(self): @@ -95,8 +95,29 @@ def push(self, data): """Push some new data into this object.""" - # Handle any previous leftovers - data, self._partial = self._partial + data, '' + # Crack into lines, but preserve the linesep characters on the end of each + parts = data.splitlines(True) + + if not parts or not parts[0].endswith(('\n', '\r')): + # No new complete lines, so just accumulate partials + self._partial += parts + return + + if self._partial: + # If there are previous leftovers, complete them now + self._partial.append(parts[0]) + parts[0:1] = ''.join(self._partial).splitlines(True) + del self._partial[:] + + # If the last element of the list does not end in a newline, then treat + # it as a partial line. We only check for '\n' here because a line + # ending with '\r' might be a line that was split in the middle of a + # '\r\n' sequence (see bugs 1555570 and 1721862). + if not parts[-1].endswith('\n'): + self._partial = [parts.pop()] + self.pushlines(parts) + + def pushlines(self, lines): # Crack into lines, but preserve the newlines on the end of each parts = NLCRE_crack.split(data) # The *ahem* interesting behaviour of re.split when supplied grouping diff --git a/lib-python/2.7/email/mime/nonmultipart.py b/lib-python/2.7/email/mime/nonmultipart.py --- a/lib-python/2.7/email/mime/nonmultipart.py +++ b/lib-python/2.7/email/mime/nonmultipart.py @@ -12,7 +12,7 @@ class MIMENonMultipart(MIMEBase): - """Base class for MIME multipart/* type messages.""" + """Base class for MIME non-multipart type messages.""" def attach(self, payload): # The public API prohibits attaching multiple subparts to MIMEBase diff --git a/lib-python/2.7/email/test/test_email.py b/lib-python/2.7/email/test/test_email.py --- a/lib-python/2.7/email/test/test_email.py +++ b/lib-python/2.7/email/test/test_email.py @@ -11,6 +11,7 @@ import warnings import textwrap from cStringIO import StringIO +from random import choice import email @@ -2578,16 +2579,64 @@ bsf.push(il) nt += n n1 = 0 - while True: - ol = bsf.readline() - if ol == NeedMoreData: - break + for ol in iter(bsf.readline, NeedMoreData): om.append(ol) n1 += 1 self.assertEqual(n, n1) self.assertEqual(len(om), nt) self.assertEqual(''.join([il for il, n in imt]), ''.join(om)) + def test_push_random(self): + from email.feedparser import BufferedSubFile, NeedMoreData + + n = 10000 + chunksize = 5 + chars = 'abcd \t\r\n' + + s = ''.join(choice(chars) for i in range(n)) + '\n' + target = s.splitlines(True) + + bsf = BufferedSubFile() + lines = [] + for i in range(0, len(s), chunksize): + chunk = s[i:i+chunksize] + bsf.push(chunk) + lines.extend(iter(bsf.readline, NeedMoreData)) + self.assertEqual(lines, target) + + +class TestFeedParsers(TestEmailBase): + + def parse(self, chunks): + from email.feedparser import FeedParser + feedparser = FeedParser() + for chunk in chunks: + feedparser.feed(chunk) + return feedparser.close() + + def test_newlines(self): + m = self.parse(['a:\nb:\rc:\r\nd:\n']) + self.assertEqual(m.keys(), ['a', 'b', 'c', 'd']) + m = self.parse(['a:\nb:\rc:\r\nd:']) + self.assertEqual(m.keys(), ['a', 'b', 'c', 'd']) + m = self.parse(['a:\rb', 'c:\n']) + self.assertEqual(m.keys(), ['a', 'bc']) + m = self.parse(['a:\r', 'b:\n']) + self.assertEqual(m.keys(), ['a', 'b']) + m = self.parse(['a:\r', '\nb:\n']) + self.assertEqual(m.keys(), ['a', 'b']) + + def test_long_lines(self): + # Expected peak memory use on 32-bit platform: 4*N*M bytes. + M, N = 1000, 20000 + m = self.parse(['a:b\n\n'] + ['x'*M] * N) + self.assertEqual(m.items(), [('a', 'b')]) + self.assertEqual(m.get_payload(), 'x'*M*N) + m = self.parse(['a:b\r\r'] + ['x'*M] * N) + self.assertEqual(m.items(), [('a', 'b')]) + self.assertEqual(m.get_payload(), 'x'*M*N) + m = self.parse(['a:\r', 'b: '] + ['x'*M] * N) + self.assertEqual(m.items(), [('a', ''), ('b', 'x'*M*N)]) class TestParsers(TestEmailBase): @@ -3180,7 +3229,6 @@ self.assertEqual(res, '=?iso-8859-2?q?abc?=') self.assertIsInstance(res, str) - # Test RFC 2231 header parameters (en/de)coding class TestRFC2231(TestEmailBase): def test_get_param(self): diff --git a/lib-python/2.7/ensurepip/__init__.py b/lib-python/2.7/ensurepip/__init__.py new file mode 100644 --- /dev/null +++ b/lib-python/2.7/ensurepip/__init__.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python2 +from __future__ import print_function + +import os +import os.path +import pkgutil +import shutil +import sys +import tempfile + + +__all__ = ["version", "bootstrap"] + + +_SETUPTOOLS_VERSION = "7.0" + +_PIP_VERSION = "1.5.6" + +# pip currently requires ssl support, so we try to provide a nicer +# error message when that is missing (http://bugs.python.org/issue19744) +_MISSING_SSL_MESSAGE = ("pip {} requires SSL/TLS".format(_PIP_VERSION)) +try: + import ssl +except ImportError: + ssl = None + + def _require_ssl_for_pip(): + raise RuntimeError(_MISSING_SSL_MESSAGE) +else: + def _require_ssl_for_pip(): + pass + +_PROJECTS = [ + ("setuptools", _SETUPTOOLS_VERSION), + ("pip", _PIP_VERSION), +] + + +def _run_pip(args, additional_paths=None): + # Add our bundled software to the sys.path so we can import it + if additional_paths is not None: + sys.path = additional_paths + sys.path + + # Install the bundled software + import pip + pip.main(args) + + +def version(): + """ + Returns a string specifying the bundled version of pip. + """ + return _PIP_VERSION + + +def _disable_pip_configuration_settings(): + # We deliberately ignore all pip environment variables + # when invoking pip + # See http://bugs.python.org/issue19734 for details + keys_to_remove = [k for k in os.environ if k.startswith("PIP_")] + for k in keys_to_remove: + del os.environ[k] + # We also ignore the settings in the default pip configuration file + # See http://bugs.python.org/issue20053 for details + os.environ['PIP_CONFIG_FILE'] = os.devnull + + +def bootstrap(root=None, upgrade=False, user=False, + altinstall=False, default_pip=True, + verbosity=0): + """ + Bootstrap pip into the current Python installation (or the given root + directory). + + Note that calling this function will alter both sys.path and os.environ. + """ + if altinstall and default_pip: + raise ValueError("Cannot use altinstall and default_pip together") + + _require_ssl_for_pip() + _disable_pip_configuration_settings() + + # By default, installing pip and setuptools installs all of the + # following scripts (X.Y == running Python version): + # + # pip, pipX, pipX.Y, easy_install, easy_install-X.Y + # + # pip 1.5+ allows ensurepip to request that some of those be left out + if altinstall: + # omit pip, pipX and easy_install + os.environ["ENSUREPIP_OPTIONS"] = "altinstall" + elif not default_pip: + # omit pip and easy_install + os.environ["ENSUREPIP_OPTIONS"] = "install" + + tmpdir = tempfile.mkdtemp() + try: + # Put our bundled wheels into a temporary directory and construct the + # additional paths that need added to sys.path + additional_paths = [] + for project, version in _PROJECTS: + wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version) + whl = pkgutil.get_data( + "ensurepip", + "_bundled/{}".format(wheel_name), + ) + with open(os.path.join(tmpdir, wheel_name), "wb") as fp: + fp.write(whl) + + additional_paths.append(os.path.join(tmpdir, wheel_name)) + + # Construct the arguments to be passed to the pip command + args = ["install", "--no-index", "--find-links", tmpdir] + if root: + args += ["--root", root] + if upgrade: + args += ["--upgrade"] + if user: + args += ["--user"] + if verbosity: + args += ["-" + "v" * verbosity] + + _run_pip(args + [p[0] for p in _PROJECTS], additional_paths) + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + +def _uninstall_helper(verbosity=0): + """Helper to support a clean default uninstall process on Windows + + Note that calling this function may alter os.environ. + """ + # Nothing to do if pip was never installed, or has been removed + try: + import pip + except ImportError: + return + + # If the pip version doesn't match the bundled one, leave it alone + if pip.__version__ != _PIP_VERSION: + msg = ("ensurepip will only uninstall a matching version " + "({!r} installed, {!r} bundled)") + print(msg.format(pip.__version__, _PIP_VERSION), file=sys.stderr) + return + + _require_ssl_for_pip() + _disable_pip_configuration_settings() + + # Construct the arguments to be passed to the pip command + args = ["uninstall", "-y"] + if verbosity: + args += ["-" + "v" * verbosity] + + _run_pip(args + [p[0] for p in reversed(_PROJECTS)]) + + +def _main(argv=None): + if ssl is None: + print("Ignoring ensurepip failure: {}".format(_MISSING_SSL_MESSAGE), + file=sys.stderr) + return + + import argparse + parser = argparse.ArgumentParser(prog="python -m ensurepip") + parser.add_argument( + "--version", + action="version", + version="pip {}".format(version()), + help="Show the version of pip that is bundled with this Python.", + ) + parser.add_argument( + "-v", "--verbose", + action="count", + default=0, + dest="verbosity", + help=("Give more output. Option is additive, and can be used up to 3 " + "times."), + ) + parser.add_argument( + "-U", "--upgrade", + action="store_true", + default=False, + help="Upgrade pip and dependencies, even if already installed.", + ) + parser.add_argument( + "--user", + action="store_true", + default=False, + help="Install using the user scheme.", + ) + parser.add_argument( + "--root", + default=None, + help="Install everything relative to this alternate root directory.", + ) + parser.add_argument( + "--altinstall", + action="store_true", + default=False, + help=("Make an alternate install, installing only the X.Y versioned" + "scripts (Default: pipX, pipX.Y, easy_install-X.Y)"), + ) + parser.add_argument( + "--default-pip", + action="store_true", + default=True, + dest="default_pip", + help=argparse.SUPPRESS, + ) + parser.add_argument( + "--no-default-pip", + action="store_false", + dest="default_pip", + help=("Make a non default install, installing only the X and X.Y " + "versioned scripts."), + ) + + args = parser.parse_args(argv) + + bootstrap( + root=args.root, + upgrade=args.upgrade, + user=args.user, + verbosity=args.verbosity, + altinstall=args.altinstall, + default_pip=args.default_pip, + ) diff --git a/lib-python/2.7/ensurepip/__main__.py b/lib-python/2.7/ensurepip/__main__.py new file mode 100644 --- /dev/null +++ b/lib-python/2.7/ensurepip/__main__.py @@ -0,0 +1,4 @@ +import ensurepip + +if __name__ == "__main__": + ensurepip._main() diff --git a/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..097ab43430d4c1302b0be353a8c16407c370693b GIT binary patch [cut] diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..fa1d1054da1dab98f8906555d31a9fda271b3a85 GIT binary patch [cut] diff --git a/lib-python/2.7/ensurepip/_uninstall.py b/lib-python/2.7/ensurepip/_uninstall.py new file mode 100644 --- /dev/null +++ b/lib-python/2.7/ensurepip/_uninstall.py @@ -0,0 +1,30 @@ +"""Basic pip uninstallation support, helper for the Windows uninstaller""" + +import argparse +import ensurepip + + +def _main(argv=None): + parser = argparse.ArgumentParser(prog="python -m ensurepip._uninstall") + parser.add_argument( + "--version", + action="version", + version="pip {}".format(ensurepip.version()), + help="Show the version of pip this will attempt to uninstall.", + ) + parser.add_argument( + "-v", "--verbose", + action="count", + default=0, + dest="verbosity", + help=("Give more output. Option is additive, and can be used up to 3 " + "times."), + ) + + args = parser.parse_args(argv) + + ensurepip._uninstall_helper(verbosity=args.verbosity) + + +if __name__ == "__main__": + _main() diff --git a/lib-python/2.7/glob.py b/lib-python/2.7/glob.py --- a/lib-python/2.7/glob.py +++ b/lib-python/2.7/glob.py @@ -35,11 +35,16 @@ patterns. """ + dirname, basename = os.path.split(pathname) if not has_magic(pathname): - if os.path.lexists(pathname): - yield pathname + if basename: + if os.path.lexists(pathname): + yield pathname + else: + # Patterns ending with a slash should match only directories + if os.path.isdir(dirname): + yield pathname return - dirname, basename = os.path.split(pathname) if not dirname: for name in glob1(os.curdir, basename): yield name diff --git a/lib-python/2.7/gzip.py b/lib-python/2.7/gzip.py --- a/lib-python/2.7/gzip.py +++ b/lib-python/2.7/gzip.py @@ -164,9 +164,16 @@ def _write_gzip_header(self): self.fileobj.write('\037\213') # magic header self.fileobj.write('\010') # compression method - fname = os.path.basename(self.name) - if fname.endswith(".gz"): - fname = fname[:-3] + try: + # RFC 1952 requires the FNAME field to be Latin-1. Do not + # include filenames that cannot be represented that way. + fname = os.path.basename(self.name) + if not isinstance(fname, str): + fname = fname.encode('latin-1') + if fname.endswith('.gz'): + fname = fname[:-3] + except UnicodeEncodeError: + fname = '' flags = 0 if fname: flags = FNAME diff --git a/lib-python/2.7/hashlib.py b/lib-python/2.7/hashlib.py --- a/lib-python/2.7/hashlib.py +++ b/lib-python/2.7/hashlib.py @@ -15,8 +15,9 @@ md5(), sha1(), sha224(), sha256(), sha384(), and sha512() -More algorithms may be available on your platform but the above are -guaranteed to exist. +More algorithms may be available on your platform but the above are guaranteed +to exist. See the algorithms_guaranteed and algorithms_available attributes +to find out what algorithm names can be passed to new(). NOTE: If you want the adler32 or crc32 hash functions they are available in the zlib module. @@ -58,9 +59,14 @@ # always available algorithm is added. __always_supported = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512') +algorithms_guaranteed = set(__always_supported) +algorithms_available = set(__always_supported) + algorithms = __always_supported -__all__ = __always_supported + ('new', 'algorithms', 'pbkdf2_hmac') +__all__ = __always_supported + ('new', 'algorithms_guaranteed', + 'algorithms_available', 'algorithms', + 'pbkdf2_hmac') def __get_builtin_constructor(name): @@ -128,6 +134,8 @@ import _hashlib new = __hash_new __get_hash = __get_openssl_constructor + algorithms_available = algorithms_available.union( + _hashlib.openssl_md_meth_names) except ImportError: new = __py_new __get_hash = __get_builtin_constructor diff --git a/lib-python/2.7/httplib.py b/lib-python/2.7/httplib.py --- a/lib-python/2.7/httplib.py +++ b/lib-python/2.7/httplib.py @@ -215,6 +215,10 @@ # maximal line length when calling readline(). _MAXLINE = 65536 +# maximum amount of headers accepted +_MAXHEADERS = 100 + + class HTTPMessage(mimetools.Message): def addheader(self, key, value): @@ -271,6 +275,8 @@ elif self.seekable: tell = self.fp.tell while True: + if len(hlist) > _MAXHEADERS: + raise HTTPException("got more than %d headers" % _MAXHEADERS) if tell: try: startofline = tell() @@ -1185,21 +1191,29 @@ def __init__(self, host, port=None, key_file=None, cert_file=None, strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - source_address=None): + source_address=None, context=None): HTTPConnection.__init__(self, host, port, strict, timeout, source_address) self.key_file = key_file self.cert_file = cert_file + if context is None: + context = ssl._create_default_https_context() + if key_file or cert_file: + context.load_cert_chain(cert_file, key_file) + self._context = context def connect(self): "Connect to a host on a given (SSL) port." - sock = self._create_connection((self.host, self.port), - self.timeout, self.source_address) + HTTPConnection.connect(self) + if self._tunnel_host: - self.sock = sock - self._tunnel() - self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file) + server_hostname = self._tunnel_host + else: + server_hostname = self.host + + self.sock = self._context.wrap_socket(self.sock, + server_hostname=server_hostname) __all__.append("HTTPSConnection") @@ -1214,14 +1228,15 @@ _connection_class = HTTPSConnection def __init__(self, host='', port=None, key_file=None, cert_file=None, - strict=None): + strict=None, context=None): # provide a default host, pass the X509 cert info # urf. compensate for bad input. if port == 0: port = None self._setup(self._connection_class(host, port, key_file, - cert_file, strict)) + cert_file, strict, + context=context)) # we never actually use these for anything, but we keep them # here for compatibility with post-1.5.2 CVS. diff --git a/lib-python/2.7/idlelib/Bindings.py b/lib-python/2.7/idlelib/Bindings.py --- a/lib-python/2.7/idlelib/Bindings.py +++ b/lib-python/2.7/idlelib/Bindings.py @@ -75,7 +75,8 @@ ('!_Auto-open Stack Viewer', '<>'), ]), ('options', [ - ('_Configure IDLE...', '<>'), + ('Configure _IDLE', '<>'), + ('Configure _Extensions', '<>'), None, ]), ('help', [ diff --git a/lib-python/2.7/idlelib/CallTipWindow.py b/lib-python/2.7/idlelib/CallTipWindow.py --- a/lib-python/2.7/idlelib/CallTipWindow.py +++ b/lib-python/2.7/idlelib/CallTipWindow.py @@ -2,9 +2,8 @@ After ToolTip.py, which uses ideas gleaned from PySol Used by the CallTips IDLE extension. - """ -from Tkinter import * +from Tkinter import Toplevel, Label, LEFT, SOLID, TclError HIDE_VIRTUAL_EVENT_NAME = "<>" HIDE_SEQUENCES = ("", "") @@ -133,35 +132,28 @@ return bool(self.tipwindow) -def _calltip_window(parent): - root = Tk() - root.title("Test calltips") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) +def _calltip_window(parent): # htest # + from Tkinter import Toplevel, Text, LEFT, BOTH - class MyEditWin: # comparenceptually an editor_window - def __init__(self): - text = self.text = Text(root) - text.pack(side=LEFT, fill=BOTH, expand=1) - text.insert("insert", "string.split") - root.update() - self.calltip = CallTip(text) + top = Toplevel(parent) + top.title("Test calltips") + top.geometry("200x100+%d+%d" % (parent.winfo_rootx() + 200, + parent.winfo_rooty() + 150)) + text = Text(top) + text.pack(side=LEFT, fill=BOTH, expand=1) + text.insert("insert", "string.split") + top.update() + calltip = CallTip(text) - text.event_add("<>", "(") - text.event_add("<>", ")") - text.bind("<>", self.calltip_show) - text.bind("<>", self.calltip_hide) - - text.focus_set() - root.mainloop() - - def calltip_show(self, event): - self.calltip.showtip("Hello world", "insert", "end") - - def calltip_hide(self, event): - self.calltip.hidetip() - - editwin = MyEditWin() + def calltip_show(event): + calltip.showtip("(s=Hello world)", "insert", "end") + def calltip_hide(event): + calltip.hidetip() + text.event_add("<>", "(") + text.event_add("<>", ")") + text.bind("<>", calltip_show) + text.bind("<>", calltip_hide) + text.focus_set() if __name__=='__main__': from idlelib.idle_test.htest import run diff --git a/lib-python/2.7/idlelib/ClassBrowser.py b/lib-python/2.7/idlelib/ClassBrowser.py --- a/lib-python/2.7/idlelib/ClassBrowser.py +++ b/lib-python/2.7/idlelib/ClassBrowser.py @@ -19,6 +19,9 @@ from idlelib.TreeWidget import TreeNode, TreeItem, ScrolledCanvas from idlelib.configHandler import idleConf +file_open = None # Method...Item and Class...Item use this. +# Normally PyShell.flist.open, but there is no PyShell.flist for htest. + class ClassBrowser: def __init__(self, flist, name, path, _htest=False): @@ -27,6 +30,9 @@ """ _htest - bool, change box when location running htest. """ + global file_open + if not _htest: + file_open = PyShell.flist.open self.name = name self.file = os.path.join(path[0], self.name + ".py") self._htest = _htest @@ -101,7 +107,7 @@ return [] try: dict = pyclbr.readmodule_ex(name, [dir] + sys.path) - except ImportError, msg: + except ImportError: return [] items = [] self.classes = {} @@ -170,7 +176,7 @@ def OnDoubleClick(self): if not os.path.exists(self.file): return - edit = PyShell.flist.open(self.file) + edit = file_open(self.file) if hasattr(self.cl, 'lineno'): lineno = self.cl.lineno edit.gotoline(lineno) @@ -206,7 +212,7 @@ def OnDoubleClick(self): if not os.path.exists(self.file): return - edit = PyShell.flist.open(self.file) + edit = file_open(self.file) edit.gotoline(self.cl.methods[self.name]) def _class_browser(parent): #Wrapper for htest @@ -221,8 +227,9 @@ dir, file = os.path.split(file) name = os.path.splitext(file)[0] flist = PyShell.PyShellFileList(parent) + global file_open + file_open = flist.open ClassBrowser(flist, name, [dir], _htest=True) - parent.mainloop() if __name__ == "__main__": from idlelib.idle_test.htest import run diff --git a/lib-python/2.7/idlelib/ColorDelegator.py b/lib-python/2.7/idlelib/ColorDelegator.py --- a/lib-python/2.7/idlelib/ColorDelegator.py +++ b/lib-python/2.7/idlelib/ColorDelegator.py @@ -2,7 +2,6 @@ import re import keyword import __builtin__ -from Tkinter import * from idlelib.Delegator import Delegator from idlelib.configHandler import idleConf @@ -34,7 +33,6 @@ prog = re.compile(make_pat(), re.S) idprog = re.compile(r"\s+(\w+)", re.S) -asprog = re.compile(r".*?\b(as)\b") class ColorDelegator(Delegator): @@ -42,7 +40,6 @@ Delegator.__init__(self) self.prog = prog self.idprog = idprog - self.asprog = asprog self.LoadTagDefs() def setdelegate(self, delegate): @@ -74,7 +71,6 @@ "DEFINITION": idleConf.GetHighlight(theme, "definition"), "SYNC": {'background':None,'foreground':None}, "TODO": {'background':None,'foreground':None}, - "BREAK": idleConf.GetHighlight(theme, "break"), "ERROR": idleConf.GetHighlight(theme, "error"), # The following is used by ReplaceDialog: "hit": idleConf.GetHighlight(theme, "hit"), @@ -216,22 +212,6 @@ self.tag_add("DEFINITION", head + "+%dc" % a, head + "+%dc" % b) - elif value == "import": - # color all the "as" words on same line, except - # if in a comment; cheap approximation to the - # truth - if '#' in chars: - endpos = chars.index('#') - else: - endpos = len(chars) - while True: - m1 = self.asprog.match(chars, b, endpos) - if not m1: - break - a, b = m1.span(1) - self.tag_add("KEYWORD", - head + "+%dc" % a, - head + "+%dc" % b) m = self.prog.search(chars, m.end()) if "SYNC" in self.tag_names(next + "-1c"): head = next @@ -255,20 +235,23 @@ for tag in self.tagdefs.keys(): self.tag_remove(tag, "1.0", "end") -def _color_delegator(parent): +def _color_delegator(parent): # htest # + from Tkinter import Toplevel, Text from idlelib.Percolator import Percolator - root = Tk() - root.title("Test ColorDelegator") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) - source = "if somename: x = 'abc' # comment\nprint" - text = Text(root, background="white") + + top = Toplevel(parent) + top.title("Test ColorDelegator") + top.geometry("200x100+%d+%d" % (parent.winfo_rootx() + 200, + parent.winfo_rooty() + 150)) + source = "if somename: x = 'abc' # comment\nprint\n" + text = Text(top, background="white") + text.pack(expand=1, fill="both") text.insert("insert", source) - text.pack(expand=1, fill="both") + text.focus_set() + p = Percolator(text) d = ColorDelegator() p.insertfilter(d) - root.mainloop() if __name__ == "__main__": from idlelib.idle_test.htest import run diff --git a/lib-python/2.7/idlelib/Debugger.py b/lib-python/2.7/idlelib/Debugger.py --- a/lib-python/2.7/idlelib/Debugger.py +++ b/lib-python/2.7/idlelib/Debugger.py @@ -1,6 +1,5 @@ import os import bdb -import types from Tkinter import * from idlelib.WindowList import ListedToplevel from idlelib.ScrolledList import ScrolledList diff --git a/lib-python/2.7/idlelib/EditorWindow.py b/lib-python/2.7/idlelib/EditorWindow.py --- a/lib-python/2.7/idlelib/EditorWindow.py +++ b/lib-python/2.7/idlelib/EditorWindow.py @@ -1,6 +1,6 @@ import sys import os -from platform import python_version +import platform import re import imp from Tkinter import * @@ -22,6 +22,8 @@ # The default tab setting for a Text widget, in average-width characters. TK_TABWIDTH_DEFAULT = 8 +_py_version = ' (%s)' % platform.python_version() + def _sphinx_version(): "Format sys.version_info to produce the Sphinx version string used to install the chm docs" major, minor, micro, level, serial = sys.version_info @@ -151,7 +153,7 @@ # Safari requires real file:-URLs EditorWindow.help_url = 'file://' + EditorWindow.help_url else: - EditorWindow.help_url = "http://docs.python.org/%d.%d" % sys.version_info[:2] + EditorWindow.help_url = "https://docs.python.org/%d.%d/" % sys.version_info[:2] currentTheme=idleConf.CurrentTheme() self.flist = flist root = root or flist.root @@ -214,6 +216,8 @@ text.bind("<>", self.python_docs) text.bind("<>", self.about_dialog) text.bind("<>", self.config_dialog) + text.bind("<>", + self.config_extensions_dialog) text.bind("<>", self.open_module) text.bind("<>", lambda event: "break") text.bind("<>", self.select_all) @@ -568,6 +572,8 @@ def config_dialog(self, event=None): configDialog.ConfigDialog(self.top,'Settings') + def config_extensions_dialog(self, event=None): + configDialog.ConfigExtensionsDialog(self.top) def help_dialog(self, event=None): if self.root: @@ -691,30 +697,29 @@ return # XXX Ought to insert current file's directory in front of path try: - (f, file, (suffix, mode, type)) = _find_module(name) + (f, file_path, (suffix, mode, mtype)) = _find_module(name) except (NameError, ImportError) as msg: tkMessageBox.showerror("Import error", str(msg), parent=self.text) return - if type != imp.PY_SOURCE: + if mtype != imp.PY_SOURCE: tkMessageBox.showerror("Unsupported type", "%s is not a source module" % name, parent=self.text) return if f: f.close() if self.flist: - self.flist.open(file) + self.flist.open(file_path) else: - self.io.loadfile(file) + self.io.loadfile(file_path) + return file_path def open_class_browser(self, event=None): filename = self.io.filename - if not filename: - tkMessageBox.showerror( - "No filename", - "This buffer has no associated filename", - master=self.text) - self.text.focus_set() - return None + if not (self.__class__.__name__ == 'PyShellEditorWindow' + and filename): + filename = self.open_module() + if filename is None: + return head, tail = os.path.split(filename) base, ext = os.path.splitext(tail) from idlelib import ClassBrowser @@ -779,7 +784,7 @@ self.color = None def ResetColorizer(self): - "Update the colour theme" + "Update the color theme" # Called from self.filename_change_hook and from configDialog.py self._rmcolorizer() self._addcolorizer() @@ -944,7 +949,7 @@ short = self.short_title() long = self.long_title() if short and long: - title = short + " - " + long + title = short + " - " + long + _py_version elif short: title = short elif long: @@ -968,14 +973,13 @@ self.undo.reset_undo() def short_title(self): - pyversion = "Python " + python_version() + ": " filename = self.io.filename if filename: filename = os.path.basename(filename) else: filename = "Untitled" # return unicode string to display non-ASCII chars correctly - return pyversion + self._filename_to_unicode(filename) + return self._filename_to_unicode(filename) def long_title(self): # return unicode string to display non-ASCII chars correctly @@ -1711,7 +1715,8 @@ tk.call('set', 'tcl_nonwordchars', '[^a-zA-Z0-9_]') -def _editor_window(parent): +def _editor_window(parent): # htest # + # error if close master window first - timer event, after script root = parent fixwordbreaks(root) if sys.argv[1:]: @@ -1721,7 +1726,8 @@ macosxSupport.setupApp(root, None) edit = EditorWindow(root=root, filename=filename) edit.text.bind("<>", edit.close_event) - parent.mainloop() + # Does not stop error, neither does following + # edit.text.bind("<>", edit.close_event) if __name__ == '__main__': diff --git a/lib-python/2.7/idlelib/GrepDialog.py b/lib-python/2.7/idlelib/GrepDialog.py --- a/lib-python/2.7/idlelib/GrepDialog.py +++ b/lib-python/2.7/idlelib/GrepDialog.py @@ -45,10 +45,10 @@ def create_entries(self): SearchDialogBase.create_entries(self) - self.globent = self.make_entry("In files:", self.globvar) + self.globent = self.make_entry("In files:", self.globvar)[0] def create_other_buttons(self): - f = self.make_frame() + f = self.make_frame()[0] btn = Checkbutton(f, anchor="w", variable=self.recvar, @@ -131,7 +131,7 @@ self.top.withdraw() -def _grep_dialog(parent): # for htest +def _grep_dialog(parent): # htest # from idlelib.PyShell import PyShellFileList root = Tk() root.title("Test GrepDialog") diff --git a/lib-python/2.7/idlelib/IOBinding.py b/lib-python/2.7/idlelib/IOBinding.py --- a/lib-python/2.7/idlelib/IOBinding.py +++ b/lib-python/2.7/idlelib/IOBinding.py @@ -19,11 +19,7 @@ from idlelib.configHandler import idleConf -try: - from codecs import BOM_UTF8 -except ImportError: - # only available since Python 2.3 - BOM_UTF8 = '\xef\xbb\xbf' +from codecs import BOM_UTF8 # Try setting the locale, so that we can find out # what encoding to use @@ -72,6 +68,7 @@ encoding = encoding.lower() coding_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)') +blank_re = re.compile(r'^[ \t\f]*(?:[#\r\n]|$)') class EncodingMessage(SimpleDialog): "Inform user that an encoding declaration is needed." @@ -130,6 +127,8 @@ match = coding_re.match(line) if match is not None: break + if not blank_re.match(line): + return None else: return None name = match.group(1) @@ -529,6 +528,8 @@ ("All files", "*"), ] + defaultextension = '.py' if sys.platform == 'darwin' else '' + def askopenfile(self): dir, base = self.defaultfilename("open") if not self.opendialog: @@ -554,8 +555,10 @@ def asksavefile(self): dir, base = self.defaultfilename("save") if not self.savedialog: - self.savedialog = tkFileDialog.SaveAs(master=self.text, - filetypes=self.filetypes) + self.savedialog = tkFileDialog.SaveAs( + master=self.text, + filetypes=self.filetypes, + defaultextension=self.defaultextension) filename = self.savedialog.show(initialdir=dir, initialfile=base) if isinstance(filename, unicode): filename = filename.encode(filesystemencoding) diff --git a/lib-python/2.7/idlelib/NEWS.txt b/lib-python/2.7/idlelib/NEWS.txt --- a/lib-python/2.7/idlelib/NEWS.txt +++ b/lib-python/2.7/idlelib/NEWS.txt @@ -1,6 +1,183 @@ +What's New in IDLE 2.7.9? +========================= + +*Release data: 2014-12-07* (projected) + +- Issue #16893: Update Idle doc chapter to match current Idle and add new + information. + +- Issue #3068: Add Idle extension configuration dialog to Options menu. + Changes are written to HOME/.idlerc/config-extensions.cfg. + Original patch by Tal Einat. + +- Issue #16233: A module browser (File : Class Browser, Alt+C) requires a + editor window with a filename. When Class Browser is requested otherwise, + from a shell, output window, or 'Untitled' editor, Idle no longer displays + an error box. It now pops up an Open Module box (Alt+M). If a valid name + is entered and a module is opened, a corresponding browser is also opened. + +- Issue #4832: Save As to type Python files automatically adds .py to the + name you enter (even if your system does not display it). Some systems + automatically add .txt when type is Text files. + +- Issue #21986: Code objects are not normally pickled by the pickle module. + To match this, they are no longer pickled when running under Idle. + +- Issue #22221: IDLE now ignores the source encoding declaration on the second + line if the first line contains anything except a comment. + +- Issue #17390: Adjust Editor window title; remove 'Python', + move version to end. + +- Issue #14105: Idle debugger breakpoints no longer disappear + when inseting or deleting lines. + + +What's New in IDLE 2.7.8? +========================= + +*Release date: 2014-06-29* + +- Issue #21940: Add unittest for WidgetRedirector. Initial patch by Saimadhav + Heblikar. + +- Issue #18592: Add unittest for SearchDialogBase. Patch by Phil Webster. + +- Issue #21694: Add unittest for ParenMatch. Patch by Saimadhav Heblikar. + +- Issue #21686: add unittest for HyperParser. Original patch by Saimadhav + Heblikar. + +- Issue #12387: Add missing upper(lower)case versions of default Windows key + bindings for Idle so Caps Lock does not disable them. Patch by Roger Serwy. + +- Issue #21695: Closing a Find-in-files output window while the search is + still in progress no longer closes Idle. + +- Issue #18910: Add unittest for textView. Patch by Phil Webster. + +- Issue #18292: Add unittest for AutoExpand. Patch by Saihadhav Heblikar. + +- Issue #18409: Add unittest for AutoComplete. Patch by Phil Webster. + + +What's New in IDLE 2.7.7? +========================= + +*Release date: 2014-05-31* + +- Issue #18104: Add idlelib/idle_test/htest.py with a few sample tests to begin + consolidating and improving human-validated tests of Idle. Change other files + as needed to work with htest. Running the module as __main__ runs all tests. + +- Issue #21139: Change default paragraph width to 72, the PEP 8 recommendation. + +- Issue #21284: Paragraph reformat test passes after user changes reformat width. + +- Issue #20406: Use Python application icons for Idle window title bars. + Patch mostly by Serhiy Storchaka. + +- Issue #21029: Occurrences of "print" are now consistently colored as + being a keyword (the colorizer doesn't know if print functions are + enabled in the source). + +- Issue #17721: Remove non-functional configuration dialog help button until we + make it actually gives some help when clicked. Patch by Guilherme Sim�es. + +- Issue #17390: Add Python version to Idle editor window title bar. + Original patches by Edmond Burnett and Kent Johnson. + +- Issue #20058: sys.stdin.readline() in IDLE now always returns only one line. + +- Issue #19481: print() of unicode, str or bytearray subclass instance in IDLE + no more hangs. + +- Issue #18270: Prevent possible IDLE AttributeError on OS X when no initial + shell window is present. + +- Issue #17654: Ensure IDLE menus are customized properly on OS X for + non-framework builds and for all variants of Tk. + + +What's New in IDLE 2.7.6? +========================= + +*Release date: 2013-11-10* + +- Issue #19426: Fixed the opening of Python source file with specified encoding. + +- Issue #18873: IDLE now detects Python source code encoding only in comment + lines. + +- Issue #18988: The "Tab" key now works when a word is already autocompleted. + +- Issue #18489: Add tests for SearchEngine. Original patch by Phil Webster. + +- Issue #18429: Format / Format Paragraph, now works when comment blocks + are selected. As with text blocks, this works best when the selection + only includes complete lines. + +- Issue #18226: Add docstrings and unittests for FormatParagraph.py. + Original patches by Todd Rovito and Phil Webster. + +- Issue #18279: Format - Strip trailing whitespace no longer marks a file as + changed when it has not been changed. This fix followed the addition of a + test file originally written by Phil Webster (the issue's main goal). + +- Issue #18539: Calltips now work for float default arguments. + +- Issue #7136: In the Idle File menu, "New Window" is renamed "New File". + Patch by Tal Einat, Roget Serwy, and Todd Rovito. + +- Issue #8515: Set __file__ when run file in IDLE. + Initial patch by Bruce Frederiksen. + +- Issue #5492: Avoid traceback when exiting IDLE caused by a race condition. + +- Issue #17511: Keep IDLE find dialog open after clicking "Find Next". + Original patch by Sarah K. + +- Issue #15392: Create a unittest framework for IDLE. + Preliminary patch by Rajagopalasarma Jayakrishnan + See Lib/idlelib/idle_test/README.txt for how to run Idle tests. + +- Issue #14146: Highlight source line while debugging on Windows. + +- Issue #17532: Always include Options menu for IDLE on OS X. + Patch by Guilherme Sim�es. + + What's New in IDLE 2.7.5? ========================= +*Release date: 2013-05-12* + +- Issue #17838: Allow sys.stdin to be reassigned. + +- Issue #14735: Update IDLE docs to omit "Control-z on Windows". + +- Issue #17585: Fixed IDLE regression. Now closes when using exit() or quit(). + +- Issue #17657: Show full Tk version in IDLE's about dialog. + Patch by Todd Rovito. + +- Issue #17613: Prevent traceback when removing syntax colorizer in IDLE. + +- Issue #1207589: Backwards-compatibility patch for right-click menu in IDLE. + +- Issue #16887: IDLE now accepts Cancel in tabify/untabify dialog box. + +- Issue #14254: IDLE now handles readline correctly across shell restarts. + +- Issue #17614: IDLE no longer raises exception when quickly closing a file. + +- Issue #6698: IDLE now opens just an editor window when configured to do so. + +- Issue #8900: Using keyboard shortcuts in IDLE to open a file no longer + raises an exception. + +- Issue #6649: Fixed missing exit status in IDLE. Patch by Guilherme Polo. + - Issue #17390: Display Python version on Idle title bar. Initial patch by Edmond Burnett. @@ -8,17 +185,67 @@ What's New in IDLE 2.7.4? ========================= +*Release date: 2013-04-06* + +- Issue #17625: In IDLE, close the replace dialog after it is used. + +- IDLE was displaying spurious SystemExit tracebacks when running scripts + that terminated by raising SystemExit (i.e. unittest and turtledemo). + +- Issue #9290: In IDLE the sys.std* streams now implement io.TextIOBase + interface and support all mandatory methods and properties. + +- Issue #16829: IDLE printing no longer fails if there are spaces or other + special characters in the file path. + +- Issue #16819: IDLE method completion now correctly works for unicode literals. + +- Issue #16504: IDLE now catches SyntaxErrors raised by tokenizer. Patch by + Roger Serwy. + +- Issue #1207589: Add Cut/Copy/Paste items to IDLE right click Context Menu + Patch by Todd Rovito. + +- Issue #13052: Fix IDLE crashing when replace string in Search/Replace dialog + ended with '\'. Patch by Roger Serwy. + +- Issue #9803: Don't close IDLE on saving if breakpoint is open. + Patch by Roger Serwy. + +- Issue #14958: Change IDLE systax highlighting to recognize all string and byte + literals currently supported in Python 2.7. + +- Issue #14962: Update text coloring in IDLE shell window after changing + options. Patch by Roger Serwy. + +- Issue #10997: Prevent a duplicate entry in IDLE's "Recent Files" menu. + +- Issue #12510: Attempting to get invalid tooltip no longer closes IDLE. + Original patch by Roger Serwy. + +- Issue #10365: File open dialog now works instead of crashing + even when parent window is closed. Patch by Roger Serwy. + +- Issue #14876: Use user-selected font for highlight configuration. + Patch by Roger Serwy. + +- Issue #14409: IDLE now properly executes commands in the Shell window + when it cannot read the normal config files on startup and + has to use the built-in default key bindings. + There was previously a bug in one of the defaults. + +- Issue #3573: IDLE hangs when passing invalid command line args + (directory(ies) instead of file(s)) (Patch by Guilherme Polo) + +- Issue #5219: Prevent event handler cascade in IDLE. + - Issue #15318: Prevent writing to sys.stdin. - Issue #13532, #15319: Check that arguments to sys.stdout.write are strings. -- Issue # 12510: Attempt to get certain tool tips no longer crashes IDLE. - -- Issue10365: File open dialog now works instead of crashing even when +- Issue #10365: File open dialog now works instead of crashing even when parent window is closed while dialog is open. -- Issue 14876: use user-selected font for highlight configuration. - - Issue #14018: Update checks for unstable system Tcl/Tk versions on OS X to include versions shipped with OS X 10.7 and 10.8 in addition to 10.6. @@ -29,6 +256,27 @@ From noreply at buildbot.pypy.org Tue Mar 10 22:52:18 2015 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 10 Mar 2015 22:52:18 +0100 (CET) Subject: [pypy-commit] pypy default: update cffi - fix/skip for win32 and setuptools Message-ID: <20150310215218.EEFB21C119D@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r76313:95905f4df35a Date: 2015-03-10 23:08 +0200 http://bitbucket.org/pypy/pypy/changeset/95905f4df35a/ Log: update cffi - fix/skip for win32 and setuptools diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py b/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py --- a/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py @@ -165,7 +165,8 @@ assert lib.sin(12.3) == math.sin(12.3) v = ffi.verifier ext = v.get_extension() - assert 'distutils.extension.Extension' in str(ext.__class__) + assert 'distutils.extension.Extension' in str(ext.__class__) or \ + 'setuptools.extension.Extension' in str(ext.__class__) assert ext.sources == [maybe_relative_path(v.sourcefilename)] assert ext.name == v.get_module_name() assert ext.define_macros == [('TEST_EXTENSION_OBJECT', '1')] @@ -194,7 +195,8 @@ assert lib.test1eoes(7.0) == 42.0 v = ffi.verifier ext = v.get_extension() - assert 'distutils.extension.Extension' in str(ext.__class__) + assert 'distutils.extension.Extension' in str(ext.__class__) or \ + 'setuptools.extension.Extension' in str(ext.__class__) assert ext.sources == [maybe_relative_path(v.sourcefilename), extra_source] assert ext.name == v.get_module_name() diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py b/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py --- a/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py @@ -4,6 +4,9 @@ import subprocess from pypy.module.test_lib_pypy.cffi_tests.udir import udir +if sys.platform == 'win32': + py.test.skip('snippets do not run on win32') + def create_venv(name): tmpdir = udir.join(name) try: @@ -13,6 +16,20 @@ except OSError as e: py.test.skip("Cannot execute virtualenv: %s" % (e,)) + try: + deepcopy = os.symlink + except: + import shutil, errno + def deepcopy(src, dst): + try: + shutil.copytree(src, dst) + except OSError as e: + if e.errno in (errno.ENOTDIR, errno.EINVAL): + shutil.copy(src, dst) + else: + print 'got errno',e.errno,'not',errno.ENOTDIR + raise + site_packages = None for dirpath, dirnames, filenames in os.walk(str(tmpdir)): if os.path.basename(dirpath) == 'site-packages': @@ -32,7 +49,7 @@ modules += ('ply',) # needed for older versions of pycparser for module in modules: target = imp.find_module(module)[1] - os.symlink(target, os.path.join(site_packages, + deepcopy(target, os.path.join(site_packages, os.path.basename(target))) return tmpdir @@ -51,7 +68,11 @@ python_f.write(py.code.Source(python_snippet)) try: os.chdir(str(SNIPPET_DIR.join(dirname))) - vp = str(venv_dir.join('bin/python')) + if os.name == 'nt': + bindir = 'Scripts' + else: + bindir = 'bin' + vp = str(venv_dir.join(bindir).join('python')) subprocess.check_call((vp, 'setup.py', 'clean')) subprocess.check_call((vp, 'setup.py', 'install')) subprocess.check_call((vp, str(python_f))) From noreply at buildbot.pypy.org Tue Mar 10 22:52:20 2015 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 10 Mar 2015 22:52:20 +0100 (CET) Subject: [pypy-commit] pypy release-2.5.x: shuffle whatsnew Message-ID: <20150310215220.19AAD1C119D@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: release-2.5.x Changeset: r76314:258a22596144 Date: 2015-03-10 23:16 +0200 http://bitbucket.org/pypy/pypy/changeset/258a22596144/ Log: shuffle whatsnew diff --git a/pypy/doc/whatsnew-2.5.0.rst b/pypy/doc/whatsnew-2.5.0.rst --- a/pypy/doc/whatsnew-2.5.0.rst +++ b/pypy/doc/whatsnew-2.5.0.rst @@ -1,6 +1,6 @@ -======================= -What's new in PyPy 2.5 -======================= +======================== +What's new in PyPy 2.5.0 +======================== .. this is a revision shortly after release-2.4.x .. startrev: 7026746cbb1b diff --git a/pypy/doc/whatsnew-2.5.1.rst b/pypy/doc/whatsnew-2.5.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-2.5.1.rst @@ -0,0 +1,40 @@ +======================== +What's new in PyPy 2.5.1 +======================== + +.. this is a revision shortly after release-2.5.0 +.. startrev: 397b96217b85 + + +Non-blocking file reads sometimes raised EAGAIN even though they +had buffered data waiting, fixed in b1c4fcb04a42 + + + +.. branch: vmprof + +.. branch: stackroot-speedup-2 +Avoid tracing all stack roots during repeated minor collections, +by ignoring the part of the stack that didn't change + +.. branch: stdlib-2.7.9 +Update stdlib to version 2.7.9 + +.. branch: fix-kqueue-error2 +Fix exception being raised by kqueue.control (CPython compatibility) + +.. branch: gitignore + +.. branch: framestate2 +Refactor rpython.flowspace.framestate.FrameState. + +.. branch: alt_errno +Add an alternative location to save LastError, errno around ctypes, +cffi external calls so things like pdb will not overwrite it + +.. branch: nonquadratic-heapcache +Speed up the warmup times of the JIT by removing a quadratic algorithm in the +heapcache. + +.. branch: online-transforms-2 +Simplify flow graphs on the fly during annotation phase. diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -2,38 +2,6 @@ What's new in PyPy 2.5+ ======================= -.. this is a revision shortly after release-2.5.x +.. this is a revision shortly after release-2.5.1 .. startrev: 397b96217b85 - -Non-blocking file reads sometimes raised EAGAIN even though they -had buffered data waiting, fixed in b1c4fcb04a42 - - -.. branch: vmprof - -.. branch: stackroot-speedup-2 -Avoid tracing all stack roots during repeated minor collections, -by ignoring the part of the stack that didn't change - -.. branch: stdlib-2.7.9 -Update stdlib to version 2.7.9 - -.. branch: fix-kqueue-error2 -Fix exception being raised by kqueue.control (CPython compatibility) - -.. branch: gitignore - -.. branch: framestate2 -Refactor rpython.flowspace.framestate.FrameState. - -.. branch: alt_errno -Add an alternative location to save LastError, errno around ctypes, -cffi external calls so things like pdb will not overwrite it - -.. branch: nonquadratic-heapcache -Speed up the warmup times of the JIT by removing a quadratic algorithm in the -heapcache. - -.. branch: online-transforms-2 -Simplify flow graphs on the fly during annotation phase. From noreply at buildbot.pypy.org Tue Mar 10 22:52:21 2015 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 10 Mar 2015 22:52:21 +0100 (CET) Subject: [pypy-commit] pypy release-2.5.x: update version tags Message-ID: <20150310215221.4F95F1C119D@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: release-2.5.x Changeset: r76315:58a359d17c10 Date: 2015-03-10 23:21 +0200 http://bitbucket.org/pypy/pypy/changeset/58a359d17c10/ Log: update version tags diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -58,7 +58,7 @@ # General information about the project. project = u'PyPy' -copyright = u'2014, The PyPy Project' +copyright = u'2015, The PyPy Project' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -67,7 +67,7 @@ # The short X.Y version. version = '2.5' # The full version, including alpha/beta/rc tags. -release = '2.5.0' +release = '2.5.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -29,7 +29,7 @@ #define PY_VERSION "2.7.9" /* PyPy version as a string */ -#define PYPY_VERSION "2.6.0-alpha0" +#define PYPY_VERSION "2.5.1" /* Subversion Revision number of this file (not of the repository). * Empty since Mercurial migration. */ diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -10,7 +10,7 @@ #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py CPYTHON_API_VERSION = 1013 #XXX # sync with include/modsupport.h -PYPY_VERSION = (2, 6, 0, "alpha", 0) #XXX # sync patchlevel.h +PYPY_VERSION = (2, 5, 1, "final", 0) #XXX # sync patchlevel.h if platform.name == 'msvc': COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600) From noreply at buildbot.pypy.org Tue Mar 10 22:52:22 2015 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 10 Mar 2015 22:52:22 +0100 (CET) Subject: [pypy-commit] pypy release-2.5.x: draft a release note Message-ID: <20150310215222.71D921C119D@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: release-2.5.x Changeset: r76316:a6952f7ac901 Date: 2015-03-10 23:52 +0200 http://bitbucket.org/pypy/pypy/changeset/a6952f7ac901/ Log: draft a release note diff --git a/pypy/doc/release-2.5.1.rst b/pypy/doc/release-2.5.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-2.5.1.rst @@ -0,0 +1,93 @@ +============================== +PyPy 2.5.1 - XXXXXXXXXXXXXXXXX +============================== + +We're pleased to announce PyPy 2.5.1, following on the heels of 2.5.0 + +You can download the PyPy 2.5.1 release here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project, and for those who donate to our three sub-projects, as well as our +volunteers and contributors. +We've shown quite a bit of progress, but we're slowly running out of funds. +Please consider donating more, or even better convince your employer to donate, +so we can finish those projects! The three sub-projects are: + +* `Py3k`_ (supporting Python 3.x): We have released a Python 3.2.5 compatible version + we call PyPy3 2.4.0, and are working toward a Python 3.3 compatible version + +* `STM`_ (software transactional memory): We have released a first working version, + and continue to try out new promising paths of achieving a fast multithreaded Python + +* `NumPy`_ which requires installation of our fork of upstream numpy, + available `on bitbucket`_ + +.. _`Py3k`: http://pypy.org/py3donate.html +.. _`STM`: http://pypy.org/tmdonate2.html +.. _`NumPy`: http://pypy.org/numpydonate.html +.. _`on bitbucket`: https://www.bitbucket.org/pypy/numpy + +We would also like to encourage new people to join the project. PyPy has many +layers and we need help with all of them: `PyPy`_ and `Rpython`_ documentation +improvements, tweaking popular `modules`_ to run on pypy, or general `help`_ with making +Rpython's JIT even better. + +.. _`PyPy`: http://doc.pypy.org +.. _`Rpython`: http://rpython.readthedocs.org +.. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly +.. _`help`: http://doc.pypy.org/en/latest/project-ideas.html + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7. It's fast (`pypy and cpython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +This release supports **x86** machines on most common operating systems +(Linux 32/64, Mac OS X 64, Windows, and OpenBSD), +as well as newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux. + +While we support 32 bit python on Windows, work on the native Windows 64 +bit python is still stalling, we would welcome a volunteer +to `handle that`_. + +.. _`pypy and cpython 2.7.x`: http://speed.pypy.org +.. _`handle that`: http://doc.pypy.org/en/latest/windows.html#what-is-missing-for-a-full-64-bit-translation + +Highlights +========== + +* The past months have seen pypy mature and grow, as rpython becomes the goto + solution for writing fast dynamic language interpreters. Our separation of + Rpython and the python interpreter PyPy is now much clearer in the + `PyPy documentation`_ and we now have seperate `RPython documentation`_. + Tell us what still isn't clear, or even better help us improve the documentation. + +* We merged version 2.7.9 of python's stdlib + +* The garbage collector now ignores parts of the stack which did not change + since the last collection, another performance boost + +* errno and LastError are saved around cffi calls so things like pdb will not + overwrite it + +* We continue to asymtotically approach a score of 7 times faster than cpython + on our benchmark suite, we now rank 6.98 on latest runs + +* Issues reported with our previous release were resolved_ after reports from users on + our issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at + #pypy. + +.. _`PyPy documentation`: http://doc.pypy.org +.. _`RPython documentation`: http://rpython.readthedocs.org +.. _resolved: http://doc.pypy.org/en/latest/whatsnew-2.5.1.html + +Please try it out and let us know what you think. We especially welcome +success stories, we know you are using PyPy, please tell us about it! + +Cheers + +The PyPy Team From noreply at buildbot.pypy.org Tue Mar 10 23:03:14 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Tue, 10 Mar 2015 23:03:14 +0100 (CET) Subject: [pypy-commit] pypy py3.3: Fix test_ztranslation of _io module Message-ID: <20150310220314.0B19A1C11A2@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3.3 Changeset: r76317:abe327b7e4fc Date: 2015-03-10 23:03 +0100 http://bitbucket.org/pypy/pypy/changeset/abe327b7e4fc/ Log: Fix test_ztranslation of _io module diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py --- a/pypy/module/_io/interp_bufferedio.py +++ b/pypy/module/_io/interp_bufferedio.py @@ -306,23 +306,22 @@ with self.lock: if self._closed(space): return - flush_exception = None + w_flush_exception = None try: space.call_method(self, "flush") - except OperationError as flush_exception: - pass + except OperationError as e: + w_flush_exception = e.get_w_value(space) + raise finally: with self.lock: try: space.call_method(self.w_raw, "close") except OperationError as e: - if flush_exception: + if w_flush_exception: space.setattr(e.get_w_value(space), space.wrap('__context__'), - flush_exception.get_w_value(space)) + w_flush_exception) raise - if flush_exception: - raise def _dealloc_warn_w(self, space, w_source): space.call_method(self.w_raw, "_dealloc_warn", w_source) From noreply at buildbot.pypy.org Wed Mar 11 10:17:48 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Wed, 11 Mar 2015 10:17:48 +0100 (CET) Subject: [pypy-commit] stmgc default: add TODOs Message-ID: <20150311091748.0551B1C00FC@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1714:9328d529308b Date: 2015-03-11 10:18 +0100 http://bitbucket.org/pypy/stmgc/changeset/9328d529308b/ Log: add TODOs diff --git a/c8/TODO b/c8/TODO new file mode 100644 --- /dev/null +++ b/c8/TODO @@ -0,0 +1,30 @@ + +- improve sync of small objs on commit (see FLAG_SYNC_LARGE in nursery.c) + +- non-zeroed nursery: + read-the-docs benchmark shows 8% time spent in memset of throw_away_nursery + +- reshare pages: + make seg0 MAP_SHARED in order to re-share private pages during major GC + +- avoid usleep(10) when waiting for an inevitable transaction: + we do this sleep when we try to commit and another inev transaction is + currently running. idea: signal the inev transaction to do the commit + for us + +- maybe re-implement the "please commit soon" signal + +- the highest_overflow_number can overflow after 2**30 non-collect-time + minor collections + +- privatize (multiple) pages at once in the write barrier instead of + triggering segfaults + +- possibly messy too, but think about not using N+1 segments but only N + +- kill "atomic" and use regular lock elision? + +- increase the memory limit + +- avoid __builtin_frame_address(0) in precisely the performance-critical + functions like the interpreter main loop From noreply at buildbot.pypy.org Wed Mar 11 10:24:14 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Wed, 11 Mar 2015 10:24:14 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c8: update TODO Message-ID: <20150311092414.DE85F1C1159@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: stmgc-c8 Changeset: r76318:0cbb11adcde0 Date: 2015-03-11 10:19 +0100 http://bitbucket.org/pypy/pypy/changeset/0cbb11adcde0/ Log: update TODO diff --git a/TODO b/TODO --- a/TODO +++ b/TODO @@ -1,8 +1,18 @@ +------------------------------------------------------------ + +better heuristic of when to break transactions? e.g., we should +rarely break if there are not threads running in parallel. + ------------------------------------------------------------ maybe statically optimize away some stm_become_inevitable(), there are some loops that call it repeatedly (may be not relevant -for performance) +for performance) + +------------------------------------------------------------ + +clang doesn't optimize multiple stm_write() in a row (unlike GCC). +Optimize them manually... ------------------------------------------------------------ @@ -30,19 +40,6 @@ ------------------------------------------------------------ -clang doesn't optimize multiple stm_write() in a row (unlike GCC). -Optimize them manually... - ------------------------------------------------------------- - -stm_read() should also be optimized (moved out of loops). E.g. -quick_sort.py suffers a 30% slowdown because it uses deque.locate() -extensively. This method happens to have a *very* tight loop -where the read barrier has a big negative effect and could -actually be moved out. - ------------------------------------------------------------- - __pypy__.thread.getsegmentlimit(): XXX This limit is so far a compile time option (STM_NB_SEGMENTS in @@ -52,13 +49,8 @@ ------------------------------------------------------------ -JIT: add an artificial malloc if the loop is so small as to contain -any! +weakrefs stay alive longer than expected:: ------------------------------------------------------------- - -weakrefs stay alive longer than expected:: - y = some object that was already alive for a while x = ref(y) del y @@ -122,7 +114,7 @@ hg diff -r b0339cb53372^2 -r dd8e2f69fe96^2 : these trunk changes are missing in the stmgc-c7 branch of PyPy. In particular, reduce the diff between stmgc-c7 and default (kill rstm.ThreadLocalRef, etc) - + ------------------------------------------------------------ there are in PyPy-STM crashes related to markers (with the JIT?). @@ -130,10 +122,6 @@ ------------------------------------------------------------ -GC: call __del__() - ------------------------------------------------------------- - look at jit.elidables as a way to specify "this function can run earlier, in a separate transaction". Useful to avoid pointless conflicts in cases the jit.edliable update some cache, like with From noreply at buildbot.pypy.org Wed Mar 11 10:24:16 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Wed, 11 Mar 2015 10:24:16 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c8: add some skipped tests to test_inevitable that we should look at when optimizing Message-ID: <20150311092416.110D11C1159@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: stmgc-c8 Changeset: r76319:eb7c50e04cca Date: 2015-03-11 10:22 +0100 http://bitbucket.org/pypy/pypy/changeset/eb7c50e04cca/ Log: add some skipped tests to test_inevitable that we should look at when optimizing the placement of become_inevitable diff --git a/rpython/translator/stm/test/test_inevitable.py b/rpython/translator/stm/test/test_inevitable.py --- a/rpython/translator/stm/test/test_inevitable.py +++ b/rpython/translator/stm/test/test_inevitable.py @@ -6,7 +6,7 @@ from rpython.translator.stm.inevitable import insert_turn_inevitable from rpython.translator.stm import inevitable from rpython.conftest import option - +import py CATEGORIES = [inevitable.ALWAYS_ALLOW_OPERATIONS, inevitable.CALLS, @@ -38,8 +38,7 @@ class LLSTMInevFrame(LLFrame): def op_stm_become_inevitable(self, info): assert info is not None - if self.llinterpreter.inevitable_cause is None: - self.llinterpreter.inevitable_cause = info + self.llinterpreter.inevitable_cause.append(info) def op_gc_dump_rpy_heap(self): pass # for test_unsupported_op @@ -65,7 +64,7 @@ if option.view: self.translator.view() # - interp.inevitable_cause = None + interp.inevitable_cause = [] result = interp.eval_graph(self.graph, args) return interp.inevitable_cause @@ -79,7 +78,7 @@ x1.foo = n res = self.interpret_inevitable(f1, [4]) - assert res is None + assert res == [] def test_unsupported_op(self): X = lltype.Struct('X', ('foo', lltype.Signed)) @@ -89,7 +88,7 @@ llop.gc_dump_rpy_heap(lltype.Void) res = self.interpret_inevitable(f1, []) - assert res == 'gc_dump_rpy_heap' + assert res == ['gc_dump_rpy_heap'] def test_raw_getfield(self): X = lltype.Struct('X', ('foo', lltype.Signed)) @@ -100,7 +99,7 @@ return x1.foo res = self.interpret_inevitable(f1, []) - assert res == 'getfield' + assert res == ['getfield'] def test_raw_getfield_immutable(self): X = lltype.Struct('X', ('foo', lltype.Signed), @@ -112,7 +111,7 @@ return x1.foo res = self.interpret_inevitable(f1, []) - assert res is None + assert res == [] def test_raw_getfield_with_hint(self): X = lltype.Struct('X', ('foo', lltype.Signed), @@ -124,7 +123,7 @@ return x1.foo res = self.interpret_inevitable(f1, []) - assert res is None + assert res == [] def test_raw_setfield(self): X = lltype.Struct('X', ('foo', lltype.Signed)) @@ -135,7 +134,7 @@ x1.foo = n res = self.interpret_inevitable(f1, [43]) - assert res == 'setfield' + assert res == ['setfield'] def test_malloc_no_inevitable(self): X = lltype.GcStruct('X', ('foo', lltype.Signed)) @@ -144,7 +143,7 @@ return lltype.malloc(X) res = self.interpret_inevitable(f1, []) - assert res is None + assert res == [] def test_raw_malloc_1(self): X = lltype.Struct('X', ('foo', lltype.Signed)) @@ -154,7 +153,7 @@ lltype.free(p, flavor='raw') res = self.interpret_inevitable(f1, []) - assert res is None + assert res == [] def test_raw_malloc_2(self): X = lltype.Struct('X', ('foo', lltype.Signed)) @@ -164,7 +163,7 @@ llmemory.raw_free(addr) res = self.interpret_inevitable(f1, []) - assert res is None + assert res == [] def test_unknown_raw_free(self): X = lltype.Struct('X', ('foo', lltype.Signed)) @@ -172,7 +171,7 @@ lltype.free(p, flavor='raw') res = self.interpret_inevitable(f2, [lltype.malloc(X, flavor='raw')]) - assert res is None + assert res == [] def test_ext_direct_call_safe(self): @@ -185,7 +184,7 @@ extfunc() res = self.interpret_inevitable(f1, []) - assert res is None + assert res == [] def test_ext_direct_call_unsafe(self): @@ -197,7 +196,7 @@ extfunc() res = self.interpret_inevitable(f1, []) - assert res == 'extfunc()' + assert res == ['extfunc()'] def test_rpy_direct_call(self): def f2(): @@ -206,7 +205,7 @@ f2() res = self.interpret_inevitable(f1, []) - assert res is None + assert res == [] def test_rpy_indirect_call(self): def f2(): @@ -221,7 +220,7 @@ f() res = self.interpret_inevitable(f1, [True]) - assert res is None + assert res == [] def test_ext_indirect_call(self): TYPE = lltype.FuncType([], lltype.Void) @@ -240,7 +239,7 @@ f() res = self.interpret_inevitable(f1, [True]) - assert res == 'indirect_call' + assert res == ['indirect_call'] def test_instantiate_indirect_call(self): # inits are necessary to generate indirect_call @@ -259,7 +258,7 @@ c() res = self.interpret_inevitable(f1, [True]) - assert res is None + assert res == [] def test_raw_class_hint(self): class A: @@ -278,7 +277,7 @@ return i res = self.interpret_inevitable(f, [2]) - assert res is None # not setfield or getfield or free + assert res == [] # not setfield or getfield or free def test_do_malloc_llops(self): def f(i): @@ -288,7 +287,7 @@ return i res = self.interpret_inevitable(f, [2]) - assert res is None + assert res == [] def test_raw_load_nonpure(self): X = lltype.Struct('X', ('foo', lltype.Signed)) @@ -300,7 +299,7 @@ lltype.Signed, llmemory.cast_ptr_to_adr(x1), 0, False) res = self.interpret_inevitable(f1, []) - assert res == 'raw_load' + assert res == ['raw_load'] def test_raw_load_pure(self): X = lltype.Struct('X', ('foo', lltype.Signed)) @@ -312,7 +311,7 @@ lltype.Signed, llmemory.cast_ptr_to_adr(x1), 0, True) res = self.interpret_inevitable(f1, []) - assert res is None + assert res == [] def test_threadlocal(self): from rpython.rlib.rthread import ThreadLocalField @@ -330,4 +329,84 @@ #assert x == 42 res = self.interpret_inevitable(f1, []) - assert res is None + assert res == [] + + + + def test_only_one_inev(self): + py.test.skip("not yet") + X = lltype.Struct('X', ('foo', lltype.Signed)) + x1 = lltype.malloc(X, immortal=True) + x1.foo = 42 + + def f1(): + r = 0 + r += x1.foo + r += x1.foo + return r + + res = self.interpret_inevitable(f1, []) + assert res == ['getfield'] + + def test_only_one_inev2(self): + py.test.skip("not yet") + X = lltype.Struct('X', ('foo', lltype.Signed)) + x1 = lltype.malloc(X, immortal=True) + x1.foo = 42 + + def f1(i): + r = 0 + if i: + r += x1.foo + r += x1.foo + return r + + res = self.interpret_inevitable(f1, [1]) + assert res == ['getfield'] + + + def test_not_for_local_raw(self): + py.test.skip("not yet") + X = lltype.Struct('X', ('foo', lltype.Signed)) + + def f1(i): + x1 = lltype.malloc(X, flavor='raw') + x1.foo = 42 + r = x1.foo + lltype.free(x1, flavor='raw') + return r + + res = self.interpret_inevitable(f1, [1]) + assert res == [] + + + def test_for_unknown_raw(self): + py.test.skip("not yet") + X = lltype.Struct('X', ('foo', lltype.Signed)) + + def f1(i): + x1 = lltype.malloc(X, flavor='raw') + x1.foo = 42 + r = x1.foo + if i: + lltype.free(x1, flavor='raw') + return r + + res = self.interpret_inevitable(f1, [1]) + assert res == ['setfield', 'getfield'] + + + def test_local_raw_in_same_transaction(self): + py.test.skip("not yet") + X = lltype.Struct('X', ('foo', lltype.Signed)) + + def f1(i): + x1 = lltype.malloc(X, flavor='raw') + x1.foo = 42 + r = x1.foo + func() # gil-release, non-gil-release, random-gc-effects???? + lltype.free(x1, flavor='raw') + return r + + res = self.interpret_inevitable(f1, [1]) + assert res == [] From noreply at buildbot.pypy.org Wed Mar 11 10:24:17 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Wed, 11 Mar 2015 10:24:17 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c8: Merge Message-ID: <20150311092417.3417E1C1159@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: stmgc-c8 Changeset: r76320:c8ea66315864 Date: 2015-03-11 10:23 +0100 http://bitbucket.org/pypy/pypy/changeset/c8ea66315864/ Log: Merge diff --git a/pypy/stm/print_stm_log.py b/pypy/stm/print_stm_log.py --- a/pypy/stm/print_stm_log.py +++ b/pypy/stm/print_stm_log.py @@ -91,7 +91,10 @@ self.cpu_time_committed = 0.0 self.cpu_time_aborted = 0.0 self.cpu_time_paused = 0.0 + self.cpu_time_gc_minor = 0.0 + self.cpu_time_gc_major = 0.0 self._prev = (0.0, "stop") + self._in_major_coll = None self.reset_counters() def reset_counters(self): @@ -99,6 +102,7 @@ self._transaction_pause_time = 0.0 self._transaction_aborting = False self._transaction_inev = None + self._in_minor_coll = None def transaction_start(self, entry): self.reset_counters() @@ -167,9 +171,29 @@ def wait_for_other_inev(self, wait_time, out_conflicts): c = self.get_conflict(self._transaction_inev[0], out_conflicts) - assert wait_time >= 0 + assert wait_time >= 0.0 c.paused_time += wait_time + def gc_minor_start(self, event): + self._in_minor_coll = event.timestamp + + def gc_minor_done(self, event): + if self._in_minor_coll is not None: + gc_time = event.timestamp - self._in_minor_coll + assert gc_time >= 0.0 + self.cpu_time_gc_minor += gc_time + self._in_minor_coll = None + + def gc_major_start(self, event): + self._in_major_coll = event.timestamp + + def gc_major_done(self, event): + if self._in_major_coll is not None: + gc_time = event.timestamp - self._in_major_coll + assert gc_time >= 0.0 + self.cpu_time_gc_major += gc_time + self._in_major_coll = None + class ConflictSummary(object): def __init__(self, event, marker): @@ -250,6 +274,14 @@ t.transaction_pause(entry) elif entry.event == STM_WAIT_DONE: t.transaction_unpause(entry, conflicts) + elif entry.event == STM_GC_MINOR_START: + t.gc_minor_start(entry) + elif entry.event == STM_GC_MINOR_DONE: + t.gc_minor_done(entry) + elif entry.event == STM_GC_MAJOR_START: + t.gc_major_start(entry) + elif entry.event == STM_GC_MAJOR_DONE: + t.gc_major_done(entry) # if cnt == 0: raise Exception("empty file") @@ -264,7 +296,7 @@ start_time = stmlog.start_time total_time = stmlog.total_time print - print 'Total real time: %.3fs' % (total_time,) + print 'Total real time: %9.3fs' % (total_time,) # total_cpu_time_committed = stmlog.get_total_cpu_time_committed() total_cpu_time_aborted = stmlog.get_total_cpu_time_aborted() @@ -272,14 +304,20 @@ total_cpu_time_total = (total_cpu_time_committed + total_cpu_time_aborted + total_cpu_time_paused) - print 'CPU time in STM mode: %.3fs (%s) committed' % ( + total_cpu_time_gc_minor = stmlog.get_total_cpu_time_gc_minor() + total_cpu_time_gc_major = stmlog.get_total_cpu_time_gc_major() + print 'CPU time in STM mode:%9.3fs (%4s) committed' % ( total_cpu_time_committed, percent(total_cpu_time_committed, total_time)) - print ' %.3fs (%s) aborted' % ( + print ' %9.3fs (%4s) aborted' % ( total_cpu_time_aborted, percent(total_cpu_time_aborted, total_time)) - print ' %.3fs (%s) paused' % ( + print ' %9.3fs (%4s) paused' % ( total_cpu_time_paused, percent(total_cpu_time_paused, total_time)) - print ' %.3fs (%s) total' % ( + print ' %9.3fs (%4s) TOTAL' % ( total_cpu_time_total, percent(total_cpu_time_total, total_time)) + print ' including %9.3fs (%4s) minor GC collections' % ( + total_cpu_time_gc_minor, percent(total_cpu_time_gc_minor, total_time)) + print ' and %9.3fs (%4s) major GC collections' % ( + total_cpu_time_gc_major, percent(total_cpu_time_gc_major, total_time)) print # values = stmlog.get_conflicts() @@ -308,6 +346,12 @@ def get_total_cpu_time_paused(self): return sum([v.cpu_time_paused for v in self.threads.values()]) + def get_total_cpu_time_gc_minor(self): + return sum([v.cpu_time_gc_minor for v in self.threads.values()]) + + def get_total_cpu_time_gc_major(self): + return sum([v.cpu_time_gc_major for v in self.threads.values()]) + def get_conflicts(self): values = self.conflicts.values() values.sort(key=ConflictSummary.sortkey) From noreply at buildbot.pypy.org Wed Mar 11 10:24:20 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 11 Mar 2015 10:24:20 +0100 (CET) Subject: [pypy-commit] stmgc c8-hashtable: We only need one privatization lock at once Message-ID: <20150311092420.198271C1159@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-hashtable Changeset: r1715:781a83c7d30f Date: 2015-03-11 10:24 +0100 http://bitbucket.org/pypy/stmgc/changeset/781a83c7d30f/ Log: We only need one privatization lock at once diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -146,14 +146,14 @@ memcpy(dest, initial_data, size_rounded_up); ((struct object_s *)dest)->stm_flags = GCFLAG_WRITE_BARRIER; - acquire_all_privatization_locks(); - long j; for (j = 1; j <= NB_SEGMENTS; j++) { const char *src = initial_data; char *dest = get_segment_base(j) + nobj; char *end = dest + size_rounded_up; + acquire_privatization_lock(j); + while (((uintptr_t)dest) / 4096 != ((uintptr_t)end - 1) / 4096) { uintptr_t count = 4096 - ((uintptr_t)dest) / 4096; _fill_preexisting_slice(j, dest, src, count); @@ -169,10 +169,9 @@ assert(!was_read_remote(get_segment_base(j), (object_t *)nobj)); } #endif + release_privatization_lock(j); } - release_all_privatization_locks(); - write_fence(); /* make sure 'nobj' is fully initialized from all threads here */ return (object_t *)nobj; From noreply at buildbot.pypy.org Wed Mar 11 10:28:09 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 11 Mar 2015 10:28:09 +0100 (CET) Subject: [pypy-commit] stmgc c8-hashtable: fix (thanks Remi) Message-ID: <20150311092809.941C41C0823@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-hashtable Changeset: r1716:e67451e5794e Date: 2015-03-11 10:28 +0100 http://bitbucket.org/pypy/stmgc/changeset/e67451e5794e/ Log: fix (thanks Remi) diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -147,7 +147,7 @@ ((struct object_s *)dest)->stm_flags = GCFLAG_WRITE_BARRIER; long j; - for (j = 1; j <= NB_SEGMENTS; j++) { + for (j = 1; j < NB_SEGMENTS; j++) { const char *src = initial_data; char *dest = get_segment_base(j) + nobj; char *end = dest + size_rounded_up; From noreply at buildbot.pypy.org Wed Mar 11 10:36:31 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 11 Mar 2015 10:36:31 +0100 (CET) Subject: [pypy-commit] stmgc c8-hashtable: Initialize the stm_flags of the copies too (by copying the seg0 Message-ID: <20150311093631.66CC81C0823@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-hashtable Changeset: r1717:9099a4a94924 Date: 2015-03-11 10:37 +0100 http://bitbucket.org/pypy/stmgc/changeset/9099a4a94924/ Log: Initialize the stm_flags of the copies too (by copying the seg0 version) (thanks Remi) diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -142,13 +142,15 @@ uintptr_t nobj = (uintptr_t)np; dprintf(("allocate_preexisting: %p\n", (object_t *)nobj)); - char *dest = stm_object_pages + nobj; - memcpy(dest, initial_data, size_rounded_up); - ((struct object_s *)dest)->stm_flags = GCFLAG_WRITE_BARRIER; + DEBUG_EXPECT_SEGFAULT(false); + + char *nobj_seg0 = stm_object_pages + nobj; + memcpy(nobj_seg0, initial_data, size_rounded_up); + ((struct object_s *)nobj_seg0)->stm_flags = GCFLAG_WRITE_BARRIER; long j; for (j = 1; j < NB_SEGMENTS; j++) { - const char *src = initial_data; + const char *src = nobj_seg0; char *dest = get_segment_base(j) + nobj; char *end = dest + size_rounded_up; @@ -174,6 +176,7 @@ write_fence(); /* make sure 'nobj' is fully initialized from all threads here */ + DEBUG_EXPECT_SEGFAULT(true); return (object_t *)nobj; } From noreply at buildbot.pypy.org Wed Mar 11 11:31:17 2015 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 11 Mar 2015 11:31:17 +0100 (CET) Subject: [pypy-commit] pypy default: free the array only if it was initialized Message-ID: <20150311103117.D88481C0318@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r76321:a280f6ad333c Date: 2015-03-11 12:27 +0200 http://bitbucket.org/pypy/pypy/changeset/a280f6ad333c/ Log: free the array only if it was initialized diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -700,7 +700,8 @@ @rgc.must_be_light_finalizer def __del__(self): - lltype.free(self.buffer, flavor='raw') + if self.buffer: + lltype.free(self.buffer, flavor='raw') def setlen(self, size, zero=False, overallocate=True): if size > 0: From noreply at buildbot.pypy.org Wed Mar 11 11:31:21 2015 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 11 Mar 2015 11:31:21 +0100 (CET) Subject: [pypy-commit] pypy default: merge Message-ID: <20150311103121.3C3241C0318@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r76322:98040ad7de44 Date: 2015-03-11 12:28 +0200 http://bitbucket.org/pypy/pypy/changeset/98040ad7de44/ Log: merge diff too long, truncating to 2000 out of 3245 lines diff --git a/lib_pypy/cffi.egg-info b/lib_pypy/cffi.egg-info --- a/lib_pypy/cffi.egg-info +++ b/lib_pypy/cffi.egg-info @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: cffi -Version: 0.8.6+ +Version: 0.9.0 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "0.8.6+" -__version_info__ = (0, 8, 6, "plus") +__version__ = "0.9.0" +__version_info__ = (0, 9, 0) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -34,3 +34,6 @@ .. branch: nonquadratic-heapcache Speed up the warmup times of the JIT by removing a quadratic algorithm in the heapcache. + +.. branch: online-transforms-2 +Simplify flow graphs on the fly during annotation phase. diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -8,7 +8,7 @@ appleveldefs = { } interpleveldefs = { - '__version__': 'space.wrap("0.8.6+")', + '__version__': 'space.wrap("0.9.0")', 'load_library': 'libraryobj.load_library', diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py --- a/pypy/module/_cffi_backend/cbuffer.py +++ b/pypy/module/_cffi_backend/cbuffer.py @@ -81,4 +81,5 @@ if size < 0: raise oefmt(space.w_TypeError, "don't know the size pointed to by '%s'", ctype.name) - return space.wrap(MiniBuffer(LLBuffer(w_cdata._cdata, size), w_cdata)) + ptr = w_cdata.unsafe_escaping_ptr() # w_cdata kept alive by MiniBuffer() + return space.wrap(MiniBuffer(LLBuffer(ptr, size), w_cdata)) diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py --- a/pypy/module/_cffi_backend/ccallback.py +++ b/pypy/module/_cffi_backend/ccallback.py @@ -48,9 +48,12 @@ raise oefmt(space.w_NotImplementedError, "%s: callback with unsupported argument or " "return type or with '...'", self.getfunctype().name) - res = clibffi.c_ffi_prep_closure(self.get_closure(), cif_descr.cif, - invoke_callback, - rffi.cast(rffi.VOIDP, self.unique_id)) + with self as ptr: + closure_ptr = rffi.cast(clibffi.FFI_CLOSUREP, ptr) + unique_id = rffi.cast(rffi.VOIDP, self.unique_id) + res = clibffi.c_ffi_prep_closure(closure_ptr, cif_descr.cif, + invoke_callback, + unique_id) if rffi.cast(lltype.Signed, res) != clibffi.FFI_OK: raise OperationError(space.w_SystemError, space.wrap("libffi failed to build this callback")) @@ -62,12 +65,9 @@ from pypy.module.thread.os_thread import setup_threads setup_threads(space) - def get_closure(self): - return rffi.cast(clibffi.FFI_CLOSUREP, self._cdata) - #@rgc.must_be_light_finalizer def __del__(self): - clibffi.closureHeap.free(self.get_closure()) + clibffi.closureHeap.free(rffi.cast(clibffi.FFI_CLOSUREP, self._ptr)) if self.ll_error: lltype.free(self.ll_error, flavor='raw') @@ -106,7 +106,7 @@ fresult = self.getfunctype().ctitem if fresult.size > 0: misc._raw_memcopy(self.ll_error, ll_res, fresult.size) - keepalive_until_here(self) + keepalive_until_here(self) # to keep self.ll_error alive global_callback_mapping = rweakref.RWeakValueDictionary(int, W_CDataCallback) diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py --- a/pypy/module/_cffi_backend/cdataobj.py +++ b/pypy/module/_cffi_backend/cdataobj.py @@ -14,21 +14,37 @@ class W_CData(W_Root): - _attrs_ = ['space', '_cdata', 'ctype', '_lifeline_'] - _immutable_fields_ = ['_cdata', 'ctype'] - _cdata = lltype.nullptr(rffi.CCHARP.TO) + _attrs_ = ['space', '_ptr', 'ctype', '_lifeline_'] + _immutable_fields_ = ['_ptr', 'ctype'] + _ptr = lltype.nullptr(rffi.CCHARP.TO) - def __init__(self, space, cdata, ctype): + def __init__(self, space, ptr, ctype): from pypy.module._cffi_backend import ctypeobj - assert lltype.typeOf(cdata) == rffi.CCHARP + assert lltype.typeOf(ptr) == rffi.CCHARP assert isinstance(ctype, ctypeobj.W_CType) self.space = space - self._cdata = cdata # don't forget keepalive_until_here! + self._ptr = ptr # don't access directly! use "with cdata as ptr:" self.ctype = ctype + def __enter__(self): + """Use 'with cdata as ptr:' to access the raw memory. It will + stay alive at least until the end of the 'with' block. + """ + return self._ptr + + def __exit__(self, *args): + keepalive_until_here(self) + + def unsafe_escaping_ptr(self): + """Generally unsafe: escape the pointer to raw memory. + If 'self' is a subclass that frees the pointer in a destructor, + it may be freed under your feet at any time. + """ + return self._ptr + def _repr_extra(self): - extra = self.ctype.extra_repr(self._cdata) - keepalive_until_here(self) + with self as ptr: + extra = self.ctype.extra_repr(ptr) return extra def _repr_extra_owning(self): @@ -54,11 +70,13 @@ self.ctype.name, extra1, extra2)) def nonzero(self): - return self.space.wrap(bool(self._cdata)) + with self as ptr: + nonzero = bool(ptr) + return self.space.wrap(nonzero) def int(self, space): - w_result = self.ctype.cast_to_int(self._cdata) - keepalive_until_here(self) + with self as ptr: + w_result = self.ctype.cast_to_int(ptr) return w_result def long(self, space): @@ -69,8 +87,8 @@ return w_result def float(self): - w_result = self.ctype.float(self._cdata) - keepalive_until_here(self) + with self as ptr: + w_result = self.ctype.float(ptr) return w_result def len(self): @@ -88,20 +106,19 @@ def _cmp(self, w_other): from pypy.module._cffi_backend.ctypeprim import W_CTypePrimitive space = self.space - cdata1 = self._cdata - if isinstance(w_other, W_CData): - cdata2 = w_other._cdata - else: + if not isinstance(w_other, W_CData): return space.w_NotImplemented - if requires_ordering: - if (isinstance(self.ctype, W_CTypePrimitive) or - isinstance(w_other.ctype, W_CTypePrimitive)): - raise OperationError(space.w_TypeError, - space.wrap("cannot do comparison on a primitive cdata")) - cdata1 = rffi.cast(lltype.Unsigned, cdata1) - cdata2 = rffi.cast(lltype.Unsigned, cdata2) - return space.newbool(op(cdata1, cdata2)) + with self as ptr1, w_other as ptr2: + if requires_ordering: + if (isinstance(self.ctype, W_CTypePrimitive) or + isinstance(w_other.ctype, W_CTypePrimitive)): + raise OperationError(space.w_TypeError, space.wrap( + "cannot do comparison on a primitive cdata")) + ptr1 = rffi.cast(lltype.Unsigned, ptr1) + ptr2 = rffi.cast(lltype.Unsigned, ptr2) + result = op(ptr1, ptr2) + return space.newbool(result) # return func_with_new_name(_cmp, name) @@ -113,7 +130,8 @@ ge = _make_comparison('ge') def hash(self): - h = rffi.cast(lltype.Signed, self._cdata) + ptr = self.unsafe_escaping_ptr() + h = rffi.cast(lltype.Signed, ptr) # To hash pointers in dictionaries. Assumes that h shows some # alignment (to 4, 8, maybe 16 bytes), so we use the following # formula to avoid the trailing bits being always 0. @@ -128,26 +146,27 @@ i = space.getindex_w(w_index, space.w_IndexError) ctype = self.ctype._check_subscript_index(self, i) w_o = self._do_getitem(ctype, i) - keepalive_until_here(self) return w_o def _do_getitem(self, ctype, i): ctitem = ctype.ctitem - return ctitem.convert_to_object( - rffi.ptradd(self._cdata, i * ctitem.size)) + with self as ptr: + return ctitem.convert_to_object( + rffi.ptradd(ptr, i * ctitem.size)) def setitem(self, w_index, w_value): space = self.space if space.isinstance_w(w_index, space.w_slice): - self._do_setslice(w_index, w_value) + with self as ptr: + self._do_setslice(w_index, w_value, ptr) else: i = space.getindex_w(w_index, space.w_IndexError) ctype = self.ctype._check_subscript_index(self, i) ctitem = ctype.ctitem - ctitem.convert_from_object( - rffi.ptradd(self._cdata, i * ctitem.size), - w_value) - keepalive_until_here(self) + with self as ptr: + ctitem.convert_from_object( + rffi.ptradd(ptr, i * ctitem.size), + w_value) def _do_getslicearg(self, w_slice): from pypy.module._cffi_backend.ctypeptr import W_CTypePointer @@ -188,14 +207,15 @@ ctarray = newtype.new_array_type(space, ctptr, space.w_None) ctptr.cache_array_type = ctarray # - p = rffi.ptradd(self._cdata, start * ctarray.ctitem.size) - return W_CDataSliced(space, p, ctarray, length) + ptr = self.unsafe_escaping_ptr() + ptr = rffi.ptradd(ptr, start * ctarray.ctitem.size) + return W_CDataSliced(space, ptr, ctarray, length) - def _do_setslice(self, w_slice, w_value): + def _do_setslice(self, w_slice, w_value, ptr): ctptr, start, length = self._do_getslicearg(w_slice) ctitem = ctptr.ctitem ctitemsize = ctitem.size - cdata = rffi.ptradd(self._cdata, start * ctitemsize) + target = rffi.ptradd(ptr, start * ctitemsize) # if isinstance(w_value, W_CData): from pypy.module._cffi_backend import ctypearray @@ -204,9 +224,8 @@ ctv.ctitem is ctitem and w_value.get_array_length() == length): # fast path: copying from exactly the correct type - s = w_value._cdata - rffi.c_memcpy(cdata, s, ctitemsize * length) - keepalive_until_here(w_value) + with w_value as source: + rffi.c_memcpy(target, source, ctitemsize * length) return # # A fast path for [0:N] = "somestring". @@ -221,7 +240,7 @@ raise oefmt(space.w_ValueError, "need a string of length %d, got %d", length, len(value)) - copy_string_to_raw(llstr(value), cdata, 0, length) + copy_string_to_raw(llstr(value), target, 0, length) return # w_iter = space.iter(w_value) @@ -233,8 +252,8 @@ raise raise oefmt(space.w_ValueError, "need %d values to unpack, got %d", length, i) - ctitem.convert_from_object(cdata, w_item) - cdata = rffi.ptradd(cdata, ctitemsize) + ctitem.convert_from_object(target, w_item) + target = rffi.ptradd(target, ctitemsize) try: space.next(w_iter) except OperationError, e: @@ -247,7 +266,8 @@ def _add_or_sub(self, w_other, sign): space = self.space i = sign * space.getindex_w(w_other, space.w_OverflowError) - return self.ctype.add(self._cdata, i) + ptr = self.unsafe_escaping_ptr() + return self.ctype.add(ptr, i) def add(self, w_other): return self._add_or_sub(w_other, +1) @@ -268,9 +288,11 @@ self.ctype.name, ct.name) # itemsize = ct.ctitem.size - if itemsize <= 0: itemsize = 1 - diff = (rffi.cast(lltype.Signed, self._cdata) - - rffi.cast(lltype.Signed, w_other._cdata)) // itemsize + if itemsize <= 0: + itemsize = 1 + with self as ptr1, w_other as ptr2: + diff = (rffi.cast(lltype.Signed, ptr1) - + rffi.cast(lltype.Signed, ptr2)) // itemsize return space.wrap(diff) # return self._add_or_sub(w_other, -1) @@ -279,17 +301,19 @@ return self.ctype.getcfield(self.space.str_w(w_attr)) def getattr(self, w_attr): - w_res = self.getcfield(w_attr).read(self._cdata) - keepalive_until_here(self) + cfield = self.getcfield(w_attr) + with self as ptr: + w_res = cfield.read(ptr) return w_res def setattr(self, w_attr, w_value): - self.getcfield(w_attr).write(self._cdata, w_value) - keepalive_until_here(self) + cfield = self.getcfield(w_attr) + with self as ptr: + cfield.write(ptr, w_value) def call(self, args_w): - w_result = self.ctype.call(self._cdata, args_w) - keepalive_until_here(self) + with self as ptr: + w_result = self.ctype.call(ptr, args_w) return w_result def iter(self): @@ -311,21 +335,21 @@ @specialize.argtype(1) def write_raw_signed_data(self, source): - misc.write_raw_signed_data(self._cdata, source, self.ctype.size) - keepalive_until_here(self) + with self as ptr: + misc.write_raw_signed_data(ptr, source, self.ctype.size) @specialize.argtype(1) def write_raw_unsigned_data(self, source): - misc.write_raw_unsigned_data(self._cdata, source, self.ctype.size) - keepalive_until_here(self) + with self as ptr: + misc.write_raw_unsigned_data(ptr, source, self.ctype.size) def write_raw_float_data(self, source): - misc.write_raw_float_data(self._cdata, source, self.ctype.size) - keepalive_until_here(self) + with self as ptr: + misc.write_raw_float_data(ptr, source, self.ctype.size) def convert_to_object(self): - w_obj = self.ctype.convert_to_object(self._cdata) - keepalive_until_here(self) + with self as ptr: + w_obj = self.ctype.convert_to_object(ptr) return w_obj def get_array_length(self): @@ -353,7 +377,7 @@ @rgc.must_be_light_finalizer def __del__(self): - lltype.free(self._cdata, flavor='raw') + lltype.free(self._ptr, flavor='raw') class W_CDataNewOwning(W_CDataMem): diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py --- a/pypy/module/_cffi_backend/ctypearray.py +++ b/pypy/module/_cffi_backend/ctypearray.py @@ -8,7 +8,6 @@ from pypy.interpreter.typedef import TypeDef from rpython.rtyper.lltypesystem import rffi -from rpython.rlib.objectmodel import keepalive_until_here from rpython.rlib.rarithmetic import ovfcheck from pypy.module._cffi_backend import cdataobj @@ -49,8 +48,8 @@ cdata = cdataobj.W_CDataNewOwning(space, datasize, self) # if not space.is_w(w_init, space.w_None): - self.convert_from_object(cdata._cdata, w_init) - keepalive_until_here(cdata) + with cdata as ptr: + self.convert_from_object(ptr, w_init) return cdata def _check_subscript_index(self, w_cdata, i): @@ -119,8 +118,8 @@ self.ctitem = ctitem self.cdata = cdata length = cdata.get_array_length() - self._next = cdata._cdata - self._stop = rffi.ptradd(cdata._cdata, length * ctitem.size) + self._next = cdata.unsafe_escaping_ptr() + self._stop = rffi.ptradd(self._next, length * ctitem.size) def iter_w(self): return self.space.wrap(self) diff --git a/pypy/module/_cffi_backend/ctypeenum.py b/pypy/module/_cffi_backend/ctypeenum.py --- a/pypy/module/_cffi_backend/ctypeenum.py +++ b/pypy/module/_cffi_backend/ctypeenum.py @@ -2,8 +2,6 @@ Enums. """ -from rpython.rlib.objectmodel import keepalive_until_here - from pypy.module._cffi_backend import misc from pypy.module._cffi_backend.ctypeprim import (W_CTypePrimitiveSigned, W_CTypePrimitiveUnsigned) @@ -47,8 +45,8 @@ return '%s: %s' % (value, s) def string(self, cdataobj, maxlen): - value = self._get_value(cdataobj._cdata) - keepalive_until_here(cdataobj) + with cdataobj as ptr: + value = self._get_value(ptr) try: s = self.enumvalues2erators[value] except KeyError: diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py --- a/pypy/module/_cffi_backend/ctypeobj.py +++ b/pypy/module/_cffi_backend/ctypeobj.py @@ -177,8 +177,8 @@ raise oefmt(space.w_AttributeError, "cdata '%s' has no attribute '%s'", self.name, attr) - def copy_and_convert_to_object(self, cdata): - return self.convert_to_object(cdata) + def copy_and_convert_to_object(self, source): + return self.convert_to_object(source) # __________ app-level attributes __________ def dir(self): diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py --- a/pypy/module/_cffi_backend/ctypeprim.py +++ b/pypy/module/_cffi_backend/ctypeprim.py @@ -5,7 +5,6 @@ import sys from rpython.rlib.rarithmetic import r_uint, r_ulonglong, intmask -from rpython.rlib.objectmodel import keepalive_until_here from rpython.rlib import jit from rpython.rtyper.lltypesystem import lltype, rffi @@ -53,7 +52,8 @@ space = self.space if (isinstance(w_ob, cdataobj.W_CData) and isinstance(w_ob.ctype, ctypeptr.W_CTypePtrOrArray)): - value = rffi.cast(lltype.Signed, w_ob._cdata) + ptr = w_ob.unsafe_escaping_ptr() + value = rffi.cast(lltype.Signed, ptr) value = self._cast_result(value) elif space.isinstance_w(w_ob, space.w_str): value = self.cast_str(w_ob) @@ -81,8 +81,8 @@ def string(self, cdataobj, maxlen): if self.size == 1: - s = cdataobj._cdata[0] - keepalive_until_here(cdataobj) + with cdataobj as ptr: + s = ptr[0] return self.space.wrap(s) return W_CType.string(self, cdataobj, maxlen) @@ -116,7 +116,8 @@ return s[0] if (isinstance(w_ob, cdataobj.W_CData) and isinstance(w_ob.ctype, W_CTypePrimitiveChar)): - return w_ob._cdata[0] + with w_ob as ptr: + return ptr[0] raise self._convert_error("string of length 1", w_ob) def convert_from_object(self, cdata, w_ob): @@ -137,8 +138,8 @@ return self.space.wrap(s) def string(self, cdataobj, maxlen): - w_res = self.convert_to_object(cdataobj._cdata) - keepalive_until_here(cdataobj) + with cdataobj as ptr: + w_res = self.convert_to_object(ptr) return w_res def _convert_to_unichar(self, w_ob): @@ -149,7 +150,8 @@ return s[0] if (isinstance(w_ob, cdataobj.W_CData) and isinstance(w_ob.ctype, W_CTypePrimitiveUniChar)): - return rffi.cast(rffi.CWCHARP, w_ob._cdata)[0] + with w_ob as ptr: + return rffi.cast(rffi.CWCHARP, ptr)[0] raise self._convert_error("unicode string of length 1", w_ob) def convert_from_object(self, cdata, w_ob): @@ -219,13 +221,15 @@ if self.size == rffi.sizeof(rffi.LONG): from rpython.rlib.rrawarray import populate_list_from_raw_array res = [] - buf = rffi.cast(rffi.LONGP, w_cdata._cdata) length = w_cdata.get_array_length() - populate_list_from_raw_array(res, buf, length) + with w_cdata as ptr: + buf = rffi.cast(rffi.LONGP, ptr) + populate_list_from_raw_array(res, buf, length) return res elif self.value_smaller_than_long: res = [0] * w_cdata.get_array_length() - misc.unpack_list_from_raw_array(res, w_cdata._cdata, self.size) + with w_cdata as ptr: + misc.unpack_list_from_raw_array(res, ptr, self.size) return res return None @@ -308,8 +312,8 @@ def unpack_list_of_int_items(self, w_cdata): if self.value_fits_long: res = [0] * w_cdata.get_array_length() - misc.unpack_unsigned_list_from_raw_array(res, w_cdata._cdata, - self.size) + with w_cdata as ptr: + misc.unpack_unsigned_list_from_raw_array(res, ptr, self.size) return res return None @@ -363,8 +367,8 @@ if not isinstance(self, W_CTypePrimitiveLongDouble): w_cdata.write_raw_float_data(value) else: - self._to_longdouble_and_write(value, w_cdata._cdata) - keepalive_until_here(w_cdata) + with w_cdata as ptr: + self._to_longdouble_and_write(value, ptr) return w_cdata def cast_to_int(self, cdata): @@ -387,13 +391,15 @@ if self.size == rffi.sizeof(rffi.DOUBLE): from rpython.rlib.rrawarray import populate_list_from_raw_array res = [] - buf = rffi.cast(rffi.DOUBLEP, w_cdata._cdata) length = w_cdata.get_array_length() - populate_list_from_raw_array(res, buf, length) + with w_cdata as ptr: + buf = rffi.cast(rffi.DOUBLEP, ptr) + populate_list_from_raw_array(res, buf, length) return res elif self.size == rffi.sizeof(rffi.FLOAT): res = [0.0] * w_cdata.get_array_length() - misc.unpack_cfloat_list_from_raw_array(res, w_cdata._cdata) + with w_cdata as ptr: + misc.unpack_cfloat_list_from_raw_array(res, ptr) return res return None @@ -423,8 +429,8 @@ def cast(self, w_ob): if (isinstance(w_ob, cdataobj.W_CData) and isinstance(w_ob.ctype, W_CTypePrimitiveLongDouble)): - w_cdata = self.convert_to_object(w_ob._cdata) - keepalive_until_here(w_ob) + with w_ob as ptr: + w_cdata = self.convert_to_object(ptr) return w_cdata else: return W_CTypePrimitiveFloat.cast(self, w_ob) @@ -451,16 +457,16 @@ def convert_to_object(self, cdata): w_cdata = cdataobj.W_CDataMem(self.space, self.size, self) - self._copy_longdouble(cdata, w_cdata._cdata) - keepalive_until_here(w_cdata) + with w_cdata as ptr: + self._copy_longdouble(cdata, ptr) return w_cdata def convert_from_object(self, cdata, w_ob): space = self.space if (isinstance(w_ob, cdataobj.W_CData) and isinstance(w_ob.ctype, W_CTypePrimitiveLongDouble)): - self._copy_longdouble(w_ob._cdata, cdata) - keepalive_until_here(w_ob) + with w_ob as ptr: + self._copy_longdouble(ptr, cdata) else: value = space.float_w(space.float(w_ob)) self._to_longdouble_and_write(value, cdata) diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py --- a/pypy/module/_cffi_backend/ctypeptr.py +++ b/pypy/module/_cffi_backend/ctypeptr.py @@ -3,7 +3,6 @@ """ from rpython.rlib import rposix -from rpython.rlib.objectmodel import keepalive_until_here from rpython.rlib.rarithmetic import ovfcheck from rpython.rtyper.annlowlevel import llstr, llunicode from rpython.rtyper.lltypesystem import lltype, rffi @@ -49,7 +48,7 @@ space = self.space if (isinstance(w_ob, cdataobj.W_CData) and isinstance(w_ob.ctype, W_CTypePtrOrArray)): - value = w_ob._cdata + value = w_ob.unsafe_escaping_ptr() else: value = misc.as_unsigned_long(space, w_ob, strict=False) value = rffi.cast(rffi.CCHARP, value) @@ -108,34 +107,33 @@ def string(self, cdataobj, maxlen): space = self.space if isinstance(self.ctitem, ctypeprim.W_CTypePrimitive): - cdata = cdataobj._cdata - if not cdata: - raise oefmt(space.w_RuntimeError, "cannot use string() on %s", - space.str_w(cdataobj.repr())) - # - from pypy.module._cffi_backend import ctypearray - length = maxlen - if length < 0 and isinstance(self, ctypearray.W_CTypeArray): - length = cdataobj.get_array_length() - # - # pointer to a primitive type of size 1: builds and returns a str - if self.ctitem.size == rffi.sizeof(lltype.Char): - if length < 0: - s = rffi.charp2str(cdata) - else: - s = rffi.charp2strn(cdata, length) - keepalive_until_here(cdataobj) - return space.wrap(s) - # - # pointer to a wchar_t: builds and returns a unicode - if self.is_unichar_ptr_or_array(): - cdata = rffi.cast(rffi.CWCHARP, cdata) - if length < 0: - u = rffi.wcharp2unicode(cdata) - else: - u = rffi.wcharp2unicoden(cdata, length) - keepalive_until_here(cdataobj) - return space.wrap(u) + with cdataobj as ptr: + if not ptr: + raise oefmt(space.w_RuntimeError, + "cannot use string() on %s", + space.str_w(cdataobj.repr())) + # + from pypy.module._cffi_backend import ctypearray + length = maxlen + if length < 0 and isinstance(self, ctypearray.W_CTypeArray): + length = cdataobj.get_array_length() + # + # pointer to a primitive type of size 1: builds and returns a str + if self.ctitem.size == rffi.sizeof(lltype.Char): + if length < 0: + s = rffi.charp2str(ptr) + else: + s = rffi.charp2strn(ptr, length) + return space.wrap(s) + # + # pointer to a wchar_t: builds and returns a unicode + if self.is_unichar_ptr_or_array(): + cdata = rffi.cast(rffi.CWCHARP, ptr) + if length < 0: + u = rffi.wcharp2unicode(cdata) + else: + u = rffi.wcharp2unicoden(cdata, length) + return space.wrap(u) # return W_CType.string(self, cdataobj, maxlen) @@ -162,7 +160,7 @@ if not (self.can_cast_anything or other.can_cast_anything): raise self._convert_error("compatible pointer", w_ob) - rffi.cast(rffi.CCHARPP, cdata)[0] = w_ob._cdata + rffi.cast(rffi.CCHARPP, cdata)[0] = w_ob.unsafe_escaping_ptr() def _alignof(self): from pypy.module._cffi_backend import newtype @@ -206,8 +204,8 @@ lltype.nullptr(rffi.CCHARP.TO), w_init, datasize) # cdatastruct = cdataobj.W_CDataNewOwning(space, datasize, ctitem) - cdata = cdataobj.W_CDataPtrToStructOrUnion(space, - cdatastruct._cdata, + ptr = cdatastruct.unsafe_escaping_ptr() + cdata = cdataobj.W_CDataPtrToStructOrUnion(space, ptr, self, cdatastruct) else: if self.is_char_or_unichar_ptr_or_array(): @@ -215,8 +213,8 @@ cdata = cdataobj.W_CDataNewOwning(space, datasize, self) # if not space.is_w(w_init, space.w_None): - ctitem.convert_from_object(cdata._cdata, w_init) - keepalive_until_here(cdata) + with cdata as ptr: + ctitem.convert_from_object(ptr, w_init) return cdata def _check_subscript_index(self, w_cdata, i): @@ -332,8 +330,9 @@ ctype2 = cdata.ctype if (isinstance(ctype2, W_CTypeStructOrUnion) or isinstance(ctype2, W_CTypePtrOrArray)): - ptrdata = rffi.ptradd(cdata._cdata, offset) - return cdataobj.W_CData(space, ptrdata, self) + ptr = cdata.unsafe_escaping_ptr() + ptr = rffi.ptradd(ptr, offset) + return cdataobj.W_CData(space, ptr, self) else: raise OperationError(space.w_TypeError, space.wrap("expected a cdata struct/union/array/pointer" diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py --- a/pypy/module/_cffi_backend/ctypestruct.py +++ b/pypy/module/_cffi_backend/ctypestruct.py @@ -7,7 +7,6 @@ from pypy.interpreter.typedef import TypeDef, interp_attrproperty from rpython.rlib import jit -from rpython.rlib.objectmodel import keepalive_until_here from rpython.rlib.rarithmetic import r_uint, r_ulonglong, r_longlong, intmask from rpython.rlib.rarithmetic import ovfcheck from rpython.rtyper.lltypesystem import lltype, rffi @@ -57,12 +56,12 @@ self.check_complete() return cdataobj.W_CData(space, cdata, self) - def copy_and_convert_to_object(self, cdata): + def copy_and_convert_to_object(self, source): space = self.space self.check_complete() ob = cdataobj.W_CDataNewOwning(space, self.size, self) - misc._raw_memcopy(cdata, ob._cdata, self.size) - keepalive_until_here(ob) + with ob as target: + misc._raw_memcopy(source, target, self.size) return ob def typeoffsetof_field(self, fieldname, following): @@ -80,8 +79,8 @@ def _copy_from_same(self, cdata, w_ob): if isinstance(w_ob, cdataobj.W_CData): if w_ob.ctype is self and self.size >= 0: - misc._raw_memcopy(w_ob._cdata, cdata, self.size) - keepalive_until_here(w_ob) + with w_ob as ptr: + misc._raw_memcopy(ptr, cdata, self.size) return True return False diff --git a/pypy/module/_cffi_backend/ctypevoid.py b/pypy/module/_cffi_backend/ctypevoid.py --- a/pypy/module/_cffi_backend/ctypevoid.py +++ b/pypy/module/_cffi_backend/ctypevoid.py @@ -13,5 +13,5 @@ def __init__(self, space): W_CType.__init__(self, space, -1, "void", len("void")) - def copy_and_convert_to_object(self, cdata): + def copy_and_convert_to_object(self, source): return self.space.w_None diff --git a/pypy/module/_cffi_backend/handle.py b/pypy/module/_cffi_backend/handle.py --- a/pypy/module/_cffi_backend/handle.py +++ b/pypy/module/_cffi_backend/handle.py @@ -34,8 +34,9 @@ raise oefmt(space.w_TypeError, "expected a 'cdata' object with a 'void *' out of " "new_handle(), got '%s'", ctype.name) - index = rffi.cast(lltype.Signed, w_cdata._cdata) - original_cdataobj = get(space).fetch_handle(index - 1) + with w_cdata as ptr: + index = rffi.cast(lltype.Signed, ptr) + original_cdataobj = get(space).fetch_handle(index - 1) # if isinstance(original_cdataobj, cdataobj.W_CDataHandle): return original_cdataobj.w_keepalive diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py --- a/pypy/module/_cffi_backend/misc.py +++ b/pypy/module/_cffi_backend/misc.py @@ -3,7 +3,7 @@ from pypy.interpreter.error import OperationError from rpython.rlib import jit -from rpython.rlib.objectmodel import keepalive_until_here, specialize +from rpython.rlib.objectmodel import specialize from rpython.rlib.rarithmetic import r_uint, r_ulonglong from rpython.rlib.unroll import unrolling_iterable from rpython.rtyper.lltypesystem import lltype, llmemory, rffi @@ -272,11 +272,11 @@ from pypy.module._cffi_backend.ctypeprim import W_CTypePrimitiveLongDouble is_cdata = isinstance(w_ob, W_CData) if is_cdata and isinstance(w_ob.ctype, W_CTypePrimitiveFloat): - if isinstance(w_ob.ctype, W_CTypePrimitiveLongDouble): - result = is_nonnull_longdouble(w_ob._cdata) - else: - result = is_nonnull_float(w_ob._cdata, w_ob.ctype.size) - keepalive_until_here(w_ob) + with w_ob as ptr: + if isinstance(w_ob.ctype, W_CTypePrimitiveLongDouble): + result = is_nonnull_longdouble(ptr) + else: + result = is_nonnull_float(ptr, w_ob.ctype.size) return result # if not is_cdata and space.lookup(w_ob, '__float__') is not None: diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -3249,4 +3249,4 @@ def test_version(): # this test is here mostly for PyPy - assert __version__ == "0.8.6+" + assert __version__ == "0.9.0" diff --git a/pypy/module/_random/test/test_random.py b/pypy/module/_random/test/test_random.py --- a/pypy/module/_random/test/test_random.py +++ b/pypy/module/_random/test/test_random.py @@ -102,3 +102,10 @@ self.x = x r = R(x=15) assert r.x == 15 + + def test_exact_result(self): + # this passes on CPython 2.7.9 on Linux 32 and Linux 64 + import _random + rnd = _random.Random(-3**31) + x = rnd.random() + assert x == 0.8181851342382107 diff --git a/pypy/module/cppyy/capi/loadable_capi.py b/pypy/module/cppyy/capi/loadable_capi.py --- a/pypy/module/cppyy/capi/loadable_capi.py +++ b/pypy/module/cppyy/capi/loadable_capi.py @@ -259,7 +259,8 @@ if not objectmodel.we_are_translated(): leakfinder.remember_free(c_call.ctype.cif_descr._obj0) state.capi_calls[name] = c_call - return c_call.ctype.rcall(c_call._cdata, args) + with c_call as ptr: + return c_call.ctype.rcall(ptr, args) def _cdata_to_cobject(space, w_cdata): return rffi.cast(C_OBJECT, space.uint_w(w_cdata)) @@ -271,8 +272,9 @@ return rffi.cast(rffi.LONG, space.int_w(w_cdata)) def _cdata_to_ptr(space, w_cdata): # TODO: this is both a hack and dreadfully slow - return rffi.cast(rffi.VOIDP, - space.interp_w(cdataobj.W_CData, w_cdata, can_be_None=False)._cdata) + w_cdata = space.interp_w(cdataobj.W_CData, w_cdata, can_be_None=False) + ptr = w_cdata.unsafe_escaping_ptr() + return rffi.cast(rffi.VOIDP, ptr) def c_load_dictionary(name): return libffi.CDLL(name) diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py --- a/pypy/module/micronumpy/base.py +++ b/pypy/module/micronumpy/base.py @@ -46,7 +46,7 @@ @staticmethod def from_shape_and_storage(space, shape, storage, dtype, storage_bytes=-1, order='C', owning=False, w_subtype=None, - w_base=None, writable=True, strides=None): + w_base=None, writable=True, strides=None, start=0): from pypy.module.micronumpy import concrete from pypy.module.micronumpy.strides import (calc_strides, calc_backstrides) @@ -75,8 +75,9 @@ raise OperationError(space.w_ValueError, space.wrap("Cannot have owning=True when specifying a buffer")) if writable: - impl = concrete.ConcreteArrayWithBase(shape, dtype, order, strides, - backstrides, storage, w_base) + impl = concrete.ConcreteArrayWithBase(shape, dtype, order, + strides, backstrides, storage, w_base, + start=start) else: impl = concrete.ConcreteNonWritableArrayWithBase(shape, dtype, order, strides, backstrides, @@ -128,6 +129,9 @@ def get_order(self): return self.implementation.order + def get_start(self): + return self.implementation.start + def ndims(self): return len(self.get_shape()) ndims._always_inline_ = True diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py --- a/pypy/module/micronumpy/concrete.py +++ b/pypy/module/micronumpy/concrete.py @@ -12,6 +12,7 @@ from pypy.module.micronumpy.strides import (Chunk, Chunks, NewAxisChunk, RecordChunk, calc_strides, calc_new_strides, shape_agreement, calculate_broadcast_strides, calc_backstrides) +from rpython.rlib.objectmodel import keepalive_until_here class BaseConcreteArray(object): @@ -312,12 +313,15 @@ l_w = [w_res.descr_getitem(space, space.wrap(d)) for d in range(nd)] return space.newtuple(l_w) - def get_storage_as_int(self, space): - return rffi.cast(lltype.Signed, self.storage) + self.start - - def get_storage(self): + ##def get_storage(self): + ## return self.storage + ## use a safer context manager + def __enter__(self): return self.storage + def __exit__(self, typ, value, traceback): + keepalive_until_here(self) + def get_buffer(self, space, readonly): return ArrayBuffer(self, readonly) @@ -331,7 +335,7 @@ class ConcreteArrayNotOwning(BaseConcreteArray): - def __init__(self, shape, dtype, order, strides, backstrides, storage): + def __init__(self, shape, dtype, order, strides, backstrides, storage, start=0): make_sure_not_resized(shape) make_sure_not_resized(strides) make_sure_not_resized(backstrides) @@ -342,6 +346,7 @@ self.strides = strides self.backstrides = backstrides self.storage = storage + self.start = start def fill(self, space, box): self.dtype.itemtype.fill(self.storage, self.dtype.elsize, @@ -350,7 +355,7 @@ def set_shape(self, space, orig_array, new_shape): strides, backstrides = calc_strides(new_shape, self.dtype, self.order) - return SliceArray(0, strides, backstrides, new_shape, self, + return SliceArray(self.start, strides, backstrides, new_shape, self, orig_array) def set_dtype(self, space, dtype): @@ -384,9 +389,10 @@ class ConcreteArrayWithBase(ConcreteArrayNotOwning): - def __init__(self, shape, dtype, order, strides, backstrides, storage, orig_base): + def __init__(self, shape, dtype, order, strides, backstrides, storage, + orig_base, start=0): ConcreteArrayNotOwning.__init__(self, shape, dtype, order, - strides, backstrides, storage) + strides, backstrides, storage, start) self.orig_base = orig_base def base(self): diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py --- a/pypy/module/micronumpy/ctors.py +++ b/pypy/module/micronumpy/ctors.py @@ -99,10 +99,12 @@ for i in range(w_object.get_size()): elems_w[i] = w_object.implementation.getitem(i * elsize) else: - sz = support.product(w_object.get_shape()) * dtype.elsize - return W_NDimArray.from_shape_and_storage(space, - w_object.get_shape(),w_object.implementation.storage, - dtype, storage_bytes=sz, w_base=w_object) + imp = w_object.implementation + with imp as storage: + sz = support.product(w_object.get_shape()) * dtype.elsize + return W_NDimArray.from_shape_and_storage(space, + w_object.get_shape(), storage, dtype, storage_bytes=sz, + w_base=w_object, start=imp.start) else: # not an array shape, elems_w = strides.find_shape_and_elems(space, w_object, dtype) diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py --- a/pypy/module/micronumpy/loop.py +++ b/pypy/module/micronumpy/loop.py @@ -604,14 +604,15 @@ iter, state = arr.create_iter() w_res_str = W_NDimArray.from_shape(space, [1], arr.get_dtype(), order='C') itemsize = arr.get_dtype().elsize - res_str_casted = rffi.cast(rffi.CArrayPtr(lltype.Char), - w_res_str.implementation.get_storage_as_int(space)) - while not iter.done(state): - w_res_str.implementation.setitem(0, iter.getitem(state)) - for i in range(itemsize): - builder.append(res_str_casted[i]) - state = iter.next(state) - return builder.build() + with w_res_str.implementation as storage: + res_str_casted = rffi.cast(rffi.CArrayPtr(lltype.Char), + support.get_storage_as_int(storage)) + while not iter.done(state): + w_res_str.implementation.setitem(0, iter.getitem(state)) + for i in range(itemsize): + builder.append(res_str_casted[i]) + state = iter.next(state) + return builder.build() getitem_int_driver = jit.JitDriver(name = 'numpy_getitem_int', greens = ['shapelen', 'indexlen', diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -532,20 +532,25 @@ self.get_dtype(), storage_bytes=sz, w_base=self) def descr_array_iface(self, space): - addr = self.implementation.get_storage_as_int(space) - # will explode if it can't - w_d = space.newdict() - space.setitem_str(w_d, 'data', - space.newtuple([space.wrap(addr), space.w_False])) - space.setitem_str(w_d, 'shape', self.descr_get_shape(space)) - space.setitem_str(w_d, 'typestr', self.get_dtype().descr_get_str(space)) - if self.implementation.order == 'C': - # Array is contiguous, no strides in the interface. - strides = space.w_None - else: - strides = self.descr_get_strides(space) - space.setitem_str(w_d, 'strides', strides) - return w_d + ''' + Note: arr.__array__.data[0] is a pointer so arr must be kept alive + while it is in use + ''' + with self.implementation as storage: + addr = support.get_storage_as_int(storage, self.get_start()) + # will explode if it can't + w_d = space.newdict() + space.setitem_str(w_d, 'data', + space.newtuple([space.wrap(addr), space.w_False])) + space.setitem_str(w_d, 'shape', self.descr_get_shape(space)) + space.setitem_str(w_d, 'typestr', self.get_dtype().descr_get_str(space)) + if self.implementation.order == 'C': + # Array is contiguous, no strides in the interface. + strides = space.w_None + else: + strides = self.descr_get_strides(space) + space.setitem_str(w_d, 'strides', strides) + return w_d w_pypy_data = None @@ -1165,7 +1170,8 @@ builder.append(box.raw_str()) state = iter.next(state) else: - builder.append_charpsize(self.implementation.get_storage(), + with self.implementation as storage: + builder.append_charpsize(storage, self.implementation.get_storage_size()) state = space.newtuple([ diff --git a/pypy/module/micronumpy/selection.py b/pypy/module/micronumpy/selection.py --- a/pypy/module/micronumpy/selection.py +++ b/pypy/module/micronumpy/selection.py @@ -33,14 +33,14 @@ self.values = values self.indexes = indexes - def getitem(self, item): + def getitem(self, idx): if count < 2: - v = raw_storage_getitem(TP, self.values, item * self.stride_size + v = raw_storage_getitem(TP, self.values, idx * self.stride_size + self.start) else: v = [] for i in range(count): - _v = raw_storage_getitem(TP, self.values, item * self.stride_size + _v = raw_storage_getitem(TP, self.values, idx * self.stride_size + self.start + step * i) v.append(_v) if comp_type == 'int': @@ -52,7 +52,7 @@ else: raise NotImplementedError('cannot reach') return (v, raw_storage_getitem(lltype.Signed, self.indexes, - item * self.index_stride_size + + idx * self.index_stride_size + self.index_start)) def setitem(self, idx, item): @@ -134,37 +134,37 @@ # create array of indexes dtype = descriptor.get_dtype_cache(space).w_longdtype index_arr = W_NDimArray.from_shape(space, arr.get_shape(), dtype) - storage = index_arr.implementation.get_storage() - if len(arr.get_shape()) == 1: - for i in range(arr.get_size()): - raw_storage_setitem(storage, i * INT_SIZE, i) - r = Repr(INT_SIZE, itemsize, arr.get_size(), arr.get_storage(), - storage, 0, arr.start) - ArgSort(r).sort() - else: - shape = arr.get_shape() - if axis < 0: - axis = len(shape) + axis - if axis < 0 or axis >= len(shape): - raise oefmt(space.w_IndexError, "Wrong axis %d", axis) - arr_iter = AllButAxisIter(arr, axis) - arr_state = arr_iter.reset() - index_impl = index_arr.implementation - index_iter = AllButAxisIter(index_impl, axis) - index_state = index_iter.reset() - stride_size = arr.strides[axis] - index_stride_size = index_impl.strides[axis] - axis_size = arr.shape[axis] - while not arr_iter.done(arr_state): - for i in range(axis_size): - raw_storage_setitem(storage, i * index_stride_size + - index_state.offset, i) - r = Repr(index_stride_size, stride_size, axis_size, - arr.get_storage(), storage, index_state.offset, arr_state.offset) + with index_arr.implementation as storage, arr as arr_storage: + if len(arr.get_shape()) == 1: + for i in range(arr.get_size()): + raw_storage_setitem(storage, i * INT_SIZE, i) + r = Repr(INT_SIZE, itemsize, arr.get_size(), arr_storage, + storage, 0, arr.start) ArgSort(r).sort() - arr_state = arr_iter.next(arr_state) - index_state = index_iter.next(index_state) - return index_arr + else: + shape = arr.get_shape() + if axis < 0: + axis = len(shape) + axis + if axis < 0 or axis >= len(shape): + raise oefmt(space.w_IndexError, "Wrong axis %d", axis) + arr_iter = AllButAxisIter(arr, axis) + arr_state = arr_iter.reset() + index_impl = index_arr.implementation + index_iter = AllButAxisIter(index_impl, axis) + index_state = index_iter.reset() + stride_size = arr.strides[axis] + index_stride_size = index_impl.strides[axis] + axis_size = arr.shape[axis] + while not arr_iter.done(arr_state): + for i in range(axis_size): + raw_storage_setitem(storage, i * index_stride_size + + index_state.offset, i) + r = Repr(index_stride_size, stride_size, axis_size, + arr_storage, storage, index_state.offset, arr_state.offset) + ArgSort(r).sort() + arr_state = arr_iter.next(arr_state) + index_state = index_iter.next(index_state) + return index_arr return argsort @@ -282,25 +282,25 @@ axis = -1 else: axis = space.int_w(w_axis) - # create array of indexes - if len(arr.get_shape()) == 1: - r = Repr(itemsize, arr.get_size(), arr.get_storage(), - arr.start) - ArgSort(r).sort() - else: - shape = arr.get_shape() - if axis < 0: - axis = len(shape) + axis - if axis < 0 or axis >= len(shape): - raise oefmt(space.w_IndexError, "Wrong axis %d", axis) - arr_iter = AllButAxisIter(arr, axis) - arr_state = arr_iter.reset() - stride_size = arr.strides[axis] - axis_size = arr.shape[axis] - while not arr_iter.done(arr_state): - r = Repr(stride_size, axis_size, arr.get_storage(), arr_state.offset) + with arr as storage: + if len(arr.get_shape()) == 1: + r = Repr(itemsize, arr.get_size(), storage, + arr.start) ArgSort(r).sort() - arr_state = arr_iter.next(arr_state) + else: + shape = arr.get_shape() + if axis < 0: + axis = len(shape) + axis + if axis < 0 or axis >= len(shape): + raise oefmt(space.w_IndexError, "Wrong axis %d", axis) + arr_iter = AllButAxisIter(arr, axis) + arr_state = arr_iter.reset() + stride_size = arr.strides[axis] + axis_size = arr.shape[axis] + while not arr_iter.done(arr_state): + r = Repr(stride_size, axis_size, storage, arr_state.offset) + ArgSort(r).sort() + arr_state = arr_iter.next(arr_state) return sort diff --git a/pypy/module/micronumpy/support.py b/pypy/module/micronumpy/support.py --- a/pypy/module/micronumpy/support.py +++ b/pypy/module/micronumpy/support.py @@ -1,6 +1,7 @@ from pypy.interpreter.error import OperationError, oefmt from rpython.rlib import jit from rpython.rlib.rarithmetic import ovfcheck +from rpython.rtyper.lltypesystem import rffi, lltype def issequence_w(space, w_obj): @@ -147,3 +148,7 @@ if cur_core_dim == 0: ufunc.core_enabled = False return 0 # for historical reasons, any failures will raise + +def get_storage_as_int(storage, start=0): + return rffi.cast(lltype.Signed, storage) + start + diff --git a/pypy/module/micronumpy/test/test_subtype.py b/pypy/module/micronumpy/test/test_subtype.py --- a/pypy/module/micronumpy/test/test_subtype.py +++ b/pypy/module/micronumpy/test/test_subtype.py @@ -2,7 +2,7 @@ class AppTestSupport(BaseNumpyAppTest): - spaceconfig = dict(usemodules=["micronumpy", "struct", "binascii"]) + spaceconfig = dict(usemodules=["micronumpy", "struct", "binascii", "mmap"]) def setup_class(cls): BaseNumpyAppTest.setup_class.im_func(cls) @@ -476,3 +476,120 @@ a = self.SubType(array([[1, 2], [3, 4]])) b = array(a, subok=False) assert type(b) is ndarray + + def test_numpypy_mmap(self): + # issue #21 on pypy/numpy + from numpy import array, ndarray, arange, dtype as dtypedescr + import mmap + import os.path + from tempfile import mkdtemp + import os.path as path + valid_filemodes = ["r", "c", "r+", "w+"] + writeable_filemodes = ["r+", "w+"] + mode_equivalents = { + "readonly":"r", + "copyonwrite":"c", + "readwrite":"r+", + "write":"w+" + } + + class memmap(ndarray): + def __new__(subtype, filename, dtype='uint8', mode='r+', offset=0, shape=None, order='C'): + # Import here to minimize 'import numpy' overhead + try: + mode = mode_equivalents[mode] + except KeyError: + if mode not in valid_filemodes: + raise ValueError("mode must be one of %s" % + (valid_filemodes + list(mode_equivalents.keys()))) + + if hasattr(filename, 'read'): + fid = filename + own_file = False + else: + fid = open(filename, (mode == 'c' and 'r' or mode)+'b') + own_file = True + + if (mode == 'w+') and shape is None: + raise ValueError("shape must be given") + + fid.seek(0, 2) + flen = fid.tell() + descr = dtypedescr(dtype) + _dbytes = descr.itemsize + + if shape is None: + bytes = flen - offset + if (bytes % _dbytes): + fid.close() + raise ValueError("Size of available data is not a " + "multiple of the data-type size.") + size = bytes // _dbytes + shape = (size,) + else: + if not isinstance(shape, tuple): + shape = (shape,) + size = 1 + for k in shape: + size *= k + + bytes = long(offset + size*_dbytes) + + if mode == 'w+' or (mode == 'r+' and flen < bytes): + fid.seek(bytes - 1, 0) + fid.write('\0') + fid.flush() + + if mode == 'c': + acc = mmap.ACCESS_COPY + elif mode == 'r': + acc = mmap.ACCESS_READ + else: + acc = mmap.ACCESS_WRITE + + start = offset - offset % mmap.ALLOCATIONGRANULARITY + bytes -= start + offset -= start + mm = mmap.mmap(fid.fileno(), bytes, access=acc, offset=start) + + self = ndarray.__new__(subtype, shape, dtype=descr, buffer=mm, + offset=offset, order=order) + self._mmap = mm + self.offset = offset + self.mode = mode + + if isinstance(filename, basestring): + self.filename = os.path.abspath(filename) + # py3 returns int for TemporaryFile().name + elif (hasattr(filename, "name") and + isinstance(filename.name, basestring)): + self.filename = os.path.abspath(filename.name) + # same as memmap copies (e.g. memmap + 1) + else: + self.filename = None + + if own_file: + fid.close() + + return self + + def flush(self): + if self.base is not None and hasattr(self.base, 'flush'): + self.base.flush() + + def asarray(obj, itemsize=None, order=None): + return array(obj, itemsize, copy=False, order=order) + + filename = path.join(mkdtemp(), 'newfile.dat') + data = arange(10*10*36).reshape(10, 10, 36) + fp = memmap(filename, dtype='float32', mode='w+', shape=data.shape) + vals = [ 242, 507, 255, 505, 315, 316, 308, 506, + 309, 255, 211, 505, 315, 316, 308, 506, + 309, 255, 255, 711, 194, 232, 711, 711, + 709, 710, 709, 710, 882, 897, 711, 245, + 711, 711, 168, 245] + fp[:] = data + fp[5:6][:,4] = vals + a = asarray(fp[5:6][:,4]) + assert (a == vals).all() + diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py --- a/pypy/module/micronumpy/test/test_zjit.py +++ b/pypy/module/micronumpy/test/test_zjit.py @@ -238,15 +238,15 @@ 'getfield_gc': 5, 'getfield_gc_pure': 51, 'guard_class': 3, - 'guard_false': 13, + 'guard_false': 12, 'guard_nonnull': 11, 'guard_nonnull_class': 3, 'guard_not_invalidated': 2, 'guard_true': 10, - 'guard_value': 5, + 'guard_value': 6, 'int_add': 13, 'int_ge': 4, - 'int_is_true': 4, + 'int_is_true': 3, 'int_is_zero': 4, 'int_le': 2, 'int_lt': 3, @@ -616,15 +616,15 @@ 'getfield_gc': 6, 'getfield_gc_pure': 63, 'guard_class': 5, - 'guard_false': 20, + 'guard_false': 19, 'guard_nonnull': 6, 'guard_nonnull_class': 1, 'guard_not_invalidated': 3, 'guard_true': 16, - 'guard_value': 2, + 'guard_value': 3, 'int_add': 24, 'int_ge': 4, - 'int_is_true': 6, + 'int_is_true': 5, 'int_is_zero': 4, 'int_le': 5, 'int_lt': 7, diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -180,13 +180,16 @@ raw_storage_setitem_unaligned(storage, i + offset, value) def read(self, arr, i, offset, dtype=None): - return self.box(self._read(arr.storage, i, offset)) + with arr as storage: + return self.box(self._read(storage, i, offset)) def read_bool(self, arr, i, offset): - return bool(self.for_computation(self._read(arr.storage, i, offset))) + with arr as storage: + return bool(self.for_computation(self._read(storage, i, offset))) def store(self, arr, i, offset, box): - self._write(arr.storage, i, offset, self.unbox(box)) + with arr as storage: + self._write(storage, i, offset, self.unbox(box)) def fill(self, storage, width, box, start, stop, offset): value = self.unbox(box) @@ -1080,8 +1083,9 @@ return bool(real) or bool(imag) def read_bool(self, arr, i, offset): - v = self.for_computation(self._read(arr.storage, i, offset)) - return bool(v[0]) or bool(v[1]) + with arr as storage: + v = self.for_computation(self._read(storage, i, offset)) + return bool(v[0]) or bool(v[1]) def get_element_size(self): return 2 * rffi.sizeof(self.T) @@ -1132,8 +1136,9 @@ return real, imag def read(self, arr, i, offset, dtype=None): - real, imag = self._read(arr.storage, i, offset) - return self.box_complex(real, imag) + with arr as storage: + real, imag = self._read(storage, i, offset) + return self.box_complex(real, imag) def _write(self, storage, i, offset, value): real, imag = value @@ -1144,7 +1149,8 @@ raw_storage_setitem_unaligned(storage, i + offset + rffi.sizeof(self.T), imag) def store(self, arr, i, offset, box): - self._write(arr.storage, i, offset, self.unbox(box)) + with arr as storage: + self._write(storage, i, offset, self.unbox(box)) def fill(self, storage, width, box, start, stop, offset): value = self.unbox(box) @@ -1633,13 +1639,14 @@ assert isinstance(item, boxes.W_FlexibleBox) i = item.ofs end = i + item.dtype.elsize - while i < end: - assert isinstance(item.arr.storage[i], str) - if item.arr.storage[i] == '\x00': - break - builder.append(item.arr.storage[i]) - i += 1 - return builder.build() + with item.arr as storage: + while i < end: + assert isinstance(storage[i], str) + if storage[i] == '\x00': + break + builder.append(storage[i]) + i += 1 + return builder.build() def str_unary_op(func): specialize.argtype(1)(func) @@ -1669,23 +1676,26 @@ w_item = space.wrap('') arg = space.str_w(space.str(w_item)) arr = VoidBoxStorage(dtype.elsize, dtype) - j = min(len(arg), dtype.elsize) - for i in range(j): - arr.storage[i] = arg[i] - for j in range(j, dtype.elsize): - arr.storage[j] = '\x00' - return boxes.W_StringBox(arr, 0, arr.dtype) + with arr as storage: + j = min(len(arg), dtype.elsize) + for i in range(j): + storage[i] = arg[i] + for j in range(j, dtype.elsize): + storage[j] = '\x00' + return boxes.W_StringBox(arr, 0, arr.dtype) def store(self, arr, i, offset, box): assert isinstance(box, boxes.W_StringBox) size = min(arr.dtype.elsize - offset, box.arr.size - box.ofs) - return self._store(arr.storage, i, offset, box, size) + with arr as storage: + return self._store(storage, i, offset, box, size) @jit.unroll_safe def _store(self, storage, i, offset, box, size): assert isinstance(box, boxes.W_StringBox) - for k in range(size): - storage[k + offset + i] = box.arr.storage[k + box.ofs] + with box.arr as box_storage: + for k in range(size): + storage[k + offset + i] = box_storage[k + box.ofs] def read(self, arr, i, offset, dtype=None): if dtype is None: @@ -1802,8 +1812,9 @@ assert i == 0 assert isinstance(box, boxes.W_VoidBox) assert box.dtype is box.arr.dtype - for k in range(box.arr.dtype.elsize): - arr.storage[k + ofs] = box.arr.storage[k + box.ofs] + with arr as arr_storage, box.arr as box_storage: + for k in range(box.arr.dtype.elsize): + arr_storage[k + ofs] = box_storage[k + box.ofs] def readarray(self, arr, i, offset, dtype=None): from pypy.module.micronumpy.base import W_NDimArray @@ -1893,12 +1904,14 @@ def store(self, arr, i, ofs, box): assert isinstance(box, boxes.W_VoidBox) - self._store(arr.storage, i, ofs, box, box.dtype.elsize) + with arr as storage: + self._store(storage, i, ofs, box, box.dtype.elsize) @jit.unroll_safe def _store(self, storage, i, ofs, box, size): - for k in range(size): - storage[k + i + ofs] = box.arr.storage[k + box.ofs] + with box.arr as box_storage: + for k in range(size): + storage[k + i + ofs] = box_storage[k + box.ofs] def fill(self, storage, width, box, start, stop, offset): assert isinstance(box, boxes.W_VoidBox) @@ -1944,9 +1957,10 @@ s1 = v1.dtype.elsize s2 = v2.dtype.elsize assert s1 == s2 - for i in range(s1): - if v1.arr.storage[v1.ofs + i] != v2.arr.storage[v2.ofs + i]: - return False + with v1.arr as v1_storage, v2.arr as v2_storage: + for i in range(s1): + if v1_storage[v1.ofs + i] != v2_storage[v2.ofs + i]: + return False return True def ne(self, v1, v2): diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py --- a/pypy/module/micronumpy/ufuncs.py +++ b/pypy/module/micronumpy/ufuncs.py @@ -13,11 +13,12 @@ from pypy.module.micronumpy.ctors import numpify from pypy.module.micronumpy.nditer import W_NDIter, coalesce_iter from pypy.module.micronumpy.strides import shape_agreement -from pypy.module.micronumpy.support import _parse_signature, product +from pypy.module.micronumpy.support import _parse_signature, product, get_storage_as_int from rpython.rlib.rawstorage import (raw_storage_setitem, free_raw_storage, alloc_raw_storage) from rpython.rtyper.lltypesystem import rffi, lltype from rpython.rlib.rarithmetic import LONG_BIT, _get_bitsize +from rpython.rlib.objectmodel import keepalive_until_here def done_if_true(dtype, val): @@ -98,7 +99,9 @@ if out is not None and not isinstance(out, W_NDimArray): raise OperationError(space.w_TypeError, space.wrap( 'output must be an array')) - return self.call(space, args_w, sig, casting, extobj) + retval = self.call(space, args_w, sig, casting, extobj) + keepalive_until_here(args_w) + return retval def descr_accumulate(self, space, w_obj, w_axis=None, w_dtype=None, w_out=None): if space.is_none(w_axis): @@ -804,11 +807,12 @@ assert isinstance(curarg, W_NDimArray) if len(arg_shapes[i]) != curarg.ndims(): # reshape + sz = product(curarg.get_shape()) * curarg.get_dtype().elsize - inargs[i] = W_NDimArray.from_shape_and_storage( - space, arg_shapes[i], curarg.implementation.storage, - curarg.get_dtype(), storage_bytes=sz, w_base=curarg) - pass + with curarg.implementation as storage: + inargs[i] = W_NDimArray.from_shape_and_storage( + space, arg_shapes[i], storage, + curarg.get_dtype(), storage_bytes=sz, w_base=curarg) need_to_cast.append(curarg.get_dtype() != dtypes[i]) for i in range(len(outargs)): j = self.nin + i @@ -819,9 +823,10 @@ elif len(arg_shapes[i]) != curarg.ndims(): # reshape sz = product(curarg.get_shape()) * curarg.get_dtype().elsize - outargs[i] = W_NDimArray.from_shape_and_storage( - space, arg_shapes[i], curarg.implementation.storage, - curarg.get_dtype(), storage_bytes=sz, w_base=curarg) + with curarg.implementation as storage: + outargs[i] = W_NDimArray.from_shape_and_storage( + space, arg_shapes[i], storage, + curarg.get_dtype(), storage_bytes=sz, w_base=curarg) curarg = outargs[i] assert isinstance(curarg, W_NDimArray) need_to_cast.append(curarg.get_dtype() != dtypes[j]) @@ -1406,8 +1411,9 @@ raise OperationError(space.w_NotImplementedError, space.wrap("cannot mix ndarray and %r (arg %d) in call to ufunc" % ( arg_i, i))) - raw_storage_setitem(dataps, CCHARP_SIZE * i, - rffi.cast(rffi.CCHARP, arg_i.implementation.get_storage_as_int(space))) + with arg_i.implementation as storage: + addr = get_storage_as_int(storage, arg_i.get_start()) + raw_storage_setitem(dataps, CCHARP_SIZE * i, rffi.cast(rffi.CCHARP, addr)) #This assumes we iterate over the whole array (it should be a view...) raw_storage_setitem(self.dims, LONG_SIZE * i, rffi.cast(rffi.LONG, arg_i.get_size())) raw_storage_setitem(self.steps, LONG_SIZE * i, rffi.cast(rffi.LONG, arg_i.get_dtype().elsize)) @@ -1415,8 +1421,9 @@ for i in range(len(args_w)): arg_i = args_w[i] assert isinstance(arg_i, W_NDimArray) - raw_storage_setitem(dataps, CCHARP_SIZE * i, - rffi.cast(rffi.CCHARP, arg_i.implementation.get_storage_as_int(space))) + with arg_i.implementation as storage: + addr = get_storage_as_int(storage, arg_i.get_start()) + raw_storage_setitem(dataps, CCHARP_SIZE * i, rffi.cast(rffi.CCHARP, addr)) try: arg1 = rffi.cast(rffi.CArrayPtr(rffi.CCHARP), dataps) arg2 = rffi.cast(npy_intpp, self.dims) @@ -1424,6 +1431,7 @@ self.func(arg1, arg2, arg3, self.data) finally: free_raw_storage(dataps, track_allocation=False) + keepalive_until_here(args_w) def set_dims_and_steps(self, space, dims, steps): if not isinstance(dims, list) or not isinstance(steps, list): diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py --- a/pypy/module/mmap/interp_mmap.py +++ b/pypy/module/mmap/interp_mmap.py @@ -22,6 +22,10 @@ self.check_valid() return MMapBuffer(self.space, self.mmap, True) + def writebuf_w(self, space): + self.check_writeable() + return MMapBuffer(self.space, self.mmap, False) + def close(self): self.mmap.close() diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py --- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py +++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py @@ -341,7 +341,7 @@ raw_store(i119, 0, i160, descr=) raw_store(i119, 2, i160, descr=) raw_store(i119, 4, i160, descr=) - setfield_gc(p167, i119, descr=) + setfield_gc(p167, i119, descr=) i123 = arraylen_gc(p67, descr=) jump(..., descr=...) """) diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py --- a/pypy/module/sys/test/test_sysmodule.py +++ b/pypy/module/sys/test/test_sysmodule.py @@ -521,9 +521,11 @@ def test_reload_doesnt_override_sys_executable(self): import sys - sys.executable = 'from_test_sysmodule' + if not hasattr(sys, 'executable'): # if not translated + sys.executable = 'from_test_sysmodule' + previous = sys.executable reload(sys) - assert sys.executable == 'from_test_sysmodule' + assert sys.executable == previous def test_settrace(self): import sys diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py --- a/rpython/annotator/annrpython.py +++ b/rpython/annotator/annrpython.py @@ -6,8 +6,8 @@ from rpython.tool.pairtype import pair from rpython.tool.error import (format_blocked_annotation_error, gather_error, source_lines) -from rpython.flowspace.model import (Variable, Constant, FunctionGraph, - c_last_exception, checkgraph) +from rpython.flowspace.model import ( + Variable, Constant, FunctionGraph, checkgraph) from rpython.translator import simplify, transform from rpython.annotator import model as annmodel, signature from rpython.annotator.argument import simple_args @@ -399,16 +399,25 @@ def flowin(self, graph, block): try: - for i, op in enumerate(block.operations): + i = 0 + while i < len(block.operations): + op = block.operations[i] self.bookkeeper.enter((graph, block, i)) try: + new_ops = op.transform(self) + if new_ops is not None: + block.operations[i:i+1] = new_ops + if not new_ops: + continue + new_ops[-1].result = op.result + op = new_ops[0] self.consider_op(op) finally: self.bookkeeper.leave() + i += 1 except BlockedInference as e: - if (e.op is block.operations[-1] and - block.exitswitch == c_last_exception): + if e.op is block.raising_op: # this is the case where the last operation of the block will # always raise an exception which is immediately caught by # an exception handler. We then only follow the exceptional @@ -450,8 +459,8 @@ # filter out those exceptions which cannot # occour for this specific, typed operation. - if block.exitswitch == c_last_exception: - op = block.operations[-1] + if block.canraise: + op = block.raising_op can_only_throw = op.get_can_only_throw(self) if can_only_throw is not None: candidates = can_only_throw diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py --- a/rpython/annotator/binaryop.py +++ b/rpython/annotator/binaryop.py @@ -2,7 +2,6 @@ Binary operations between SomeValues. """ -import operator from rpython.tool.pairtype import pair, pairtype from rpython.annotator.model import ( SomeObject, SomeInteger, SomeBool, s_Bool, SomeString, SomeChar, SomeList, @@ -14,7 +13,7 @@ read_can_only_throw, add_knowntypedata, merge_knowntypedata,) from rpython.annotator.bookkeeper import immutablevalue -from rpython.flowspace.model import Variable, Constant +from rpython.flowspace.model import Variable, Constant, const from rpython.flowspace.operation import op from rpython.rlib import rarithmetic from rpython.annotator.model import AnnotatorError @@ -689,12 +688,16 @@ return super(thistype, pair(ins1, ins2)).improve() -class __extend__(pairtype(SomeInstance, SomeObject)): - def getitem((s_ins, s_idx)): - return s_ins._emulate_call("__getitem__", s_idx) + at op.getitem.register_transform(SomeInstance, SomeObject) +def getitem_SomeInstance(annotator, v_ins, v_idx): + get_getitem = op.getattr(v_ins, const('__getitem__')) + return [get_getitem, op.simple_call(get_getitem.result, v_idx)] - def setitem((s_ins, s_idx), s_value): - return s_ins._emulate_call("__setitem__", s_idx, s_value) + at op.setitem.register_transform(SomeInstance, SomeObject) +def setitem_SomeInstance(annotator, v_ins, v_idx, v_value): + get_setitem = op.getattr(v_ins, const('__setitem__')) + return [get_setitem, + op.simple_call(get_setitem.result, v_idx, v_value)] class __extend__(pairtype(SomeIterator, SomeIterator)): diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py --- a/rpython/annotator/unaryop.py +++ b/rpython/annotator/unaryop.py @@ -5,6 +5,7 @@ from __future__ import absolute_import from rpython.flowspace.operation import op +from rpython.flowspace.model import const from rpython.annotator.model import (SomeObject, SomeInteger, SomeBool, SomeString, SomeChar, SomeList, SomeDict, SomeTuple, SomeImpossibleValue, SomeUnicodeCodePoint, SomeInstance, SomeBuiltin, SomeBuiltinMethod, @@ -686,27 +687,33 @@ if not self.can_be_None: s.const = True - def _emulate_call(self, meth_name, *args_s): - bk = getbookkeeper() - s_attr = self._true_getattr(meth_name) - # record for calltables - bk.emulate_pbc_call(bk.position_key, s_attr, args_s) - return s_attr.call(simple_args(args_s)) + at op.len.register_transform(SomeInstance) +def len_SomeInstance(annotator, v_arg): + get_len = op.getattr(v_arg, const('__len__')) + return [get_len, op.simple_call(get_len.result)] - def iter(self): - return self._emulate_call('__iter__') + at op.iter.register_transform(SomeInstance) +def iter_SomeInstance(annotator, v_arg): + get_iter = op.getattr(v_arg, const('__iter__')) + return [get_iter, op.simple_call(get_iter.result)] - def next(self): - return self._emulate_call('next') + at op.next.register_transform(SomeInstance) +def next_SomeInstance(annotator, v_arg): + get_next = op.getattr(v_arg, const('next')) + return [get_next, op.simple_call(get_next.result)] - def len(self): - return self._emulate_call('__len__') + at op.getslice.register_transform(SomeInstance) +def getslice_SomeInstance(annotator, v_obj, v_start, v_stop): + get_getslice = op.getattr(v_obj, const('__getslice__')) + return [get_getslice, op.simple_call(get_getslice.result, v_start, v_stop)] - def getslice(self, s_start, s_stop): - return self._emulate_call('__getslice__', s_start, s_stop) - def setslice(self, s_start, s_stop, s_iterable): - return self._emulate_call('__setslice__', s_start, s_stop, s_iterable) + at op.setslice.register_transform(SomeInstance) +def setslice_SomeInstance(annotator, v_obj, v_start, v_stop, v_iterable): + get_setslice = op.getattr(v_obj, const('__setslice__')) + return [get_setslice, + op.simple_call(get_setslice.result, v_start, v_stop, v_iterable)] + class __extend__(SomeBuiltin): def call(self, args, implicit_init=False): diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py --- a/rpython/flowspace/flowcontext.py +++ b/rpython/flowspace/flowcontext.py @@ -7,7 +7,6 @@ import __builtin__ from rpython.tool.error import source_lines -from rpython.translator.simplify import eliminate_empty_blocks from rpython.rlib import rstackovf from rpython.flowspace.argument import CallSpec from rpython.flowspace.model import (Constant, Variable, Block, Link, @@ -124,7 +123,6 @@ def guessexception(self, ctx, *cases): block = self.crnt_block - bvars = vars = vars2 = block.getvariables() links = [] for case in [None] + list(cases): if case is not None: diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py --- a/rpython/flowspace/model.py +++ b/rpython/flowspace/model.py @@ -190,6 +190,15 @@ txt = "%s(%s)" % (txt, self.exitswitch) return txt + @property + def canraise(self): + return self.exitswitch is c_last_exception + + @property + def raising_op(self): + if self.canraise: + return self.operations[-1] + def getvariables(self): "Return all variables mentioned in this Block." result = self.inputargs[:] @@ -591,11 +600,11 @@ assert len(block.exits) <= 1 if block.exits: assert block.exits[0].exitcase is None - elif block.exitswitch == Constant(last_exception): + elif block.canraise: assert len(block.operations) >= 1 # check if an exception catch is done on a reasonable # operation - assert block.operations[-1].opname not in ("keepalive", + assert block.raising_op.opname not in ("keepalive", "cast_pointer", "same_as") assert len(block.exits) >= 2 diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py --- a/rpython/flowspace/operation.py +++ b/rpython/flowspace/operation.py @@ -57,8 +57,10 @@ setattr(op, cls.opname, cls) if cls.dispatch == 1: cls._registry = {} + cls._transform = {} elif cls.dispatch == 2: cls._registry = DoubleDispatchRegistry() + cls._transform = DoubleDispatchRegistry() class HLOperation(SpaceOperation): @@ -104,6 +106,14 @@ def get_can_only_throw(self, annotator): return None + def get_transformer(self, *args_s): + return lambda *args: None + + def transform(self, annotator): + args_s = [annotator.annotation(arg) for arg in self.args] + transformer = self.get_transformer(*args_s) + return transformer(annotator, *self.args) + class PureOperation(HLOperation): pure = True @@ -185,6 +195,37 @@ except AttributeError: return cls._dispatch(type(s_arg)) + @classmethod + def get_specialization(cls, s_arg, *_ignored): + try: + impl = getattr(s_arg, cls.opname) + + def specialized(annotator, arg, *other_args): + return impl(*[annotator.annotation(x) for x in other_args]) + try: + specialized.can_only_throw = impl.can_only_throw + except AttributeError: + pass + return specialized + except AttributeError: + return cls._dispatch(type(s_arg)) + + @classmethod + def register_transform(cls, Some_cls): + def decorator(func): + cls._transform[Some_cls] = func + return func + return decorator + + @classmethod + def get_transformer(cls, s_arg, *_ignored): + for c in type(s_arg).__mro__: + try: + return cls._transform[c] + except KeyError: + pass + return lambda *args: None + class DoubleDispatchMixin(object): dispatch = 2 @@ -216,6 +257,20 @@ spec = type(self).get_specialization(*args_s) return read_can_only_throw(spec, args_s[0], args_s[1]) + @classmethod + def register_transform(cls, Some1, Some2): From noreply at buildbot.pypy.org Wed Mar 11 11:31:22 2015 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 11 Mar 2015 11:31:22 +0100 (CET) Subject: [pypy-commit] pypy default: merge Message-ID: <20150311103122.C3D591C0318@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r76323:48dadfa8f69e Date: 2015-03-11 12:31 +0200 http://bitbucket.org/pypy/pypy/changeset/48dadfa8f69e/ Log: merge diff --git a/lib_pypy/cffi.egg-info b/lib_pypy/cffi.egg-info --- a/lib_pypy/cffi.egg-info +++ b/lib_pypy/cffi.egg-info @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: cffi -Version: 0.9.0 +Version: 0.9.1 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "0.9.0" -__version_info__ = (0, 9, 0) +__version__ = "0.9.1" +__version_info__ = (0, 9, 1) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -2,13 +2,15 @@ from pypy.interpreter.mixedmodule import MixedModule from rpython.rlib import rdynload +VERSION = "0.9.1" + class Module(MixedModule): appleveldefs = { } interpleveldefs = { - '__version__': 'space.wrap("0.9.0")', + '__version__': 'space.wrap("%s")' % VERSION, 'load_library': 'libraryobj.load_library', diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -3249,4 +3249,4 @@ def test_version(): # this test is here mostly for PyPy - assert __version__ == "0.9.0" + assert __version__ == "0.9.1" diff --git a/pypy/module/_cffi_backend/test/test_file.py b/pypy/module/_cffi_backend/test/test_file.py --- a/pypy/module/_cffi_backend/test/test_file.py +++ b/pypy/module/_cffi_backend/test/test_file.py @@ -15,3 +15,15 @@ "Update test/_backend_test_c.py by copying it from " "https://bitbucket.org/cffi/cffi/raw/default/c/test_c.py " "and killing the import lines at the start") + +def test_egginfo_version(): + from pypy.module._cffi_backend import VERSION + line = "Version: %s\n" % VERSION + eggfile = py.path.local(__file__).join('..', '..', '..', '..', '..', + 'lib_pypy', 'cffi.egg-info') + assert line in eggfile.readlines() + +def test_app_version(): + from pypy.module import _cffi_backend + from lib_pypy import cffi + assert _cffi_backend.VERSION == cffi.__version__ diff --git a/pypy/module/_random/interp_random.py b/pypy/module/_random/interp_random.py --- a/pypy/module/_random/interp_random.py +++ b/pypy/module/_random/interp_random.py @@ -4,7 +4,7 @@ from pypy.interpreter.typedef import TypeDef from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.baseobjspace import W_Root -from rpython.rlib.rarithmetic import r_uint, intmask +from rpython.rlib.rarithmetic import r_uint, intmask, widen from rpython.rlib import rbigint, rrandom, rstring @@ -54,8 +54,8 @@ def getstate(self, space): state = [None] * (rrandom.N + 1) for i in range(rrandom.N): - state[i] = space.newint(intmask(self._rnd.state[i])) - state[rrandom.N] = space.newint(self._rnd.index) + state[i] = space.wrap(widen(self._rnd.state[i])) + state[rrandom.N] = space.newlong(self._rnd.index) return space.newtuple(state) def setstate(self, space, w_state): diff --git a/pypy/module/_random/test/test_random.py b/pypy/module/_random/test/test_random.py --- a/pypy/module/_random/test/test_random.py +++ b/pypy/module/_random/test/test_random.py @@ -41,6 +41,17 @@ # does not crash rnd1.setstate((-1, ) * 624 + (0, )) + def test_state_repr(self): + # since app-level jumpahead salts with repr(state), + # it is important the repr is consistent with cpython + import _random + rnd = _random.Random() + rnd.seed(1234) + state = rnd.getstate() + s = repr(state) + assert len(s) == 7956 + assert s.count('L') == 625 + def test_seed(self): import _random, sys rnd = _random.Random() diff --git a/pypy/module/_ssl/__init__.py b/pypy/module/_ssl/__init__.py --- a/pypy/module/_ssl/__init__.py +++ b/pypy/module/_ssl/__init__.py @@ -51,6 +51,11 @@ super(Module, cls).buildloaders() + def setup_after_space_initialization(self): + """NOT_RPYTHON""" + from pypy.module._ssl.interp_ssl import PWINFO_STORAGE + PWINFO_STORAGE.clear() + def startup(self, space): from rpython.rlib.ropenssl import init_ssl init_ssl() diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -1462,6 +1462,7 @@ imag = GetSetProperty(W_NDimArray.descr_get_imag, W_NDimArray.descr_set_imag), conj = interp2app(W_NDimArray.descr_conj), + conjugate = interp2app(W_NDimArray.descr_conj), argsort = interp2app(W_NDimArray.descr_argsort), sort = interp2app(W_NDimArray.descr_sort), diff --git a/pypy/module/micronumpy/test/test_complex.py b/pypy/module/micronumpy/test/test_complex.py --- a/pypy/module/micronumpy/test/test_complex.py +++ b/pypy/module/micronumpy/test/test_complex.py @@ -382,6 +382,7 @@ assert np.conjugate(1+2j) == 1-2j eye2 = np.array([[1, 0], [0, 1]]) + assert (eye2.conjugate() == eye2).all() x = eye2 + 1j * eye2 for a, b in zip(np.conjugate(x), np.array([[ 1.-1.j, 0.-0.j], [ 0.-0.j, 1.-1.j]])): assert a[0] == b[0] diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py b/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py --- a/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py @@ -165,7 +165,8 @@ assert lib.sin(12.3) == math.sin(12.3) v = ffi.verifier ext = v.get_extension() - assert 'distutils.extension.Extension' in str(ext.__class__) + assert 'distutils.extension.Extension' in str(ext.__class__) or \ + 'setuptools.extension.Extension' in str(ext.__class__) assert ext.sources == [maybe_relative_path(v.sourcefilename)] assert ext.name == v.get_module_name() assert ext.define_macros == [('TEST_EXTENSION_OBJECT', '1')] @@ -194,7 +195,8 @@ assert lib.test1eoes(7.0) == 42.0 v = ffi.verifier ext = v.get_extension() - assert 'distutils.extension.Extension' in str(ext.__class__) + assert 'distutils.extension.Extension' in str(ext.__class__) or \ + 'setuptools.extension.Extension' in str(ext.__class__) assert ext.sources == [maybe_relative_path(v.sourcefilename), extra_source] assert ext.name == v.get_module_name() diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py b/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py --- a/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py @@ -4,6 +4,9 @@ import subprocess from pypy.module.test_lib_pypy.cffi_tests.udir import udir +if sys.platform == 'win32': + py.test.skip('snippets do not run on win32') + def create_venv(name): tmpdir = udir.join(name) try: @@ -13,6 +16,20 @@ except OSError as e: py.test.skip("Cannot execute virtualenv: %s" % (e,)) + try: + deepcopy = os.symlink + except: + import shutil, errno + def deepcopy(src, dst): + try: + shutil.copytree(src, dst) + except OSError as e: + if e.errno in (errno.ENOTDIR, errno.EINVAL): + shutil.copy(src, dst) + else: + print 'got errno',e.errno,'not',errno.ENOTDIR + raise + site_packages = None for dirpath, dirnames, filenames in os.walk(str(tmpdir)): if os.path.basename(dirpath) == 'site-packages': @@ -32,7 +49,7 @@ modules += ('ply',) # needed for older versions of pycparser for module in modules: target = imp.find_module(module)[1] - os.symlink(target, os.path.join(site_packages, + deepcopy(target, os.path.join(site_packages, os.path.basename(target))) return tmpdir @@ -51,7 +68,11 @@ python_f.write(py.code.Source(python_snippet)) try: os.chdir(str(SNIPPET_DIR.join(dirname))) - vp = str(venv_dir.join('bin/python')) + if os.name == 'nt': + bindir = 'Scripts' + else: + bindir = 'bin' + vp = str(venv_dir.join(bindir).join('python')) subprocess.check_call((vp, 'setup.py', 'clean')) subprocess.check_call((vp, 'setup.py', 'install')) subprocess.check_call((vp, str(python_f))) diff --git a/pypy/objspace/fake/checkmodule.py b/pypy/objspace/fake/checkmodule.py --- a/pypy/objspace/fake/checkmodule.py +++ b/pypy/objspace/fake/checkmodule.py @@ -10,6 +10,7 @@ mod = __import__('pypy.module.%s' % modname, None, None, ['__doc__']) # force computation and record what we wrap module = mod.Module(space, W_Root()) + module.setup_after_space_initialization() module.startup(space) for name in module.loaders: seeobj_w.append(module._load_lazily(space, name)) diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py --- a/rpython/annotator/test/test_annrpython.py +++ b/rpython/annotator/test/test_annrpython.py @@ -4326,13 +4326,6 @@ assert isinstance(s, annmodel.SomeString) assert not s.can_be_none() - def test_nonnulify(self): - s = annmodel.SomeString(can_be_None=True).nonnulify() - assert s.can_be_None is True - assert s.no_nul is True - s = annmodel.SomeChar().nonnulify() - assert s.no_nul is True - def g(n): return [0, 1, 2, n] diff --git a/rpython/annotator/test/test_model.py b/rpython/annotator/test/test_model.py --- a/rpython/annotator/test/test_model.py +++ b/rpython/annotator/test/test_model.py @@ -117,3 +117,11 @@ assert s_int != SomeInteger() assert not_const(s_int) == SomeInteger() assert not_const(s_None) == s_None + + +def test_nonnulify(): + s = SomeString(can_be_None=True).nonnulify() + assert s.can_be_None is True + assert s.no_nul is True + s = SomeChar().nonnulify() + assert s.no_nul is True diff --git a/rpython/doc/jit/virtualizable.rst b/rpython/doc/jit/virtualizable.rst --- a/rpython/doc/jit/virtualizable.rst +++ b/rpython/doc/jit/virtualizable.rst @@ -44,8 +44,14 @@ virtualizable arrays that can be very confusing. Those will usually end up with a compile-time error (as opposed to strange behavior). The rules are: +* A virtualizable array must be a fixed-size list. After it is + initialized (e.g. in ``Frame.__init__``) you cannot resize it at all. + You cannot assign a different list to the field, or even pass around the + list. You can only access ``frame.array[index]`` directly. + * Each array access must be with a known positive index that cannot raise - an ``IndexError``. Using ``no = jit.hint(no, promote=True)`` might be useful + an ``IndexError``. + Using ``index = jit.hint(index, promote=True)`` might be useful to get a constant-number access. This is only safe if the index is actually constant or changing rarely within the context of the user's code. diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -143,13 +143,16 @@ return for v in list: if v in self.vable_array_vars: + vars = self.vable_array_vars[v] + (v_base, arrayfielddescr, arraydescr) = vars raise AssertionError( "A virtualizable array is passed around; it should\n" "only be used immediately after being read. Note\n" "that a possible cause is indexing with an index not\n" "known non-negative, or catching IndexError, or\n" "not inlining at all (for tests: use listops=True).\n" - "Occurred in: %r" % self.graph) + "This is about: %r\n" + "Occurred in: %r" % (arrayfielddescr, self.graph)) # extra explanation: with the way things are organized in # rpython/rlist.py, the ll_getitem becomes a function call # that is typically meant to be inlined by the JIT, but diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py --- a/rpython/jit/codewriter/test/test_flatten.py +++ b/rpython/jit/codewriter/test/test_flatten.py @@ -999,6 +999,7 @@ e = py.test.raises(AssertionError, self.encoding_test, f, [], "!", transform=True) assert str(e.value).startswith("A virtualizable array is passed aroun") + assert "" in str(e.value) def test_vable_attribute_list_copied_around(self): class F: @@ -1014,6 +1015,7 @@ e = py.test.raises(AssertionError, self.encoding_test, f, [], "!", transform=True) assert str(e.value).startswith("A virtualizable array is passed aroun") + assert "" in str(e.value) def check_force_cast(FROM, TO, operations, value): From noreply at buildbot.pypy.org Wed Mar 11 11:40:13 2015 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 11 Mar 2015 11:40:13 +0100 (CET) Subject: [pypy-commit] pypy default: (arigo, fijal) free(NULL) should work and not do anything Message-ID: <20150311104013.8E2A01C00F8@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r76324:96a3f1b070d8 Date: 2015-03-11 12:40 +0200 http://bitbucket.org/pypy/pypy/changeset/96a3f1b070d8/ Log: (arigo, fijal) free(NULL) should work and not do anything diff --git a/rpython/translator/c/src/mem.c b/rpython/translator/c/src/mem.c --- a/rpython/translator/c/src/mem.c +++ b/rpython/translator/c/src/mem.c @@ -29,6 +29,8 @@ RPY_EXTERN void pypy_debug_alloc_stop(void *addr) { + if (!addr) + return; struct pypy_debug_alloc_s **p; for (p = &pypy_debug_alloc_list; *p; p = &((*p)->next)) if ((*p)->addr == addr) From noreply at buildbot.pypy.org Wed Mar 11 12:23:26 2015 From: noreply at buildbot.pypy.org (aachurin) Date: Wed, 11 Mar 2015 12:23:26 +0100 (CET) Subject: [pypy-commit] pypy default: # python 'property' for rpython Message-ID: <20150311112326.0D76D1C0318@cobra.cs.uni-duesseldorf.de> Author: Andrey Churin Branch: Changeset: r76325:8c6cd439d1d1 Date: 2015-03-11 14:13 +0300 http://bitbucket.org/pypy/pypy/changeset/8c6cd439d1d1/ Log: # python 'property' for rpython diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py --- a/rpython/annotator/bookkeeper.py +++ b/rpython/annotator/bookkeeper.py @@ -12,7 +12,7 @@ SomeBuiltin, SomePBC, SomeInteger, TLS, SomeUnicodeCodePoint, s_None, s_ImpossibleValue, SomeBool, SomeTuple, SomeImpossibleValue, SomeUnicodeString, SomeList, HarmlesslyBlocked, - SomeWeakRef, SomeByteArray, SomeConstantType) + SomeWeakRef, SomeByteArray, SomeConstantType, SomeProperty) from rpython.annotator.classdef import InstanceSource, ClassDef from rpython.annotator.listdef import ListDef, ListItem from rpython.annotator.dictdef import DictDef @@ -301,6 +301,8 @@ s1 = self.immutablevalue(x1) assert isinstance(s1, SomeInstance) result = SomeWeakRef(s1.classdef) + elif tp is property: + return SomeProperty(x) elif ishashable(x) and x in BUILTIN_ANALYZERS: _module = getattr(x,"__module__","unknown") result = SomeBuiltin(BUILTIN_ANALYZERS[x], methodname="%s.%s" % (_module, x.__name__)) diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py --- a/rpython/annotator/description.py +++ b/rpython/annotator/description.py @@ -482,6 +482,19 @@ self.all_enforced_attrs = [] # no attribute allowed def add_source_attribute(self, name, value, mixin=False): + if isinstance(value, property): + # special case for property object + if value.fget is not None: + newname = name + '__getter__' + func = func_with_new_name(value.fget, newname) + self.add_source_attribute(newname, func, mixin) + if value.fset is not None: + newname = name + '__setter__' + func = func_with_new_name(value.fset, newname) + self.add_source_attribute(newname, func, mixin) + self.classdict[name] = Constant(value) + return + if isinstance(value, types.FunctionType): # for debugging if not hasattr(value, 'class_'): diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py --- a/rpython/annotator/model.py +++ b/rpython/annotator/model.py @@ -587,6 +587,19 @@ return False +class SomeProperty(SomeObject): + # used for union error only + immutable = True + knowntype = type(property) + + def __init__(self, prop): + self.fget = prop.fget + self.fset = prop.fset + + def can_be_none(self): + return False + + s_None = SomeNone() s_Bool = SomeBool() s_Int = SomeInteger() diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py --- a/rpython/annotator/test/test_annrpython.py +++ b/rpython/annotator/test/test_annrpython.py @@ -4326,6 +4326,82 @@ assert isinstance(s, annmodel.SomeString) assert not s.can_be_none() + def test_property_getter(self): + class O1(object): + def __init__(self, x): + self._x = x + @property + def x(self): + return self._x + def f(n): + o = O1(n) + return o.x + getattr(o, 'x') + a = self.RPythonAnnotator() + s = a.build_types(f, [int]) + assert isinstance(s, annmodel.SomeInteger) + op = list(graphof(a, f).iterblocks())[0].operations + i = 0 + c = 0 + while i < len(op): + if op[i].opname == 'getattr': + c += 1 + assert op[i].args[1].value == 'x__getter__' + i += 1 + assert i < len(op) and op[i].opname == 'simple_call' and \ + op[i].args[0] == op[i - 1].result + i += 1 + assert c == 2 + + def test_property_setter(self): + class O2(object): + def __init__(self): + self._x = 0 + def set_x(self, v): + self._x = v + x = property(fset=set_x) + def f(n): + o = O2() + o.x = n + setattr(o, 'x', n) + a = self.RPythonAnnotator() + s = a.build_types(f, [int]) + op = list(graphof(a, f).iterblocks())[0].operations + i = 0 + c = 0 + while i < len(op): + if op[i].opname == 'getattr': + c += 1 + assert op[i].args[1].value == 'x__setter__' + i += 1 + assert i < len(op) and op[i].opname == 'simple_call' and \ + op[i].args[0] == op[i - 1].result and len(op[i].args) == 2 + i += 1 + assert c == 2 + + def test_property_unionerr(self): + class O1(object): + def __init__(self, x): + self._x = x + @property + def x(self): + return self._x + class O2(O1): + def set_x(self, v): + self._x = v + x = property(fset=set_x) + def f1(n): + o = O2(n) + return o.x + def f2(n): + o = O2(n) + o.x = 20 + a = self.RPythonAnnotator() + with py.test.raises(annmodel.UnionError) as exc: + a.build_types(f1, [int]) + a = self.RPythonAnnotator() + with py.test.raises(annmodel.UnionError) as exc: + a.build_types(f2, [int]) + def g(n): return [0, 1, 2, n] diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py --- a/rpython/annotator/unaryop.py +++ b/rpython/annotator/unaryop.py @@ -5,7 +5,7 @@ from __future__ import absolute_import from rpython.flowspace.operation import op -from rpython.flowspace.model import const +from rpython.flowspace.model import const, Constant from rpython.annotator.model import (SomeObject, SomeInteger, SomeBool, SomeString, SomeChar, SomeList, SomeDict, SomeTuple, SomeImpossibleValue, SomeUnicodeCodePoint, SomeInstance, SomeBuiltin, SomeBuiltinMethod, @@ -715,6 +715,50 @@ op.simple_call(get_setslice.result, v_start, v_stop, v_iterable)] +def _find_property_meth(s_obj, attr, meth): + result = [] + for clsdef in s_obj.classdef.getmro(): + dct = clsdef.classdesc.classdict + if attr not in dct: + continue + obj = dct[attr] + if (not isinstance(obj, Constant) or + not isinstance(obj.value, property)): + return + result.append(getattr(obj.value, meth)) + return result + + + at op.getattr.register_transform(SomeInstance) +def getattr_SomeInstance(annotator, v_obj, v_attr): + s_attr = annotator.annotation(v_attr) + if not s_attr.is_constant() or not isinstance(s_attr.const, str): + return + attr = s_attr.const + getters = _find_property_meth(annotator.annotation(v_obj), attr, 'fget') + if getters: + if all(getters): + get_getter = op.getattr(v_obj, const(attr + '__getter__')) + return [get_getter, op.simple_call(get_getter.result)] + elif not any(getters): + raise AnnotatorError("Attribute %r is unreadable" % attr) + + + at op.setattr.register_transform(SomeInstance) +def setattr_SomeInstance(annotator, v_obj, v_attr, v_value): + s_attr = annotator.annotation(v_attr) + if not s_attr.is_constant() or not isinstance(s_attr.const, str): + return + attr = s_attr.const + setters = _find_property_meth(annotator.annotation(v_obj), attr, 'fset') + if setters: + if all(setters): + get_setter = op.getattr(v_obj, const(attr + '__setter__')) + return [get_setter, op.simple_call(get_setter.result, v_value)] + elif not any(setters): + raise AnnotatorError("Attribute %r is unwritable" % attr) + + class __extend__(SomeBuiltin): def call(self, args, implicit_init=False): args_s, kwds = args.unpack() From noreply at buildbot.pypy.org Wed Mar 11 12:23:27 2015 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 11 Mar 2015 12:23:27 +0100 (CET) Subject: [pypy-commit] pypy default: Merged in aachurin/pypy (pull request #309) Message-ID: <20150311112327.4CBCB1C0318@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r76326:6181ecf3eb79 Date: 2015-03-11 13:23 +0200 http://bitbucket.org/pypy/pypy/changeset/6181ecf3eb79/ Log: Merged in aachurin/pypy (pull request #309) 'property' for rpython diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py --- a/rpython/annotator/bookkeeper.py +++ b/rpython/annotator/bookkeeper.py @@ -12,7 +12,7 @@ SomeBuiltin, SomePBC, SomeInteger, TLS, SomeUnicodeCodePoint, s_None, s_ImpossibleValue, SomeBool, SomeTuple, SomeImpossibleValue, SomeUnicodeString, SomeList, HarmlesslyBlocked, - SomeWeakRef, SomeByteArray, SomeConstantType) + SomeWeakRef, SomeByteArray, SomeConstantType, SomeProperty) from rpython.annotator.classdef import InstanceSource, ClassDef from rpython.annotator.listdef import ListDef, ListItem from rpython.annotator.dictdef import DictDef @@ -301,6 +301,8 @@ s1 = self.immutablevalue(x1) assert isinstance(s1, SomeInstance) result = SomeWeakRef(s1.classdef) + elif tp is property: + return SomeProperty(x) elif ishashable(x) and x in BUILTIN_ANALYZERS: _module = getattr(x,"__module__","unknown") result = SomeBuiltin(BUILTIN_ANALYZERS[x], methodname="%s.%s" % (_module, x.__name__)) diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py --- a/rpython/annotator/description.py +++ b/rpython/annotator/description.py @@ -482,6 +482,19 @@ self.all_enforced_attrs = [] # no attribute allowed def add_source_attribute(self, name, value, mixin=False): + if isinstance(value, property): + # special case for property object + if value.fget is not None: + newname = name + '__getter__' + func = func_with_new_name(value.fget, newname) + self.add_source_attribute(newname, func, mixin) + if value.fset is not None: + newname = name + '__setter__' + func = func_with_new_name(value.fset, newname) + self.add_source_attribute(newname, func, mixin) + self.classdict[name] = Constant(value) + return + if isinstance(value, types.FunctionType): # for debugging if not hasattr(value, 'class_'): diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py --- a/rpython/annotator/model.py +++ b/rpython/annotator/model.py @@ -587,6 +587,19 @@ return False +class SomeProperty(SomeObject): + # used for union error only + immutable = True + knowntype = type(property) + + def __init__(self, prop): + self.fget = prop.fget + self.fset = prop.fset + + def can_be_none(self): + return False + + s_None = SomeNone() s_Bool = SomeBool() s_Int = SomeInteger() diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py --- a/rpython/annotator/test/test_annrpython.py +++ b/rpython/annotator/test/test_annrpython.py @@ -4326,6 +4326,82 @@ assert isinstance(s, annmodel.SomeString) assert not s.can_be_none() + def test_property_getter(self): + class O1(object): + def __init__(self, x): + self._x = x + @property + def x(self): + return self._x + def f(n): + o = O1(n) + return o.x + getattr(o, 'x') + a = self.RPythonAnnotator() + s = a.build_types(f, [int]) + assert isinstance(s, annmodel.SomeInteger) + op = list(graphof(a, f).iterblocks())[0].operations + i = 0 + c = 0 + while i < len(op): + if op[i].opname == 'getattr': + c += 1 + assert op[i].args[1].value == 'x__getter__' + i += 1 + assert i < len(op) and op[i].opname == 'simple_call' and \ + op[i].args[0] == op[i - 1].result + i += 1 + assert c == 2 + + def test_property_setter(self): + class O2(object): + def __init__(self): + self._x = 0 + def set_x(self, v): + self._x = v + x = property(fset=set_x) + def f(n): + o = O2() + o.x = n + setattr(o, 'x', n) + a = self.RPythonAnnotator() + s = a.build_types(f, [int]) + op = list(graphof(a, f).iterblocks())[0].operations + i = 0 + c = 0 + while i < len(op): + if op[i].opname == 'getattr': + c += 1 + assert op[i].args[1].value == 'x__setter__' + i += 1 + assert i < len(op) and op[i].opname == 'simple_call' and \ + op[i].args[0] == op[i - 1].result and len(op[i].args) == 2 + i += 1 + assert c == 2 + + def test_property_unionerr(self): + class O1(object): + def __init__(self, x): + self._x = x + @property + def x(self): + return self._x + class O2(O1): + def set_x(self, v): + self._x = v + x = property(fset=set_x) + def f1(n): + o = O2(n) + return o.x + def f2(n): + o = O2(n) + o.x = 20 + a = self.RPythonAnnotator() + with py.test.raises(annmodel.UnionError) as exc: + a.build_types(f1, [int]) + a = self.RPythonAnnotator() + with py.test.raises(annmodel.UnionError) as exc: + a.build_types(f2, [int]) + def g(n): return [0, 1, 2, n] diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py --- a/rpython/annotator/unaryop.py +++ b/rpython/annotator/unaryop.py @@ -5,7 +5,7 @@ from __future__ import absolute_import from rpython.flowspace.operation import op -from rpython.flowspace.model import const +from rpython.flowspace.model import const, Constant from rpython.annotator.model import (SomeObject, SomeInteger, SomeBool, SomeString, SomeChar, SomeList, SomeDict, SomeTuple, SomeImpossibleValue, SomeUnicodeCodePoint, SomeInstance, SomeBuiltin, SomeBuiltinMethod, @@ -715,6 +715,50 @@ op.simple_call(get_setslice.result, v_start, v_stop, v_iterable)] +def _find_property_meth(s_obj, attr, meth): + result = [] + for clsdef in s_obj.classdef.getmro(): + dct = clsdef.classdesc.classdict + if attr not in dct: + continue + obj = dct[attr] + if (not isinstance(obj, Constant) or + not isinstance(obj.value, property)): + return + result.append(getattr(obj.value, meth)) + return result + + + at op.getattr.register_transform(SomeInstance) +def getattr_SomeInstance(annotator, v_obj, v_attr): + s_attr = annotator.annotation(v_attr) + if not s_attr.is_constant() or not isinstance(s_attr.const, str): + return + attr = s_attr.const + getters = _find_property_meth(annotator.annotation(v_obj), attr, 'fget') + if getters: + if all(getters): + get_getter = op.getattr(v_obj, const(attr + '__getter__')) + return [get_getter, op.simple_call(get_getter.result)] + elif not any(getters): + raise AnnotatorError("Attribute %r is unreadable" % attr) + + + at op.setattr.register_transform(SomeInstance) +def setattr_SomeInstance(annotator, v_obj, v_attr, v_value): + s_attr = annotator.annotation(v_attr) + if not s_attr.is_constant() or not isinstance(s_attr.const, str): + return + attr = s_attr.const + setters = _find_property_meth(annotator.annotation(v_obj), attr, 'fset') + if setters: + if all(setters): + get_setter = op.getattr(v_obj, const(attr + '__setter__')) + return [get_setter, op.simple_call(get_setter.result, v_value)] + elif not any(setters): + raise AnnotatorError("Attribute %r is unwritable" % attr) + + class __extend__(SomeBuiltin): def call(self, args, implicit_init=False): args_s, kwds = args.unpack() From noreply at buildbot.pypy.org Wed Mar 11 15:01:25 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 11 Mar 2015 15:01:25 +0100 (CET) Subject: [pypy-commit] pypy vmprof: tweaks Message-ID: <20150311140125.8A28F1C015A@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof Changeset: r76327:847458219f4d Date: 2015-03-11 15:01 +0100 http://bitbucket.org/pypy/pypy/changeset/847458219f4d/ Log: tweaks diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py --- a/rpython/jit/backend/llsupport/codemap.py +++ b/rpython/jit/backend/llsupport/codemap.py @@ -72,7 +72,7 @@ static pypy_codemap_storage pypy_cs_g; -long bisect_right(long *a, long x, long hi) +static long bisect_right(long *a, long x, long hi) { long lo, mid; lo = 0; @@ -84,7 +84,7 @@ return lo; } -long bisect_right_addr(pypy_codemap_item *a, long x, long hi) +static long bisect_right_addr(pypy_codemap_item *a, long x, long hi) { long lo, mid; lo = 0; @@ -101,7 +101,7 @@ long pos; pos = bisect_right(pypy_cs_g.jit_addr_map, loc, pypy_cs_g.jit_addr_map_used); - if (pos == 0 || pos == pypy_cs_g.jit_addr_map_used) + if (pos == 0) return -1; return pypy_cs_g.jit_frame_depth_map[pos - 1]; } From noreply at buildbot.pypy.org Wed Mar 11 16:05:50 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 11 Mar 2015 16:05:50 +0100 (CET) Subject: [pypy-commit] pypy.org extradoc: update the values Message-ID: <20150311150550.64A7D1C015A@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r580:6e79f3d0571c Date: 2015-03-11 16:06 +0100 http://bitbucket.org/pypy/pypy.org/changeset/6e79f3d0571c/ Log: update the values diff --git a/don1.html b/don1.html --- a/don1.html +++ b/don1.html @@ -15,7 +15,7 @@ - $58892 of $105000 (56.1%) + $58921 of $105000 (56.1%)
From noreply at buildbot.pypy.org Wed Mar 11 16:23:29 2015 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 11 Mar 2015 16:23:29 +0100 (CET) Subject: [pypy-commit] pypy optresult: make the fisrt heapcache test pass Message-ID: <20150311152329.AA3B31C0EE0@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult Changeset: r76328:9b36048a749a Date: 2015-03-11 17:05 +0200 http://bitbucket.org/pypy/pypy/changeset/9b36048a749a/ Log: make the fisrt heapcache test pass diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py --- a/rpython/jit/backend/llgraph/runner.py +++ b/rpython/jit/backend/llgraph/runner.py @@ -89,11 +89,15 @@ return getkind(self.RESULT)[0] class SizeDescr(AbstractDescr): - def __init__(self, S, runner): + def __init__(self, S, is_object, runner): self.S = S + self._is_object = is_object self.all_fielddescrs = heaptracker.all_fielddescrs(runner, S, get_field_descr=LLGraphCPU.fielddescrof) + def is_object(self): + return self._is_object + def as_vtable_size_descr(self): return self @@ -374,12 +378,12 @@ self.descrs[key] = descr return descr - def sizeof(self, S): + def sizeof(self, S, is_object): key = ('size', S) try: return self.descrs[key] except KeyError: - descr = SizeDescr(S, self) + descr = SizeDescr(S, is_object, self) self.descrs[key] = descr return descr @@ -390,6 +394,8 @@ except KeyError: descr = FieldDescr(S, fieldname) self.descrs[key] = descr + is_obj = heaptracker.has_gcstruct_a_vtable(S) + descr.parent_descr = self.sizeof(S, is_obj) if self.vinfo_for_tests is not None: descr.vinfo = self.vinfo_for_tests return descr diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py --- a/rpython/jit/backend/llsupport/descr.py +++ b/rpython/jit/backend/llsupport/descr.py @@ -48,10 +48,16 @@ def repr_of_descr(self): return '' % self.size + def is_object(self): + return False + class SizeDescrWithVTable(SizeDescr): def as_vtable_size_descr(self): return self + def is_object(self): + return True + BaseSizeDescr = SizeDescr def get_size_descr(gccache, STRUCT): diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py --- a/rpython/jit/codewriter/heaptracker.py +++ b/rpython/jit/codewriter/heaptracker.py @@ -87,7 +87,7 @@ def register_known_gctype(cpu, vtable, STRUCT): # register the correspondance 'vtable' <-> 'STRUCT' in the cpu - sizedescr = cpu.sizeof(STRUCT) + sizedescr = cpu.sizeof(STRUCT, has_gcstruct_a_vtable(STRUCT)) assert sizedescr.as_vtable_size_descr() is sizedescr try: assert sizedescr._corresponding_vtable == vtable diff --git a/rpython/jit/metainterp/optimizeopt/earlyforce.py b/rpython/jit/metainterp/optimizeopt/earlyforce.py --- a/rpython/jit/metainterp/optimizeopt/earlyforce.py +++ b/rpython/jit/metainterp/optimizeopt/earlyforce.py @@ -1,3 +1,4 @@ + from rpython.jit.codewriter.effectinfo import EffectInfo from rpython.jit.metainterp.optimizeopt.optimizer import Optimization from rpython.jit.metainterp.resoperation import rop @@ -14,8 +15,7 @@ def propagate_forward(self, op): opnum = op.getopnum() - if 0: # XXX - if (opnum != rop.SETFIELD_GC and + if (opnum != rop.SETFIELD_GC and opnum != rop.SETARRAYITEM_GC and opnum != rop.SETARRAYITEM_RAW and opnum != rop.QUASIIMMUT_FIELD and @@ -26,9 +26,7 @@ not is_raw_free(op, opnum)): for arg in op.getarglist(): - if arg in self.optimizer.values: - value = self.getvalue(arg) - value.force_box(self) + self.optimizer.force_box(arg) self.emit_operation(op) def setup(self): diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -5,8 +5,7 @@ from rpython.jit.metainterp.history import Const, ConstInt from rpython.jit.metainterp.jitexc import JitException from rpython.jit.metainterp.optimizeopt.optimizer import Optimization, REMOVED -from rpython.jit.metainterp.optimizeopt.info import MODE_ARRAY,\ - LEVEL_KNOWNCLASS +from rpython.jit.metainterp.optimizeopt.info import MODE_ARRAY from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method from rpython.jit.metainterp.optimizeopt.intutils import IntBound from rpython.jit.metainterp.optimize import InvalidLoop @@ -30,6 +29,7 @@ # value pending in the ResOperation is *not* visible in # 'cached_fields'. # + Xxxx self._cached_fields = {} self._cached_fields_getfield_op = {} self._lazy_setfield = None @@ -186,8 +186,8 @@ """Cache repeated heap accesses""" def __init__(self): - # cached fields: {descr: CachedField} - self.cached_fields = {} + # mapping descr -> infos to invalidate + self.infos_to_invalidate = {} # cached array items: {array descr: {index: CachedField}} self.cached_arrayitems = {} # cached dict items: {dict descr: {(optval, index): box-or-const}} @@ -203,15 +203,6 @@ def setup(self): self.optimizer.optheap = self - def value_updated(self, oldvalue, newvalue): - # XXXX very unhappy about that - for cf in self.cached_fields.itervalues(): - cf.value_updated(oldvalue, newvalue, self.optimizer.exporting_state) - for submap in self.cached_arrayitems.itervalues(): - for cf in submap.itervalues(): - cf.value_updated(oldvalue, newvalue, - self.optimizer.exporting_state) - def force_at_end_of_preamble(self): self.cached_dict_reads.clear() self.corresponding_array_descrs.clear() @@ -247,9 +238,22 @@ for index, d in submap.items(): d.produce_potential_short_preamble_ops(self.optimizer, sb, descr) + def invalidate_descr(self, descr, lst=None): + if lst is not None: + lst = self.infos_to_invalidate.get(descr, None) + if lst is None: + return + for info in lst: + info.clear_cache() + del lst[:] + + def register_dirty_field(self, descr, info): + self.infos_to_invalidate.setdefault(descr, []).append(info) + def clean_caches(self): del self._lazy_setfields_and_arrayitems[:] - self.cached_fields.clear() + for descr, info in self.infos_to_invalidate.iteritems(): + self.invalidate_descr(descr, info) self.cached_arrayitems.clear() self.cached_dict_reads.clear() @@ -478,7 +482,15 @@ return pendingfields def optimize_GETFIELD_GC_I(self, op): + opinfo = self.ensure_ptr_info_arg0(op) + fld = opinfo.getfield(op.getdescr()) + if fld is not None: + self.make_equal_to(op, fld) + return self.emit_operation(op) + opinfo.setfield(op.getdescr(), op, self) + return + xxx return structvalue = self.getvalue(op.getarg(0)) cf = self.field_cache(op.getdescr()) @@ -522,6 +534,8 @@ cf.do_setfield(self, op) def optimize_GETARRAYITEM_GC_I(self, op): + self.emit_operation(op) + return # XXX arrayvalue = self.getvalue(op.getarg(0)) indexvalue = self.getvalue(op.getarg(1)) cf = None @@ -569,6 +583,8 @@ optimize_GETARRAYITEM_GC_PURE_F = optimize_GETARRAYITEM_GC_PURE_I def optimize_SETARRAYITEM_GC(self, op): + self.emit_operation(op) + return opnum = OpHelpers.getarrayitem_pure_for_descr(op.getdescr()) if self.has_pure_result(opnum, [op.getarg(0), op.getarg(1)], op.getdescr()): diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py --- a/rpython/jit/metainterp/optimizeopt/info.py +++ b/rpython/jit/metainterp/optimizeopt/info.py @@ -8,19 +8,20 @@ lower two bits are LEVEL """ -LEVEL_UNKNOWN = 0 -LEVEL_NONNULL = 1 -LEVEL_KNOWNCLASS = 2 # might also mean KNOWNARRAYDESCR, for arrays -LEVEL_CONSTANT = 3 MODE_ARRAY = '\x00' MODE_STR = '\x01' MODE_UNICODE = '\x02' +MODE_INSTANCE = '\x03' +MODE_STRUCT = '\x04' INFO_NULL = 0 INFO_NONNULL = 1 INFO_UNKNOWN = 2 +FLAG_VIRTUAL = 1 +FLAG_DIRTY = 2 + class AbstractInfo(AbstractValue): is_info_class = True @@ -55,39 +56,54 @@ return True class AbstractVirtualPtrInfo(NonNullPtrInfo): - _attrs_ = ('_is_virtual',) + _attrs_ = ('flags',) + + flags = 0 def force_box(self, op, optforce): - if self._is_virtual: + if self.is_virtual(): op.set_forwarded(None) optforce.emit_operation(op) newop = optforce.getlastop() op.set_forwarded(newop) newop.set_forwarded(self) - self._is_virtual = False - self._force_elements(newop, optforce) + self.flags &= ~FLAG_VIRTUAL # clean the virtual flag + if self._force_elements(newop, optforce): + self.flags |= FLAG_DIRTY return newop return op def is_virtual(self): - return self._is_virtual + return self.flags & FLAG_VIRTUAL class AbstractStructPtrInfo(AbstractVirtualPtrInfo): - _attrs_ = ('_is_virtual', '_fields') + _attrs_ = ('_fields',) def init_fields(self, descr): self._fields = [None] * len(descr.all_fielddescrs) - def setfield_virtual(self, descr, op): + def clear_cache(self): + assert self.flags & (FLAG_DIRTY | FLAG_VIRTUAL) == FLAG_DIRTY + self.flags = 0 + self._fields = [None] * len(self._fields) + + def setfield(self, descr, op, optheap=None): + if not self.is_virtual(): + if self.flags & FLAG_DIRTY == 0: + self.flags |= FLAG_DIRTY + assert optheap is not None + # we should only call it with virtuals without optheap + optheap.register_dirty_field(descr, self) self._fields[descr.index] = op - def getfield_virtual(self, descr): + def getfield(self, descr): return self._fields[descr.index] def _force_elements(self, op, optforce): if self._fields is None: - return + return 0 descr = op.getdescr() + count = 0 for i, flddescr in enumerate(descr.all_fielddescrs): fld = self._fields[i] if fld is not None: @@ -95,6 +111,10 @@ setfieldop = ResOperation(rop.SETFIELD_GC, [op, subbox], descr=flddescr) optforce.emit_operation(setfieldop) + if optforce.optheap: + optforce.optheap.register_dirty_field(flddescr, self) + count += 1 + return count class InstancePtrInfo(AbstractStructPtrInfo): _attrs_ = ('_known_class') @@ -102,25 +122,44 @@ def __init__(self, known_class=None, is_virtual=False): self._known_class = known_class - self._is_virtual = is_virtual + if is_virtual: + self.flags = FLAG_VIRTUAL def get_known_class(self, cpu): return self._known_class class StructPtrInfo(AbstractStructPtrInfo): - pass + def __init__(self, is_virtual=False): + if is_virtual: + self.flags = FLAG_VIRTUAL class ArrayPtrInfo(AbstractVirtualPtrInfo): - _attrs_ = ('_is_virtual', 'length', '_items', '_descr') + _attrs_ = ('length', '_items', '_descr') def __init__(self, descr, const, size, clear, is_virtual): - self._is_virtual = is_virtual + if is_virtual: + self.flags = FLAG_VIRTUAL self.length = size if clear: self._items = [const] * size else: self._items = [None] * size + def _force_elements(self, op, optforce): + arraydescr = op.getdescr() + count = 0 + for i in range(self.length): + item = self._items[i] + if item is not None: + subbox = optforce.force_box(item) + setop = ResOperation(rop.SETARRAYITEM_GC, + [op, ConstInt(i), subbox], + descr=arraydescr) + optforce.emit_operation(setop) + # xxxx optforce.optheap + count += 1 + return count + def setitem_virtual(self, index, item): self._items[index] = item @@ -134,7 +173,8 @@ def __init__(self, descr, size, is_virtual): self.length = size lgt = len(descr.all_interiorfielddescrs) - self._is_virtual = is_virtual + if is_virtual: + self.flags = FLAG_VIRTUAL self._items = [None] * (size * lgt) def _compute_index(self, index, fielddescr): @@ -152,6 +192,7 @@ def _force_elements(self, op, optforce): i = 0 fielddescrs = op.getdescr().all_interiorfielddescrs + count = 0 for index in range(self.length): for flddescr in fielddescrs: fld = self._items[i] @@ -161,7 +202,10 @@ [op, ConstInt(index), subbox], descr=flddescr) optforce.emit_operation(setfieldop) + # XXX optforce.optheap + count += 1 i += 1 + return count class StrPtrInfo(NonNullPtrInfo): _attrs_ = () diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py --- a/rpython/jit/metainterp/optimizeopt/intbounds.py +++ b/rpython/jit/metainterp/optimizeopt/intbounds.py @@ -71,21 +71,21 @@ optimize_GUARD_VALUE = _optimize_guard_true_false_value def optimize_INT_OR_or_XOR(self, op): - v1 = self.getvalue(op.getarg(0)) - v2 = self.getvalue(op.getarg(1)) - if v1.box is v2.box: + v1 = self.get_box_replacement(op.getarg(0)) + b1 = self.getintbound(v1) + v2 = self.get_box_replacement(op.getarg(1)) + b2 = self.getintbound(v2) + if v1 is v2: if op.getopnum() == rop.INT_OR: self.make_equal_to(op, v1) else: self.make_constant_int(op, 0) return self.emit_operation(op) - bound1 = v1.getintbound() - bound2 = v2.getintbound() - if bound1.known_ge(IntBound(0, 0)) and \ - bound2.known_ge(IntBound(0, 0)): - r = self.getvalue(op).getintbound() - mostsignificant = bound1.upper | bound2.upper + if b1.known_ge(IntBound(0, 0)) and \ + b2.known_ge(IntBound(0, 0)): + r = self.getintbound(op) + mostsignificant = b1.upper | b2.upper r.intersect(IntBound(0, next_pow2_m1(mostsignificant))) optimize_INT_OR = optimize_INT_OR_or_XOR diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -304,7 +304,7 @@ op.set_forwarded(opinfo) return opinfo - def getptrinfo(self, op): + def getptrinfo(self, op, create=False, is_object=False): assert op.type == 'r' op = self.get_box_replacement(op) assert op.type == 'r' @@ -325,8 +325,8 @@ def replace_op_with(self, op, newopnum, args=None, descr=None): return self.optimizer.replace_op_with(op, newopnum, args, descr) - def get_box_replacement(self, box): - return self.optimizer.get_box_replacement(box) + def ensure_ptr_info_arg0(self, op): + return self.optimizer.ensure_ptr_info_arg0(op) def make_constant(self, box, constbox): return self.optimizer.make_constant(box, constbox) @@ -575,6 +575,35 @@ return op.set_forwarded(info.NonNullPtrInfo()) + def ensure_ptr_info_arg0(self, op): + arg0 = self.get_box_replacement(op.getarg(0)) + if arg0.is_constant(): + return info.ConstPtrInfo(arg0) + opinfo = arg0.get_forwarded() + if isinstance(opinfo, info.AbstractVirtualPtrInfo): + return opinfo + assert opinfo is None or opinfo.__class__ is info.NonNullPtrInfo + if op.is_getfield() or op.getopnum() == rop.SETFIELD_GC: + is_object = op.getdescr().parent_descr.is_object() + if is_object: + opinfo = info.InstancePtrInfo() + else: + xxx + opinfo.init_fields(op.getdescr().parent_descr) + else: + yyy + arg0.set_forwarded(opinfo) + return opinfo + + def make_ptr_info(self, op, mode): + op = self.get_box_replacement(op) + if op.is_constant(): + return info.ConstPtrInfo(op) + opinfo = op.get_forwarded() + if isinstance(opinfo, info.AbstractVirtualPtrInfo): + return opinfo + xxx + def new_const(self, fieldofs): if fieldofs.is_pointer_field(): return self.cpu.ts.CONST_NULL diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -199,14 +199,14 @@ def test_remove_guard_class_2(self): ops = """ [i0] - p0 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p0 = new_with_vtable(descr=nodesize) escape_n(p0) guard_class(p0, ConstClass(node_vtable)) [] jump(i0) """ expected = """ [i0] - p0 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p0 = new_with_vtable(descr=nodesize) escape_n(p0) jump(i0) """ @@ -426,7 +426,7 @@ def test_ooisnull_oononnull_via_virtual(self): ops = """ [p0] - pv = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + pv = new_with_vtable(descr=nodesize) setfield_gc(pv, p0, descr=valuedescr) guard_nonnull(p0) [] p1 = getfield_gc_r(pv, descr=valuedescr) @@ -575,7 +575,7 @@ [i1, p2, p3] i3 = getfield_gc_i(p3, descr=valuedescr) escape_n(i3) - p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p1 = new_with_vtable(descr=nodesize) setfield_gc(p1, i1, descr=valuedescr) jump(i1, p1, p2) """ @@ -587,9 +587,9 @@ [i1, p2, p3] i3 = getfield_gc_i(p3, descr=valuedescr) escape_n(i3) - p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p1 = new_with_vtable(descr=nodesize) setfield_gc(p1, i1, descr=valuedescr) - p1sub = new_with_vtable(ConstClass(node_vtable2), descr=nodesize2) + p1sub = new_with_vtable(descr=nodesize2) setfield_gc(p1sub, i1, descr=valuedescr) setfield_gc(p1, p1sub, descr=nextdescr) jump(i1, p1, p2) @@ -604,10 +604,10 @@ p3sub = getfield_gc_r(p3, descr=nextdescr) i3 = getfield_gc_i(p3sub, descr=valuedescr) escape_n(i3) - p2sub = new_with_vtable(ConstClass(node_vtable2), descr=nodesize2) + p2sub = new_with_vtable(descr=nodesize2) setfield_gc(p2sub, i1, descr=valuedescr) setfield_gc(p2, p2sub, descr=nextdescr) - p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p1 = new_with_vtable(descr=nodesize) jump(i1, p1, p2) """ # The same as test_p123_simple, but in the end the "old" p2 contains @@ -736,6 +736,7 @@ expected) def test_virtual_2(self): + py.test.skip("XXX") ops = """ [i, p0] i0 = getfield_gc(p0, descr=valuedescr) @@ -749,7 +750,6 @@ i1 = int_add(i2, i) jump(i, i1) """ - py.test.skip("XXX") self.optimize_loop(ops, 'Not, Virtual(node_vtable, valuedescr=Not)', expected) @@ -807,6 +807,7 @@ self.optimize_loop(ops, expected2) def test_virtual_default_field(self): + py.test.skip("XXX") ops = """ [p0] i0 = getfield_gc(p0, descr=valuedescr) @@ -822,14 +823,13 @@ """ # the 'expected' is sub-optimal, but it should be done by another later # optimization step. See test_find_nodes_default_field() for why. - py.test.skip("XXX") self.optimize_loop(ops, 'Virtual(node_vtable, valuedescr=Not)', expected) def test_virtual_3(self): ops = """ [i] - p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p1 = new_with_vtable(descr=nodesize) setfield_gc(p1, i, descr=valuedescr) i0 = getfield_gc_i(p1, descr=valuedescr) i1 = int_add(i0, 1) @@ -843,13 +843,14 @@ self.optimize_loop(ops, expected) def test_virtual_4(self): + py.test.skip("XXX") ops = """ [i0, p0] guard_class(p0, ConstClass(node_vtable)) [] i1 = getfield_gc(p0, descr=valuedescr) i2 = int_sub(i1, 1) i3 = int_add(i0, i1) - p1 = new_with_vtable(ConstClass(node_vtable)) + p1 = new_with_vtable(descr=nodesize) setfield_gc(p1, i2, descr=valuedescr) jump(i3, p1) """ @@ -859,11 +860,11 @@ i3 = int_add(i0, i1) jump(i3, i2) """ - py.test.skip("XXX") self.optimize_loop(ops, 'Not, Virtual(node_vtable, valuedescr=Not)', expected) def test_virtual_5(self): + py.test.skip("XXX") ops = """ [i0, p0] guard_class(p0, ConstClass(node_vtable)) [] @@ -883,7 +884,6 @@ i3 = int_add(i0, i1) jump(i3, i2, i1) """ - py.test.skip("XXX") self.optimize_loop(ops, '''Not, Virtual(node_vtable, valuedescr=Not, @@ -894,7 +894,7 @@ def test_virtual_constant_isnull(self): ops = """ [i0] - p0 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p0 = new_with_vtable(descr=nodesize) setfield_gc(p0, NULL, descr=nextdescr) p2 = getfield_gc_r(p0, descr=nextdescr) i1 = ptr_eq(p2, NULL) @@ -909,7 +909,7 @@ def test_virtual_constant_isnonnull(self): ops = """ [i0] - p0 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p0 = new_with_vtable(descr=nodesize) setfield_gc(p0, ConstPtr(myptr), descr=nextdescr) p2 = getfield_gc_r(p0, descr=nextdescr) i1 = ptr_eq(p2, NULL) @@ -1020,7 +1020,7 @@ def test_nonvirtual_1(self): ops = """ [i] - p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p1 = new_with_vtable(descr=nodesize) setfield_gc(p1, i, descr=valuedescr) i0 = getfield_gc_i(p1, descr=valuedescr) i1 = int_add(i0, 1) @@ -1031,7 +1031,7 @@ expected = """ [i] i1 = int_add(i, 1) - p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p1 = new_with_vtable(descr=nodesize) setfield_gc(p1, i, descr=valuedescr) escape_n(p1) escape_n(p1) @@ -1045,7 +1045,7 @@ i0 = getfield_gc_i(p0, descr=valuedescr) escape_n(p0) i1 = int_add(i0, i) - p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p1 = new_with_vtable(descr=nodesize) setfield_gc(p1, i1, descr=valuedescr) jump(i, p1) """ @@ -1055,7 +1055,7 @@ def test_nonvirtual_later(self): ops = """ [i] - p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p1 = new_with_vtable(descr=nodesize) setfield_gc(p1, i, descr=valuedescr) i1 = getfield_gc_i(p1, descr=valuedescr) escape_n(p1) @@ -1065,7 +1065,7 @@ """ expected = """ [i] - p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p1 = new_with_vtable(descr=nodesize) setfield_gc(p1, i, descr=valuedescr) escape_n(p1) i2 = getfield_gc_i(p1, descr=valuedescr) @@ -1077,7 +1077,7 @@ def test_nonvirtual_write_null_fields_on_force(self): ops = """ [i] - p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p1 = new_with_vtable(descr=nodesize) setfield_gc(p1, i, descr=valuedescr) i1 = getfield_gc_i(p1, descr=valuedescr) setfield_gc(p1, 0, descr=valuedescr) @@ -1087,7 +1087,7 @@ """ expected = """ [i] - p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p1 = new_with_vtable(descr=nodesize) setfield_gc(p1, 0, descr=valuedescr) escape_n(p1) i2 = getfield_gc_i(p1, descr=valuedescr) @@ -1098,7 +1098,7 @@ def test_getfield_gc_pure_1(self): ops = """ [i] - p1 = new_with_vtable(ConstClass(node_vtable), descr=nodesize) + p1 = new_with_vtable(descr=nodesize) setfield_gc(p1, i, descr=valuedescr) i1 = getfield_gc_pure_i(p1, descr=valuedescr) jump(i1) @@ -1189,8 +1189,8 @@ """ expected = """ [i1, p0] + setarrayitem_gc(p0, 0, i1, descr=arraydescr) p1 = new_array(i1, descr=arraydescr) - setarrayitem_gc(p0, 0, i1, descr=arraydescr) jump(i1, p1) """ self.optimize_loop(ops, expected) @@ -1250,7 +1250,7 @@ def test_varray_forced_1(self): ops = """ [] - p2 = new_with_vtable(ConstClass(node_vtable)) + p2 = new_with_vtable(descr=nodesize) setfield_gc(p2, 3, descr=valuedescr) i1 = getfield_gc_i(p2, descr=valuedescr) # i1 = const 3 p1 = new_array(i1, descr=arraydescr) @@ -1270,6 +1270,7 @@ self.optimize_loop(ops, expected) def test_vstruct_1(self): + py.test.skip("XXX") ops = """ [i1, p2] i2 = getfield_gc(p2, descr=adescr) @@ -1283,7 +1284,6 @@ escape_n(i2) jump(i1, i1) """ - py.test.skip("XXX") self.optimize_loop(ops, 'Not, VStruct(ssize, adescr=Not)', expected) def test_p123_vstruct(self): @@ -1565,6 +1565,7 @@ self.optimize_loop(ops, expected) def test_duplicate_setfield_5(self): + xxx ops = """ [p0, i1] p1 = new_with_vtable(ConstClass(node_vtable)) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py --- a/rpython/jit/metainterp/optimizeopt/test/test_util.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py @@ -109,8 +109,8 @@ myptr2 = lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(NODE)) nullptr = lltype.nullptr(llmemory.GCREF.TO) #nodebox2 = InputArgRef(lltype.cast_opaque_ptr(llmemory.GCREF, node2)) - nodesize = cpu.sizeof(NODE) - nodesize2 = cpu.sizeof(NODE2) + nodesize = cpu.sizeof(NODE, True) + nodesize2 = cpu.sizeof(NODE2, True) valuedescr = cpu.fielddescrof(NODE, 'value') floatdescr = cpu.fielddescrof(NODE, 'floatval') chardescr = cpu.fielddescrof(NODE, 'charval') @@ -122,7 +122,7 @@ QUASI = lltype.GcStruct('QUASIIMMUT', ('inst_field', lltype.Signed), ('mutate_field', rclass.OBJECTPTR), hints={'immutable_fields': accessor}) - quasisize = cpu.sizeof(QUASI) + quasisize = cpu.sizeof(QUASI, False) quasi = lltype.malloc(QUASI, immortal=True) quasi.inst_field = -4247 quasifielddescr = cpu.fielddescrof(QUASI, 'inst_field') @@ -157,7 +157,7 @@ # a GcStruct not inheriting from OBJECT S = lltype.GcStruct('TUPLE', ('a', lltype.Signed), ('b', lltype.Ptr(NODE))) - ssize = cpu.sizeof(S) + ssize = cpu.sizeof(S, False) adescr = cpu.fielddescrof(S, 'a') bdescr = cpu.fielddescrof(S, 'b') #sbox = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S))) @@ -166,7 +166,7 @@ T = lltype.GcStruct('TUPLE', ('c', lltype.Signed), ('d', lltype.Ptr(lltype.GcArray(lltype.Ptr(NODE))))) - tsize = cpu.sizeof(T) + tsize = cpu.sizeof(T, False) cdescr = cpu.fielddescrof(T, 'c') ddescr = cpu.fielddescrof(T, 'd') arraydescr3 = cpu.arraydescrof(lltype.GcArray(lltype.Ptr(NODE))) @@ -176,7 +176,7 @@ ('one', lltype.Ptr(lltype.GcArray(lltype.Ptr(NODE))))) u_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True) u_vtable_adr = llmemory.cast_ptr_to_adr(u_vtable) - usize = cpu.sizeof(U) + usize = cpu.sizeof(U, True) onedescr = cpu.fielddescrof(U, 'one') FUNC = lltype.FuncType([lltype.Signed], lltype.Signed) diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -1,6 +1,6 @@ from rpython.jit.codewriter.effectinfo import EffectInfo from rpython.jit.metainterp.executor import execute -from rpython.jit.codewriter.heaptracker import vtable2descr +from rpython.jit.codewriter.heaptracker import vtable2descr, descr2vtable from rpython.jit.metainterp.history import Const, ConstInt, BoxInt from rpython.jit.metainterp.history import CONST_NULL, BoxPtr from rpython.jit.metainterp.optimizeopt import info, optimizer @@ -15,7 +15,6 @@ class AbstractVirtualInfo(info.PtrInfo): _attrs_ = ('_cached_vinfo',) - _tag = info.LEVEL_NONNULL is_about_raw = False _cached_vinfo = None @@ -188,7 +187,6 @@ fieldvalue.visitor_walk_recursive(visitor) class VirtualInfo(AbstractVirtualStructInfo): - _tag = info.LEVEL_KNOWNCLASS def __init__(self, known_class, descr): AbstractVirtualStructInfo.__init__(self) @@ -539,9 +537,10 @@ return opinfo def make_vstruct(self, structdescr, source_op): - vvalue = VStructValue(self.optimizer.cpu, structdescr, source_op) - self.make_equal_to(source_op, vvalue) - return vvalue + opinfo = info.StructPtrInfo(True) + opinfo.init_fields(structdescr) + source_op.set_forwarded(opinfo) + return opinfo def make_virtual_raw_memory(self, size, source_op): logops = self.optimizer.loop.logops @@ -678,7 +677,7 @@ # self.make_equal_to(op, fieldvalue) # return if opinfo and opinfo.is_virtual(): - fieldop = opinfo.getfield_virtual(op.getdescr()) + fieldop = opinfo.getfield(op.getdescr()) if fieldop is None: xxx fieldvalue = self.optimizer.new_const(op.getdescr()) @@ -698,14 +697,15 @@ def optimize_SETFIELD_GC(self, op): opinfo = self.getptrinfo(op.getarg(0)) if opinfo is not None and opinfo.is_virtual(): - opinfo.setfield_virtual(op.getdescr(), - self.get_box_replacement(op.getarg(1))) + opinfo.setfield(op.getdescr(), + self.get_box_replacement(op.getarg(1))) else: self.make_nonnull(op.getarg(0)) self.emit_operation(op) def optimize_NEW_WITH_VTABLE(self, op): - self.make_virtual(op.getarg(0), op, op.getdescr()) + known_class = ConstInt(descr2vtable(self.optimizer.cpu, op.getdescr())) + self.make_virtual(known_class, op, op.getdescr()) def optimize_NEW(self, op): self.make_vstruct(op.getdescr(), op) diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py --- a/rpython/jit/metainterp/optimizeopt/virtualstate.py +++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py @@ -3,8 +3,6 @@ ConstPtr, ConstFloat) from rpython.jit.metainterp.optimizeopt import virtualize from rpython.jit.metainterp.optimizeopt.intutils import IntUnbounded -from rpython.jit.metainterp.optimizeopt.info import (LEVEL_CONSTANT, - LEVEL_KNOWNCLASS, LEVEL_NONNULL, LEVEL_UNKNOWN) from rpython.jit.metainterp.resoperation import rop, ResOperation,\ AbstractInputArg from rpython.jit.metainterp.compile import Memo diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -223,6 +223,10 @@ def is_call(self): return rop._CALL_FIRST <= self.getopnum() <= rop._CALL_LAST + def is_getfield(self): + return self.opnum in (rop.GETFIELD_GC_I, rop.GETFIELD_GC_F, + rop.GETFIELD_GC_R) + def is_real_call(self): opnum = self.opnum return (opnum == rop.CALL_I or @@ -719,7 +723,7 @@ 'GETFIELD_RAW/1d/fi', '_MALLOC_FIRST', 'NEW/0d/r', #-> GcStruct, gcptrs inside are zeroed (not the rest) - 'NEW_WITH_VTABLE/1d/r',#-> GcStruct with vtable, gcptrs inside are zeroed + 'NEW_WITH_VTABLE/0d/r',#-> GcStruct with vtable, gcptrs inside are zeroed 'NEW_ARRAY/1d/r', #-> GcArray, not zeroed. only for arrays of primitives 'NEW_ARRAY_CLEAR/1d/r',#-> GcArray, fully zeroed 'NEWSTR/1/r', #-> STR, the hash field is zeroed From noreply at buildbot.pypy.org Wed Mar 11 16:23:30 2015 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 11 Mar 2015 16:23:30 +0100 (CET) Subject: [pypy-commit] pypy optresult: some cheap edits to get to 100 passed tests (metrics, metrics, metrics) Message-ID: <20150311152330.E2DB91C0EE0@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult Changeset: r76329:cc7eab561500 Date: 2015-03-11 17:23 +0200 http://bitbucket.org/pypy/pypy/changeset/cc7eab561500/ Log: some cheap edits to get to 100 passed tests (metrics, metrics, metrics) diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py --- a/rpython/jit/metainterp/optimizeopt/intbounds.py +++ b/rpython/jit/metainterp/optimizeopt/intbounds.py @@ -92,23 +92,22 @@ optimize_INT_XOR = optimize_INT_OR_or_XOR def optimize_INT_AND(self, op): - v1 = self.getvalue(op.getarg(0)) - v2 = self.getvalue(op.getarg(1)) + b1 = self.getintbound(op.getarg(0)) + b2 = self.getintbound(op.getarg(1)) self.emit_operation(op) - r = self.getvalue(op) - if v2.is_constant(): - val = v2.box.getint() + r = self.getintbound(op) + if b2.is_constant(): + val = b2.lower if val >= 0: r.getintbound().intersect(IntBound(0, val)) - elif v1.is_constant(): - val = v1.box.getint() + elif b1.is_constant(): + val = b1.getint() if val >= 0: r.getintbound().intersect(IntBound(0, val)) - elif v1.getintbound().known_ge(IntBound(0, 0)) and \ - v2.getintbound().known_ge(IntBound(0, 0)): - lesser = min(v1.getintbound().upper, v2.getintbound().upper) - r.getintbound().intersect(IntBound(0, next_pow2_m1(lesser))) + elif b1.known_ge(IntBound(0, 0)) and b2.known_ge(IntBound(0, 0)): + lesser = min(b1.upper, b2.upper) + r.intersect(IntBound(0, next_pow2_m1(lesser))) def optimize_INT_SUB(self, op): self.emit_operation(op) @@ -419,7 +418,7 @@ self.emit_operation(op) descr = op.getdescr() if descr and descr.is_item_integer_bounded(): - intbound = self.getvalue(op).getintbound() + intbound = self.getintbound(op) intbound.make_ge(IntLowerBound(descr.get_item_integer_min())) intbound.make_le(IntUpperBound(descr.get_item_integer_max())) diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -75,33 +75,33 @@ return False def optimize_INT_AND(self, op): - v1 = self.getvalue(op.getarg(0)) - v2 = self.getvalue(op.getarg(1)) - if v1.is_null() or v2.is_null(): + b1 = self.getintbound(op.getarg(0)) + b2 = self.getintbound(op.getarg(1)) + if b1.equal(0) or b2.equal(0): self.make_constant_int(op, 0) return - elif v2.is_constant(): - val = v2.box.getint() - if val == -1 or v1.getintbound().lower >= 0 \ - and v1.getintbound().upper <= val & ~(val + 1): - self.make_equal_to(op, v1) + elif b2.is_constant(): + val = b2.lower + if val == -1 or b1.lower >= 0 \ + and b1.upper <= val & ~(val + 1): + self.make_equal_to(op, op.getarg(0)) return - elif v1.is_constant(): - val = v1.box.getint() - if val == -1 or v2.getintbound().lower >= 0 \ - and v2.getintbound().upper <= val & ~(val + 1): - self.make_equal_to(op, v2) + elif b1.is_constant(): + val = b1.lower + if val == -1 or b2.lower >= 0 \ + and b2.upper <= val & ~(val + 1): + self.make_equal_to(op, op.getarg(1)) return self.emit_operation(op) def optimize_INT_OR(self, op): - v1 = self.getvalue(op.getarg(0)) - v2 = self.getvalue(op.getarg(1)) - if v1.is_null(): - self.make_equal_to(op, v2) - elif v2.is_null(): - self.make_equal_to(op, v1) + b1 = self.getintbound(op.getarg(0)) + b2 = self.getintbound(op.getarg(1)) + if b1.equal(0): + self.make_equal_to(op, op.getarg(1)) + elif b2.equal(0): + self.make_equal_to(op, op.getarg(0)) else: self.emit_operation(op) @@ -189,13 +189,13 @@ self.emit_operation(op) def optimize_INT_XOR(self, op): - v1 = self.getvalue(op.getarg(0)) - v2 = self.getvalue(op.getarg(1)) + b1 = self.getintbound(op.getarg(0)) + b2 = self.getintbound(op.getarg(1)) - if v1.is_constant() and v1.box.getint() == 0: - self.make_equal_to(op, v2) - elif v2.is_constant() and v2.box.getint() == 0: - self.make_equal_to(op, v1) + if b1.equal(0): + self.make_equal_to(op, op.getarg(1)) + elif b2.equal(0): + self.make_equal_to(op, op.getarg(0)) else: self.emit_operation(op) From noreply at buildbot.pypy.org Wed Mar 11 16:37:48 2015 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 11 Mar 2015 16:37:48 +0100 (CET) Subject: [pypy-commit] pypy optresult: a bit more of easy hacking Message-ID: <20150311153748.8BDD41C04BE@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult Changeset: r76330:ae0ec245d6ea Date: 2015-03-11 17:37 +0200 http://bitbucket.org/pypy/pypy/changeset/ae0ec245d6ea/ Log: a bit more of easy hacking diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py --- a/rpython/jit/metainterp/optimizeopt/intbounds.py +++ b/rpython/jit/metainterp/optimizeopt/intbounds.py @@ -100,11 +100,11 @@ if b2.is_constant(): val = b2.lower if val >= 0: - r.getintbound().intersect(IntBound(0, val)) + r.intersect(IntBound(0, val)) elif b1.is_constant(): - val = b1.getint() + val = b1.lower if val >= 0: - r.getintbound().intersect(IntBound(0, val)) + r.intersect(IntBound(0, val)) elif b1.known_ge(IntBound(0, 0)) and b2.known_ge(IntBound(0, 0)): lesser = min(b1.upper, b2.upper) r.intersect(IntBound(0, next_pow2_m1(lesser))) @@ -249,17 +249,19 @@ r.intersect(resbound) def optimize_INT_SUB_OVF(self, op): - v1 = self.getvalue(op.getarg(0)) - v2 = self.getvalue(op.getarg(1)) - if v1.box is v2.box: + arg0 = self.get_box_replacement(op.getarg(0)) + arg1 = self.get_box_replacement(op.getarg(1)) + b0 = self.getintbound(arg0) + b1 = self.getintbound(arg1) + if arg0.same_box(arg1): self.make_constant_int(op, 0) return - resbound = v1.getintbound().sub_bound(v2.getintbound()) + resbound = b0.sub_bound(b1) if resbound.bounded(): op = self.replace_op_with(op, rop.INT_SUB) self.emit_operation(op) # emit the op - r = self.getvalue(op) - r.getintbound().intersect(resbound) + r = self.getintbound(op) + r.intersect(resbound) def optimize_INT_MUL_OVF(self, op): v1 = self.getvalue(op.getarg(0)) @@ -320,13 +322,15 @@ self.emit_operation(op) def optimize_INT_EQ(self, op): - v1 = self.getvalue(op.getarg(0)) - v2 = self.getvalue(op.getarg(1)) - if v1.getintbound().known_gt(v2.getintbound()): + arg0 = self.get_box_replacement(op.getarg(0)) + arg1 = self.get_box_replacement(op.getarg(1)) + b1 = self.getintbound(op.getarg(0)) + b2 = self.getintbound(op.getarg(1)) + if b1.known_gt(b2): self.make_constant_int(op, 0) - elif v1.getintbound().known_lt(v2.getintbound()): + elif b1.known_lt(b2): self.make_constant_int(op, 0) - elif v1.box is v2.box: + elif arg0.same_box(arg1): self.make_constant_int(op, 1) else: self.emit_operation(op) @@ -401,9 +405,9 @@ self.emit_operation(op) descr = op.getdescr() if descr.is_integer_bounded(): - v1 = self.getvalue(op) - v1.getintbound().make_ge(IntLowerBound(descr.get_integer_min())) - v1.getintbound().make_le(IntUpperBound(descr.get_integer_max())) + b1 = self.getintbound(op) + b1.make_ge(IntLowerBound(descr.get_integer_min())) + b1.make_le(IntUpperBound(descr.get_integer_max())) optimize_GETFIELD_RAW_F = optimize_GETFIELD_RAW_I optimize_GETFIELD_GC_I = optimize_GETFIELD_RAW_I @@ -455,7 +459,7 @@ self.make_int_le(box2, box1) def propagate_bounds_INT_LT(self, op): - r = self.get_box_replacement(op) + r = self.getintbound(op) if r.is_constant(): if r.getint() == 1: self.make_int_lt(op.getarg(0), op.getarg(1)) @@ -464,7 +468,7 @@ self.make_int_ge(op.getarg(0), op.getarg(1)) def propagate_bounds_INT_GT(self, op): - r = self.get_box_replacement(op) + r = self.getintbound(op) if r.is_constant(): if r.getint() == 1: self.make_int_gt(op.getarg(0), op.getarg(1)) @@ -473,7 +477,7 @@ self.make_int_le(op.getarg(0), op.getarg(1)) def propagate_bounds_INT_LE(self, op): - r = self.get_box_replacement(op) + r = self.getintbound(op) if r.is_constant(): if r.getint() == 1: self.make_int_le(op.getarg(0), op.getarg(1)) @@ -482,7 +486,7 @@ self.make_int_gt(op.getarg(0), op.getarg(1)) def propagate_bounds_INT_GE(self, op): - r = self.get_box_replacement(op) + r = self.getintbound(op) if r.is_constant(): if r.getint() == 1: self.make_int_ge(op.getarg(0), op.getarg(1)) @@ -491,29 +495,29 @@ self.make_int_lt(op.getarg(0), op.getarg(1)) def propagate_bounds_INT_EQ(self, op): - r = self.getvalue(op) + r = self.getintbound(op) if r.is_constant(): - if r.box.same_constant(CONST_1): - v1 = self.getvalue(op.getarg(0)) - v2 = self.getvalue(op.getarg(1)) - if v1.getintbound().intersect(v2.getintbound()): - self.propagate_bounds_backward(op.getarg(0), v1) - if v2.getintbound().intersect(v1.getintbound()): - self.propagate_bounds_backward(op.getarg(1), v2) + if r.equal(1): + b1 = self.getintbound(op.getarg(0)) + b2 = self.getintbound(op.getarg(1)) + if b1.intersect(b2): + self.propagate_bounds_backward(op.getarg(0)) + if b2.intersect(b1): + self.propagate_bounds_backward(op.getarg(1)) def propagate_bounds_INT_NE(self, op): - r = self.getvalue(op) + r = self.getintbound(op) if r.is_constant(): - if r.box.same_constant(CONST_0): - v1 = self.getvalue(op.getarg(0)) - v2 = self.getvalue(op.getarg(1)) - if v1.getintbound().intersect(v2.getintbound()): - self.propagate_bounds_backward(op.getarg(0), v1) - if v2.getintbound().intersect(v1.getintbound()): - self.propagate_bounds_backward(op.getarg(1), v2) + if r.equal(0): + b1 = self.getintbound(op.getarg(0)) + b2 = self.getintbound(op.getarg(1)) + if b1.intersect(b2): + self.propagate_bounds_backward(op.getarg(0)) + if b2.intersect(b1): + self.propagate_bounds_backward(op.getarg(1)) def _propagate_int_is_true_or_zero(self, op, valnonzero, valzero): - r = self.get_box_replacement(op) + r = self.getintbound(op) if r.is_constant(): if r.getint() == valnonzero: b1 = self.getintbound(op.getarg(0)) diff --git a/rpython/jit/metainterp/optimizeopt/intutils.py b/rpython/jit/metainterp/optimizeopt/intutils.py --- a/rpython/jit/metainterp/optimizeopt/intutils.py +++ b/rpython/jit/metainterp/optimizeopt/intutils.py @@ -49,23 +49,15 @@ def is_constant(self): return self.has_upper and self.has_lower and self.lower == self.upper + def getint(self): + assert self.is_constant() + return self.lower + def equal(self, value): if not self.is_constant(): return False return self.lower == value - def make_constant(self, value): - XXXX # don't call me - self.has_lower = True - self.has_upper = True - self.lower = value - self.upper = value - - def make_unbounded(self): - XXX # hum - self.has_lower = False - self.has_upper = False - def bounded(self): return self.has_lower and self.has_upper diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -641,6 +641,13 @@ if op.returns_bool_result(): self.getintbound(op).make_bool() self._emit_operation(op) + op = self.get_box_replacement(op) + if op.type == 'i': + opinfo = op.get_forwarded() + if opinfo is not None: + assert isinstance(opinfo, IntBound) + if opinfo.is_constant(): + op.set_forwarded(ConstInt(opinfo.getint())) @specialize.argtype(0) def _emit_operation(self, op): From noreply at buildbot.pypy.org Wed Mar 11 18:36:36 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Wed, 11 Mar 2015 18:36:36 +0100 (CET) Subject: [pypy-commit] pypy default: Regularise implementations of FOR_ITER and op.next Message-ID: <20150311173636.60C2D1C149B@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r76331:dd46b08a538f Date: 2015-03-11 03:19 +0000 http://bitbucket.org/pypy/pypy/changeset/dd46b08a538f/ Log: Regularise implementations of FOR_ITER and op.next diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py --- a/rpython/flowspace/flowcontext.py +++ b/rpython/flowspace/flowcontext.py @@ -772,15 +772,10 @@ def FOR_ITER(self, target): w_iterator = self.peekvalue() - try: - w_nextitem = op.next(w_iterator).eval(self) - self.pushvalue(w_nextitem) - except Raise as e: - if self.exception_match(e.w_exc.w_type, const(StopIteration)): - self.popvalue() - return target - else: - raise + self.blockstack.append(IterBlock(self, target)) + w_nextitem = op.next(w_iterator).eval(self) + self.blockstack.pop() + self.pushvalue(w_nextitem) def SETUP_LOOP(self, target): block = LoopBlock(self, target) @@ -1333,6 +1328,16 @@ ctx.last_exception = w_exc return self.handlerposition # jump to the handler +class IterBlock(ExceptBlock): + """A pseudo-block to catch the StopIteration inside FOR_ITER""" + def handle(self, ctx, unroller): + w_exc = unroller.w_exc + if ctx.exception_match(w_exc.w_type, const(StopIteration)): + ctx.popvalue() + return self.handlerposition + else: + return ctx.unroll(unroller) + class FinallyBlock(FrameBlock): """A try:finally: block. Stores the position of the exception handler.""" diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py --- a/rpython/flowspace/operation.py +++ b/rpython/flowspace/operation.py @@ -555,7 +555,7 @@ opname = 'next' arity = 1 can_overflow = False - canraise = [] + canraise = [StopIteration, RuntimeError] pyfunc = staticmethod(next) def eval(self, ctx): @@ -571,9 +571,7 @@ else: ctx.replace_in_stack(it, next_unroller) return const(v) - w_item = ctx.do_op(self) - ctx.recorder.guessexception(ctx, StopIteration, RuntimeError) - return w_item + return HLOperation.eval(self, ctx) class GetAttr(SingleDispatchMixin, HLOperation): opname = 'getattr' From noreply at buildbot.pypy.org Wed Mar 11 19:58:37 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 11 Mar 2015 19:58:37 +0100 (CET) Subject: [pypy-commit] pypy vmprof: Add a skiplist implementation Message-ID: <20150311185837.30F981C00EF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof Changeset: r76332:4c96d8444ca3 Date: 2015-03-11 19:58 +0100 http://bitbucket.org/pypy/pypy/changeset/4c96d8444ca3/ Log: Add a skiplist implementation diff --git a/rpython/jit/backend/llsupport/src/skiplist.c b/rpython/jit/backend/llsupport/src/skiplist.c new file mode 100644 --- /dev/null +++ b/rpython/jit/backend/llsupport/src/skiplist.c @@ -0,0 +1,95 @@ +#include +#include + + +#define SKIPLIST_HEIGHT 8 + +typedef struct skipnode_s { + uintptr_t key; + char *data; + struct skipnode_s *next[SKIPLIST_HEIGHT]; /* may be smaller */ +} skipnode_t; + +static skipnode_t *skiplist_malloc(uintptr_t datasize) +{ + char *result; + uintptr_t basesize; + uintptr_t length = 1; + while (length < SKIPLIST_HEIGHT && (rand() & 3) == 0) + length++; + basesize = sizeof(skipnode_t) - + (SKIPLIST_HEIGHT - length) * sizeof(skipnode_t *); + result = malloc(basesize + datasize); + if (result != NULL) { + ((skipnode_t *)result)->data = result + basesize; + } + return (skipnode_t *)result; +} + +static skipnode_t *skiplist_search(skipnode_t *head, uintptr_t searchkey) +{ + /* Returns the skipnode with key closest (but <=) searchkey. + Note that if there is no item with key <= searchkey in the list, + this will return the head node. */ + uintptr_t level = SKIPLIST_HEIGHT - 1; + while (1) { + skipnode_t *next = head->next[level]; + if (next != NULL && next->key <= searchkey) { + head = next; + } + else { + if (level == 0) + break; + level -= 1; + } + } + return head; +} + +static void skiplist_insert(skipnode_t *head, skipnode_t *new) +{ + uintptr_t size0 = sizeof(skipnode_t) - + SKIPLIST_HEIGHT * sizeof(skipnode_t *); + uintptr_t height_of_new = (new->data - ((char *)new + size0)) / + sizeof(skipnode_t *); + + uintptr_t level = SKIPLIST_HEIGHT - 1; + uintptr_t searchkey = new->key; + while (1) { + skipnode_t *next = head->next[level]; + if (next != NULL && next->key <= searchkey) { + head = next; + } + else { + if (level < height_of_new) { + new->next[level] = next; + head->next[level] = new; + if (level == 0) + break; + } + level -= 1; + } + } +} + +static void skiplist_remove(skipnode_t *head, uintptr_t exact_key) +{ + uintptr_t level = SKIPLIST_HEIGHT - 1; + while (1) { + skipnode_t *next = head->next[level]; + if (next != NULL && next->key <= exact_key) { + if (next->key == exact_key) { + head->next[level] = next->next[level]; + if (level == 0) + return; /* successfully removed */ + level -= 1; + } + else + head = next; + } + else { + assert(level > 0); /* else, 'exact_key' not found! */ + level -= 1; + } + } +} diff --git a/rpython/jit/backend/llsupport/test/test_skiplist.py b/rpython/jit/backend/llsupport/test/test_skiplist.py new file mode 100644 --- /dev/null +++ b/rpython/jit/backend/llsupport/test/test_skiplist.py @@ -0,0 +1,85 @@ +import random, os +import cffi + +ffi = cffi.FFI() + +ffi.cdef(""" +typedef struct { + uintptr_t key; + char *data; + ...; +} skipnode_t; + +skipnode_t *skiplist_malloc(uintptr_t datasize); +skipnode_t *skiplist_search(skipnode_t *head, uintptr_t searchkey); +void skiplist_insert(skipnode_t *head, skipnode_t *new); +void skiplist_remove(skipnode_t *head, uintptr_t exact_key); +""") + +filename = os.path.join(os.path.dirname(__file__), '..', 'src', 'skiplist.c') +lib = ffi.verify(open(filename).read()) + + +def test_insert_search_remove(): + my_head = ffi.new("skipnode_t *") + assert lib.skiplist_search(my_head, 0) == my_head + # + keys = random.sample(xrange(2, 10**9), 50000) + nodes = {} + for key in keys: + node = lib.skiplist_malloc(4) + node.key = key + ffi.cast("int *", node.data)[0] = key + lib.skiplist_insert(my_head, node) + nodes[key] = node + # + random.shuffle(keys) + for key in keys: + node = lib.skiplist_search(my_head, key) + assert nodes[key] == node + if key + 1 not in nodes: + assert node == lib.skiplist_search(my_head, key + 1) + # + keys.sort() + following = {} + preceeding = {} + for key, next_key in zip(keys[:-1], keys[1:]): + following[key] = next_key + preceeding[next_key] = key + following[0] = keys[0] + following[keys[-1]] = 10**9 + preceeding[keys[0]] = 0 + # + for i in range(100000): + random_key = random.randrange(2, 10**9) + node = lib.skiplist_search(my_head, random_key) + assert node.key <= random_key + if node == my_head: + assert random_key < following[0] + else: + assert node == nodes[node.key] + assert following[node.key] > random_key + # + random_keys = list(keys) + random.shuffle(random_keys) + for i in range(10000): + node = nodes.pop(random_keys.pop()) + prev = preceeding[node.key] + next = following[node.key] + following[prev] = next + preceeding[next] = prev + lib.skiplist_remove(my_head, node.key) + if prev == 0: + assert lib.skiplist_search(my_head, node.key) == my_head + else: + assert lib.skiplist_search(my_head, node.key) == nodes[prev] + # + for i in range(100000): + random_key = random.randrange(2, 10**9) + node = lib.skiplist_search(my_head, random_key) + assert node.key <= random_key + if node == my_head: + assert random_key < following[0] + else: + assert node == nodes[node.key] + assert following[node.key] > random_key From noreply at buildbot.pypy.org Wed Mar 11 21:24:11 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 11 Mar 2015 21:24:11 +0100 (CET) Subject: [pypy-commit] pypy vmprof: in-progress Message-ID: <20150311202411.655C01C0221@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof Changeset: r76333:17b356ba6aa7 Date: 2015-03-11 20:30 +0100 http://bitbucket.org/pypy/pypy/changeset/17b356ba6aa7/ Log: in-progress diff --git a/pypy/module/_vmprof/src/get_custom_offset.c b/pypy/module/_vmprof/src/get_custom_offset.c --- a/pypy/module/_vmprof/src/get_custom_offset.c +++ b/pypy/module/_vmprof/src/get_custom_offset.c @@ -2,8 +2,8 @@ long pypy_jit_start_addr(); long pypy_jit_end_addr(); long pypy_jit_stack_depth_at_loc(long); -long pypy_find_codemap_at_addr(long); -long pypy_yield_codemap_at_addr(long, long, long*); +void *pypy_find_codemap_at_addr(long); +long pypy_yield_codemap_at_addr(void *, long, long *); extern volatile int pypy_codemap_currently_invalid; @@ -28,26 +28,20 @@ static long vmprof_write_header_for_jit_addr(void **result, long n, void *ip, int max_depth) { - long codemap_pos; + void *codemap; long current_pos = 0; intptr_t id; intptr_t addr = (intptr_t)ip; - if (addr < pypy_jit_start_addr() || addr > pypy_jit_end_addr()) { + codemap = pypy_find_codemap_at_addr(addr); + if (codemap == NULL) return n; + + while (n < max_depth) { + id = pypy_yield_codemap_at_addr(codemap, addr, ¤t_pos); + if (id == 0) + break; + result[n++] = (void *)id; } - codemap_pos = pypy_find_codemap_at_addr(addr); - if (codemap_pos == -1) { - return n; - } - while (1) { - id = pypy_yield_codemap_at_addr(codemap_pos, addr, ¤t_pos); - if (id == 0) { - return n; - } - result[n++] = id; - if (n >= max_depth) { - return n; - } - } + return n; } diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py --- a/rpython/jit/backend/llsupport/codemap.py +++ b/rpython/jit/backend/llsupport/codemap.py @@ -17,48 +17,13 @@ from rpython.rtyper.lltypesystem import lltype, rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo -INT_LIST = rffi.CArray(lltype.Signed) - -CODEMAP = lltype.Struct( - 'pypy_codemap_item', - ('addr', lltype.Signed), - ('machine_code_size', lltype.Signed), - ('bytecode_info_size', lltype.Signed), - ('bytecode_info', lltype.Ptr(INT_LIST)), - hints=dict(external=True, c_name='pypy_codemap_item')) -CODEMAP_LIST = rffi.CArray(CODEMAP) - -CODEMAP_STORAGE = lltype.Struct( - 'pypy_codemap_storage', - ('jit_addr_map_used', lltype.Signed), - ('jit_frame_depth_map_used', lltype.Signed), - ('jit_codemap_used', lltype.Signed), - ('jit_addr_map', lltype.Ptr(INT_LIST)), - ('jit_frame_depth_map', lltype.Ptr(INT_LIST)), - ('jit_codemap', lltype.Ptr(CODEMAP_LIST)), - hints=dict(external=True, c_name='pypy_codemap_storage')) - -CODEMAP_GCARRAY = lltype.GcArray(CODEMAP) - -_codemap = None eci = ExternalCompilationInfo(post_include_bits=[""" -RPY_EXTERN volatile int pypy_codemap_currently_invalid; -RPY_EXTERN void pypy_codemap_invalid_set(int); +RPY_EXTERN long pypy_jit_codemap_add(uintptr_t addr, long machine_code_size, + long *bytecode_info, + long bytecode_info_size); +RPY_EXTERN void pypy_jit_codemap_del(uintptr_t addr); -typedef struct pypy_codemap_item { - long addr, machine_code_size, bytecode_info_size; - long* bytecode_info; -} pypy_codemap_item; - -typedef struct pypy_codemap_storage { - long jit_addr_map_used; - long jit_frame_depth_map_used; - long jit_codemap_used; - long* jit_addr_map; - long* jit_frame_depth_map; - pypy_codemap_item* jit_codemap; -} pypy_codemap_storage; RPY_EXTERN pypy_codemap_storage *pypy_get_codemap_storage(); RPY_EXTERN long pypy_jit_stack_depth_at_loc(long loc); @@ -67,98 +32,9 @@ RPY_EXTERN long pypy_jit_end_addr(void); RPY_EXTERN long pypy_yield_codemap_at_addr(long, long, long*); -"""], separate_module_sources=[""" -volatile int pypy_codemap_currently_invalid = 0; - -static pypy_codemap_storage pypy_cs_g; - -static long bisect_right(long *a, long x, long hi) -{ - long lo, mid; - lo = 0; - while (lo < hi) { - mid = (lo+hi) / 2; - if (x < a[mid]) { hi = mid; } - else { lo = mid+1; } - } - return lo; -} - -static long bisect_right_addr(pypy_codemap_item *a, long x, long hi) -{ - long lo, mid; - lo = 0; - while (lo < hi) { - mid = (lo+hi) / 2; - if (x < a[mid].addr) { hi = mid; } - else { lo = mid+1; } - } - return lo; -} - -long pypy_jit_stack_depth_at_loc(long loc) -{ - long pos; - pos = bisect_right(pypy_cs_g.jit_addr_map, loc, - pypy_cs_g.jit_addr_map_used); - if (pos == 0) - return -1; - return pypy_cs_g.jit_frame_depth_map[pos - 1]; -} - -long pypy_find_codemap_at_addr(long addr) -{ - return bisect_right_addr(pypy_cs_g.jit_codemap, addr, - pypy_cs_g.jit_codemap_used) - 1; -} - -long pypy_jit_start_addr(void) -{ - return pypy_cs_g.jit_addr_map[0]; -} - -long pypy_jit_end_addr(void) -{ - return pypy_cs_g.jit_addr_map[pypy_cs_g.jit_addr_map_used - 1]; -} - -long pypy_yield_codemap_at_addr(long codemap_no, long addr, - long* current_pos_addr) -{ - // will return consecutive unique_ids from codemap, starting from position - // `pos` until addr - pypy_codemap_item *codemap = &(pypy_cs_g.jit_codemap[codemap_no]); - long current_pos = *current_pos_addr; - long start_addr = codemap->addr; - long rel_addr = addr - start_addr; - long next_start, next_stop; - - while (1) { - if (current_pos >= codemap->bytecode_info_size) - return 0; - next_start = codemap->bytecode_info[current_pos + 1]; - if (next_start > rel_addr) - return 0; - next_stop = codemap->bytecode_info[current_pos + 2]; - if (next_stop > rel_addr) { - *current_pos_addr = current_pos + 4; - return codemap->bytecode_info[current_pos]; - } - // we need to skip potentially more than one - current_pos = codemap->bytecode_info[current_pos + 3]; - } -} - -pypy_codemap_storage *pypy_get_codemap_storage(void) -{ - return &pypy_cs_g; -} - -void pypy_codemap_invalid_set(int value) -{ - pypy_codemap_currently_invalid = value; -} -"""]) +"""], separate_module_files=[ + os.path.join(os.path.dirname(__file__), 'src', 'codemap.c') +]) def llexternal(name, args, res): return rffi.llexternal(name, args, res, compilation_info=eci, @@ -190,15 +66,11 @@ dest[di] = source[si] + baseline class ListStorageMixin(object): - # XXX this code has wrong complexity, we should come up with a better - # data structure ideally _mixin_ = True - jit_addr_map_allocated = 0 - jit_codemap_allocated = 0 - jit_frame_depth_map_allocated = 0 @specialize.arg(1) def extend_with(self, name, to_insert, pos, baseline=0): + xxxx # first check if we need to reallocate g = pypy_get_codemap_storage() used = getattr(g, name + '_used') @@ -328,22 +200,13 @@ start) pypy_codemap_invalid_set(0) - def register_codemap(self, codemap): - start = codemap[0] - g = pypy_get_codemap_storage() - pos = bisect_left_addr(g.jit_codemap, start, g.jit_codemap_used) - items = lltype.malloc(INT_LIST, len(codemap[2]), flavor='raw', + def register_codemap(self, (start, size, l)): + items = lltype.malloc(INT_LIST, len(l), flavor='raw', track_allocation=False) - for i in range(len(codemap[2])): - items[i] = codemap[2][i] - s = lltype.malloc(CODEMAP_GCARRAY, 1) - s[0].addr = codemap[0] - s[0].machine_code_size = codemap[1] - s[0].bytecode_info = items - s[0].bytecode_info_size = len(codemap[2]) - pypy_codemap_invalid_set(1) - self.extend_with('jit_codemap', s, pos) - pypy_codemap_invalid_set(0) + for i in range(len(l)): + items[i] = l[i] + if pypy_jit_codemap_add(start, size, items, len(l)) < 0: + lltype.free(items, flavor='raw', track_allocation=False) def finish_once(self): self.free() diff --git a/rpython/jit/backend/llsupport/src/codemap.c b/rpython/jit/backend/llsupport/src/codemap.c new file mode 100644 --- /dev/null +++ b/rpython/jit/backend/llsupport/src/codemap.c @@ -0,0 +1,133 @@ +#include "skiplist.c" + +volatile int pypy_codemap_currently_invalid = 0; + +void pypy_codemap_invalid_set(int value) +{ + if (value) + __sync_lock_test_and_set(&pypy_codemap_currently_invalid, 1); + else + __sync_lock_release(&pypy_codemap_currently_invalid); +} + + +/************************************************************/ +/*** codemap storage ***/ +/************************************************************/ + +typedef struct { + long machine_code_size; + long *bytecode_info; + long bytecode_info_size; +} codemap_data_t; + +static skipnode_t jit_codemap_head; + +/*** interface used from codemap.py ***/ + +long pypy_jit_codemap_add(uintptr_t addr, long machine_code_size, + long *bytecode_info, long bytecode_info_size) +{ + skipnode_t *new = skiplist_malloc(sizeof(codemap_data_t)); + codemap_data_t *data; + if (new == NULL) + return -1; /* too bad */ + + new->key = addr; + data = (codemap_data_t *)new->data; + data->machine_code_size = machine_code_size; + data->bytecode_info = bytecode_info; + data->bytecode_info_size = bytecode_info_size; + + pypy_codemap_invalid_set(1); + skiplist_insert(&jit_codemap_head, new); + pypy_codemap_invalid_set(0); + return 0; +} + +void pypy_jit_codemap_del(uintptr_t addr) +{ + skiplist_remove(&jit_codemap_head, addr); +} + +/*** interface used from pypy/module/_vmprof ***/ + +void *pypy_find_codemap_at_addr(long addr) +{ + skiplist_t *codemap = skiplist_search(&jit_codemap_head, addr); + codemap_data_t *data; + uintptr_t rel_addr; + + if (codemap->key == NULL) + return NULL; + + rel_addr = (uintptr_t)addr - codemap->key; + data = (codemap_data_t *)codemap->data; + if (rel_addr >= data->machine_code_size) + return NULL; + + return (void *)codemap; +} + +long pypy_yield_codemap_at_addr(void *codemap_raw, long addr, + long *current_pos_addr) +{ + // will return consecutive unique_ids from codemap, starting from position + // `pos` until addr + skiplist_t *codemap = (skiplist_t *)codemap_raw; + long current_pos = *current_pos_addr; + long rel_addr = addr - codemap->key; + long next_start, next_stop; + codemap_data_t *data = (codemap_data_t *)codemap->data; + + while (1) { + if (current_pos >= data->bytecode_info_size) + return 0; + next_start = data->bytecode_info[current_pos + 1]; + if (next_start > rel_addr) + return 0; + next_stop = data->bytecode_info[current_pos + 2]; + if (next_stop > rel_addr) { + *current_pos_addr = current_pos + 4; + return data->bytecode_info[current_pos]; + } + // we need to skip potentially more than one + current_pos = data->bytecode_info[current_pos + 3]; + } +} + +/************************************************************/ + + + +long pypy_jit_stack_depth_at_loc(long loc) +{ + long pos; + pos = bisect_right(pypy_cs_g.jit_addr_map, loc, + pypy_cs_g.jit_addr_map_used); + if (pos == 0) + return -1; + return pypy_cs_g.jit_frame_depth_map[pos - 1]; +} + +long pypy_find_codemap_at_addr(long addr) +{ + return bisect_right_addr(pypy_cs_g.jit_codemap, addr, + pypy_cs_g.jit_codemap_used) - 1; +} + +long pypy_jit_start_addr(void) +{ + return pypy_cs_g.jit_addr_map[0]; +} + +long pypy_jit_end_addr(void) +{ + return pypy_cs_g.jit_addr_map[pypy_cs_g.jit_addr_map_used - 1]; +} + + +pypy_codemap_storage *pypy_get_codemap_storage(void) +{ + return &pypy_cs_g; +} From noreply at buildbot.pypy.org Wed Mar 11 21:24:12 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 11 Mar 2015 21:24:12 +0100 (CET) Subject: [pypy-commit] pypy vmprof: more in-progress Message-ID: <20150311202412.8C8071C0221@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof Changeset: r76334:0caa9fafbe64 Date: 2015-03-11 20:49 +0100 http://bitbucket.org/pypy/pypy/changeset/0caa9fafbe64/ Log: more in-progress diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py --- a/rpython/jit/backend/llsupport/codemap.py +++ b/rpython/jit/backend/llsupport/codemap.py @@ -19,18 +19,15 @@ eci = ExternalCompilationInfo(post_include_bits=[""" -RPY_EXTERN long pypy_jit_codemap_add(uintptr_t addr, long machine_code_size, +RPY_EXTERN long pypy_jit_codemap_add(uintptr_t addr, + unsigned int machine_code_size, long *bytecode_info, - long bytecode_info_size); + unsigned int bytecode_info_size); RPY_EXTERN void pypy_jit_codemap_del(uintptr_t addr); - -RPY_EXTERN pypy_codemap_storage *pypy_get_codemap_storage(); -RPY_EXTERN long pypy_jit_stack_depth_at_loc(long loc); -RPY_EXTERN long pypy_find_codemap_at_addr(long addr); -RPY_EXTERN long pypy_jit_start_addr(void); -RPY_EXTERN long pypy_jit_end_addr(void); -RPY_EXTERN long pypy_yield_codemap_at_addr(long, long, long*); +RPY_EXTERN long pypy_jit_depthmap_add(uintptr_t addr, unsigned int size, + unsigned int stackdepth); +RPY_EXTERN void pypy_jit_depthmap_clear(uintptr_t addr, unsigned int size); """], separate_module_files=[ os.path.join(os.path.dirname(__file__), 'src', 'codemap.c') diff --git a/rpython/jit/backend/llsupport/src/codemap.c b/rpython/jit/backend/llsupport/src/codemap.c --- a/rpython/jit/backend/llsupport/src/codemap.c +++ b/rpython/jit/backend/llsupport/src/codemap.c @@ -16,17 +16,17 @@ /************************************************************/ typedef struct { - long machine_code_size; + unsigned int machine_code_size; + unsigned int bytecode_info_size; long *bytecode_info; - long bytecode_info_size; } codemap_data_t; static skipnode_t jit_codemap_head; /*** interface used from codemap.py ***/ -long pypy_jit_codemap_add(uintptr_t addr, long machine_code_size, - long *bytecode_info, long bytecode_info_size) +long pypy_jit_codemap_add(uintptr_t addr, unsigned int machine_code_size, + long *bytecode_info, unsigned int bytecode_info_size) { skipnode_t *new = skiplist_malloc(sizeof(codemap_data_t)); codemap_data_t *data; @@ -47,7 +47,9 @@ void pypy_jit_codemap_del(uintptr_t addr) { + pypy_codemap_invalid_set(1); skiplist_remove(&jit_codemap_head, addr); + pypy_codemap_invalid_set(0); } /*** interface used from pypy/module/_vmprof ***/ @@ -58,7 +60,7 @@ codemap_data_t *data; uintptr_t rel_addr; - if (codemap->key == NULL) + if (codemap == &jit_codemap_head) return NULL; rel_addr = (uintptr_t)addr - codemap->key; @@ -97,37 +99,71 @@ } /************************************************************/ +/*** depthmap storage ***/ +/************************************************************/ +typedef struct { + unsigned int block_size; + unsigned int stack_depth; +} depthmap_data_t; +static skipnode_t jit_depthmap_head; + +/*** interface used from codemap.py ***/ + +long pypy_jit_depthmap_add(uintptr_t addr, unsigned int size, + unsigned int stackdepth) +{ + skipnode_t *new = skiplist_malloc(sizeof(depthmap_data_t)); + depthmap_data_t *data; + if (new == NULL) + return -1; /* too bad */ + + new->key = addr; + data = (depthmap_data_t *)new->data; + data->block_size = size; + data->stack_depth = stackdepth; + + pypy_codemap_invalid_set(1); + skiplist_insert(&jit_depthmap_head, new); + pypy_codemap_invalid_set(0); + return 0; +} + +void pypy_jit_depthmap_clear(uintptr_t addr, unsigned int size) +{ + uintptr_t search_key = addr + size - 1; + if (size == 0) + return; + + pypy_codemap_invalid_set(1); + while (1) { + /* search for all nodes belonging to the range, and remove them */ + skipnode_t *node = skiplist_search(&jit_depthmap_head, search_key); + if (node->addr < addr) + break; /* exhausted */ + skiplist_remove(&jit_depthmap_head, node->addr); + } + pypy_codemap_invalid_set(0); +} + +/*** interface used from pypy/module/_vmprof ***/ long pypy_jit_stack_depth_at_loc(long loc) { - long pos; - pos = bisect_right(pypy_cs_g.jit_addr_map, loc, - pypy_cs_g.jit_addr_map_used); - if (pos == 0) + skiplist_t *depthmap = skiplist_search(&jit_depthmap_head, (uintptr_t)loc); + depthmap_data_t *data; + uintptr_t rel_addr; + + if (depthmap == &jit_depthmap_head) return -1; - return pypy_cs_g.jit_frame_depth_map[pos - 1]; + + rel_addr = (uintptr_t)loc - depthmap->key; + data = (codemap_data_t *)depthmap->data; + if (rel_addr >= data->block_size) + return -1; + + return data->stack_depth; } -long pypy_find_codemap_at_addr(long addr) -{ - return bisect_right_addr(pypy_cs_g.jit_codemap, addr, - pypy_cs_g.jit_codemap_used) - 1; -} - -long pypy_jit_start_addr(void) -{ - return pypy_cs_g.jit_addr_map[0]; -} - -long pypy_jit_end_addr(void) -{ - return pypy_cs_g.jit_addr_map[pypy_cs_g.jit_addr_map_used - 1]; -} - - -pypy_codemap_storage *pypy_get_codemap_storage(void) -{ - return &pypy_cs_g; -} +/************************************************************/ From noreply at buildbot.pypy.org Wed Mar 11 21:43:51 2015 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 11 Mar 2015 21:43:51 +0100 (CET) Subject: [pypy-commit] pypy release-2.5.x: fix typo (rguillebert), add links Message-ID: <20150311204351.5A8971C00F8@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: release-2.5.x Changeset: r76335:c38583015ef8 Date: 2015-03-11 22:45 +0200 http://bitbucket.org/pypy/pypy/changeset/c38583015ef8/ Log: fix typo (rguillebert), add links diff --git a/pypy/doc/release-2.5.1.rst b/pypy/doc/release-2.5.1.rst --- a/pypy/doc/release-2.5.1.rst +++ b/pypy/doc/release-2.5.1.rst @@ -74,7 +74,7 @@ * errno and LastError are saved around cffi calls so things like pdb will not overwrite it -* We continue to asymtotically approach a score of 7 times faster than cpython +* We continue to asymptotically approach a score of 7 times faster than cpython on our benchmark suite, we now rank 6.98 on latest runs * Issues reported with our previous release were resolved_ after reports from users on @@ -85,9 +85,12 @@ .. _`RPython documentation`: http://rpython.readthedocs.org .. _resolved: http://doc.pypy.org/en/latest/whatsnew-2.5.1.html -Please try it out and let us know what you think. We especially welcome -success stories, we know you are using PyPy, please tell us about it! +Please try it out and let us know what you think. We welcome +success stories, `experiments`_, or `benchmarks`_, we know you are using PyPy, please tell us about it! Cheers The PyPy Team + +.. _`experiments`: http://morepypy.blogspot.com/2015/02/experiments-in-pyrlang-with-rpython.html +.. _`benchmarks`: https://mithrandi.net/blog/2015/03/axiom-benchmark-results-on-pypy-2-5-0 From noreply at buildbot.pypy.org Wed Mar 11 22:05:27 2015 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 11 Mar 2015 22:05:27 +0100 (CET) Subject: [pypy-commit] pypy release-2.5.x: update idices Message-ID: <20150311210527.DF4C61C00F8@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: release-2.5.x Changeset: r76336:cbd4090ee565 Date: 2015-03-11 22:55 +0200 http://bitbucket.org/pypy/pypy/changeset/cbd4090ee565/ Log: update idices diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-2.5.1.rst release-2.5.0.rst release-2.4.0.rst release-2.3.1.rst diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -7,6 +7,7 @@ .. toctree:: whatsnew-head.rst + whatsnew-2.5.1.rst whatsnew-2.5.0.rst whatsnew-2.4.0.rst whatsnew-2.3.1.rst From noreply at buildbot.pypy.org Thu Mar 12 01:12:34 2015 From: noreply at buildbot.pypy.org (mjacob) Date: Thu, 12 Mar 2015 01:12:34 +0100 (CET) Subject: [pypy-commit] pypy default: Update these two links which were pointing to the old bug tracker. Message-ID: <20150312001234.543491C0221@cobra.cs.uni-duesseldorf.de> Author: Manuel Jacob Branch: Changeset: r76337:0b9db7400f8b Date: 2015-03-12 01:12 +0100 http://bitbucket.org/pypy/pypy/changeset/0b9db7400f8b/ Log: Update these two links which were pointing to the old bug tracker. diff --git a/rpython/doc/arm.rst b/rpython/doc/arm.rst --- a/rpython/doc/arm.rst +++ b/rpython/doc/arm.rst @@ -160,5 +160,5 @@ The gcrootfinder option is needed to work around `issue 1377`_ and the jit-backend works around `issue 1376`_ -.. _issue 1377: https://bugs.pypy.org/issue1377 -.. _issue 1376: https://bugs.pypy.org/issue1376 +.. _issue 1377: https://bitbucket.org/pypy/pypy/issue/1377 +.. _issue 1376: https://bitbucket.org/pypy/pypy/issue/1376 From noreply at buildbot.pypy.org Thu Mar 12 11:00:50 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 12 Mar 2015 11:00:50 +0100 (CET) Subject: [pypy-commit] pypy vmprof: One new test passes Message-ID: <20150312100050.052801C0A8A@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof Changeset: r76338:402728053b00 Date: 2015-03-12 11:00 +0100 http://bitbucket.org/pypy/pypy/changeset/402728053b00/ Log: One new test passes diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py --- a/rpython/jit/backend/llsupport/codemap.py +++ b/rpython/jit/backend/llsupport/codemap.py @@ -9,6 +9,7 @@ """ +import os from rpython.rlib import rgc from rpython.rlib.objectmodel import specialize, we_are_translated from rpython.rlib.entrypoint import jit_entrypoint @@ -16,6 +17,10 @@ from rpython.rlib.rbisect import bisect_left, bisect_left_addr from rpython.rtyper.lltypesystem import lltype, rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo +from rpython.translator import cdir + + +INT_LIST_PTR = rffi.CArrayPtr(lltype.Signed) eci = ExternalCompilationInfo(post_include_bits=[""" @@ -23,7 +28,11 @@ unsigned int machine_code_size, long *bytecode_info, unsigned int bytecode_info_size); -RPY_EXTERN void pypy_jit_codemap_del(uintptr_t addr); +RPY_EXTERN long *pypy_jit_codemap_del(uintptr_t addr); +RPY_EXTERN uintptr_t pypy_jit_codemap_firstkey(void); +RPY_EXTERN void *pypy_find_codemap_at_addr(long addr); +RPY_EXTERN long pypy_yield_codemap_at_addr(void *codemap_raw, long addr, + long *current_pos_addr); RPY_EXTERN long pypy_jit_depthmap_add(uintptr_t addr, unsigned int size, unsigned int stackdepth); @@ -31,16 +40,21 @@ """], separate_module_files=[ os.path.join(os.path.dirname(__file__), 'src', 'codemap.c') -]) +], include_dirs=[cdir]) def llexternal(name, args, res): return rffi.llexternal(name, args, res, compilation_info=eci, releasegil=False) -ll_pypy_codemap_invalid_set = llexternal('pypy_codemap_invalid_set', - [rffi.INT], lltype.Void) -pypy_get_codemap_storage = llexternal('pypy_get_codemap_storage', - [], lltype.Ptr(CODEMAP_STORAGE)) +pypy_jit_codemap_add = llexternal('pypy_jit_codemap_add', + [lltype.Signed, lltype.Signed, + INT_LIST_PTR, lltype.Signed], + lltype.Signed) +pypy_jit_codemap_del = llexternal('pypy_jit_codemap_del', + [lltype.Signed], INT_LIST_PTR) +pypy_jit_codemap_firstkey = llexternal('pypy_jit_codemap_firstkey', + [], lltype.Signed) + stack_depth_at_loc = llexternal('pypy_jit_stack_depth_at_loc', [lltype.Signed], lltype.Signed) find_codemap_at_addr = llexternal('pypy_find_codemap_at_addr', @@ -50,10 +64,6 @@ rffi.CArrayPtr(lltype.Signed)], lltype.Signed) -def pypy_codemap_invalid_set(val): - #if we_are_translated(): - ll_pypy_codemap_invalid_set(val) - @specialize.ll() def copy_item(source, dest, si, di, baseline=0): TP = lltype.typeOf(dest) @@ -62,100 +72,23 @@ else: dest[di] = source[si] + baseline -class ListStorageMixin(object): - _mixin_ = True - @specialize.arg(1) - def extend_with(self, name, to_insert, pos, baseline=0): - xxxx - # first check if we need to reallocate - g = pypy_get_codemap_storage() - used = getattr(g, name + '_used') - allocated = getattr(self, name + '_allocated') - lst = getattr(g, name) - if used + len(to_insert) > allocated or pos != used: - old_lst = lst - if used + len(to_insert) > allocated: - new_size = max(4 * allocated, - (allocated + len(to_insert)) * 2) - else: - new_size = allocated - lst = lltype.malloc(lltype.typeOf(lst).TO, new_size, - flavor='raw', - track_allocation=False) - setattr(self, name + '_allocated', new_size) - for i in range(0, pos): - copy_item(old_lst, lst, i, i) - j = 0 - for i in range(pos, pos + len(to_insert)): - copy_item(to_insert, lst, j, i, baseline) - j += 1 - j = pos - for i in range(pos + len(to_insert), len(to_insert) + used): - copy_item(old_lst, lst, j, i) - j += 1 - self.free_lst(name, old_lst) - else: - for i in range(len(to_insert)): - copy_item(to_insert, lst, i, i + pos, baseline) - setattr(g, name, lst) - setattr(g, name + '_used', len(to_insert) + used) - - @specialize.arg(1) - def remove(self, name, start, end): - g = pypy_get_codemap_storage() - lst = getattr(g, name) - used = getattr(g, name + '_used') - j = end - for i in range(start, used - (end - start)): - info = lltype.nullptr(INT_LIST) - if name == 'jit_codemap': - if i < end: - info = lst[i].bytecode_info - copy_item(lst, lst, j, i) - if name == 'jit_codemap': - if info: - lltype.free(info, flavor='raw', track_allocation=False) - j += 1 - setattr(g, name + '_used', used - (end - start)) - - def free(self): - g = pypy_get_codemap_storage() - # if setup has not been called - if g.jit_addr_map_used: - lltype.free(g.jit_addr_map, flavor='raw', track_allocation=False) - g.jit_addr_map_used = 0 - g.jit_addr_map = lltype.nullptr(INT_LIST) - i = 0 - while i < g.jit_codemap_used: - lltype.free(g.jit_codemap[i].bytecode_info, flavor='raw', - track_allocation=False) - i += 1 - if g.jit_codemap_used: - lltype.free(g.jit_codemap, flavor='raw', - track_allocation=False) - g.jit_codemap_used = 0 - g.jit_codemap = lltype.nullptr(CODEMAP_LIST) - if g.jit_frame_depth_map_used: - lltype.free(g.jit_frame_depth_map, flavor='raw', - track_allocation=False) - g.jit_frame_depth_map_used = 0 - g.jit_frame_depth_map = lltype.nullptr(INT_LIST) - - @specialize.arg(1) - def free_lst(self, name, lst): - if lst: - lltype.free(lst, flavor='raw', track_allocation=False) - -class CodemapStorage(ListStorageMixin): +class CodemapStorage(object): """ An immortal wrapper around underlaying jit codemap data """ def setup(self): - g = pypy_get_codemap_storage() - if g.jit_addr_map_used != 0: - # someone failed to call free(), in tests only anyway + if not we_are_translated(): + # in case someone failed to call free(), in tests only anyway self.free() + def free(self): + while True: + key = pypy_jit_codemap_firstkey() + if not key: + break + items = pypy_jit_codemap_del(key) + lltype.free(items, flavor='raw', track_allocation=False) + def free_asm_block(self, start, stop): # fix up jit_addr_map g = pypy_get_codemap_storage() @@ -198,7 +131,7 @@ pypy_codemap_invalid_set(0) def register_codemap(self, (start, size, l)): - items = lltype.malloc(INT_LIST, len(l), flavor='raw', + items = lltype.malloc(INT_LIST_PTR.TO, len(l), flavor='raw', track_allocation=False) for i in range(len(l)): items[i] = l[i] @@ -209,14 +142,14 @@ self.free() def unpack_traceback(addr): - codemap_pos = find_codemap_at_addr(addr) - if codemap_pos == -1: + codemap_raw = find_codemap_at_addr(addr) + if not codemap_raw: return [] # no codemap for that position storage = lltype.malloc(rffi.CArray(lltype.Signed), 1, flavor='raw') storage[0] = 0 res = [] while True: - item = yield_bytecode_at_addr(codemap_pos, addr, storage) + item = yield_bytecode_at_addr(codemap_raw, addr, storage) if item == 0: break res.append(item) diff --git a/rpython/jit/backend/llsupport/src/codemap.c b/rpython/jit/backend/llsupport/src/codemap.c --- a/rpython/jit/backend/llsupport/src/codemap.c +++ b/rpython/jit/backend/llsupport/src/codemap.c @@ -1,3 +1,4 @@ +#include "src/precommondefs.h" #include "skiplist.c" volatile int pypy_codemap_currently_invalid = 0; @@ -25,6 +26,7 @@ /*** interface used from codemap.py ***/ +RPY_EXTERN long pypy_jit_codemap_add(uintptr_t addr, unsigned int machine_code_size, long *bytecode_info, unsigned int bytecode_info_size) { @@ -45,18 +47,35 @@ return 0; } -void pypy_jit_codemap_del(uintptr_t addr) +RPY_EXTERN +long *pypy_jit_codemap_del(uintptr_t addr) { + long *result; + skipnode_t *node; + pypy_codemap_invalid_set(1); - skiplist_remove(&jit_codemap_head, addr); + node = skiplist_remove(&jit_codemap_head, addr); pypy_codemap_invalid_set(0); + + if (node == NULL) + return NULL; + result = ((codemap_data_t *)node->data)->bytecode_info; + free(node); + return result; +} + +RPY_EXTERN +uintptr_t pypy_jit_codemap_firstkey(void) +{ + return skiplist_firstkey(&jit_codemap_head); } /*** interface used from pypy/module/_vmprof ***/ +RPY_EXTERN void *pypy_find_codemap_at_addr(long addr) { - skiplist_t *codemap = skiplist_search(&jit_codemap_head, addr); + skipnode_t *codemap = skiplist_search(&jit_codemap_head, addr); codemap_data_t *data; uintptr_t rel_addr; @@ -71,12 +90,13 @@ return (void *)codemap; } +RPY_EXTERN long pypy_yield_codemap_at_addr(void *codemap_raw, long addr, long *current_pos_addr) { // will return consecutive unique_ids from codemap, starting from position // `pos` until addr - skiplist_t *codemap = (skiplist_t *)codemap_raw; + skipnode_t *codemap = (skipnode_t *)codemap_raw; long current_pos = *current_pos_addr; long rel_addr = addr - codemap->key; long next_start, next_stop; @@ -132,6 +152,8 @@ void pypy_jit_depthmap_clear(uintptr_t addr, unsigned int size) { + abort(); +#if 0 uintptr_t search_key = addr + size - 1; if (size == 0) return; @@ -145,13 +167,14 @@ skiplist_remove(&jit_depthmap_head, node->addr); } pypy_codemap_invalid_set(0); +#endif } /*** interface used from pypy/module/_vmprof ***/ long pypy_jit_stack_depth_at_loc(long loc) { - skiplist_t *depthmap = skiplist_search(&jit_depthmap_head, (uintptr_t)loc); + skipnode_t *depthmap = skiplist_search(&jit_depthmap_head, (uintptr_t)loc); depthmap_data_t *data; uintptr_t rel_addr; diff --git a/rpython/jit/backend/llsupport/src/skiplist.c b/rpython/jit/backend/llsupport/src/skiplist.c --- a/rpython/jit/backend/llsupport/src/skiplist.c +++ b/rpython/jit/backend/llsupport/src/skiplist.c @@ -72,7 +72,7 @@ } } -static void skiplist_remove(skipnode_t *head, uintptr_t exact_key) +static skipnode_t *skiplist_remove(skipnode_t *head, uintptr_t exact_key) { uintptr_t level = SKIPLIST_HEIGHT - 1; while (1) { @@ -81,15 +81,23 @@ if (next->key == exact_key) { head->next[level] = next->next[level]; if (level == 0) - return; /* successfully removed */ + return next; /* successfully removed */ level -= 1; } else head = next; } else { - assert(level > 0); /* else, 'exact_key' not found! */ + if (level == 0) + return NULL; /* 'exact_key' not found! */ level -= 1; } } } + +static uintptr_t skiplist_firstkey(skipnode_t *head) +{ + if (head->next[0] == NULL) + return 0; + return head->next[0]->key; +} diff --git a/rpython/jit/backend/llsupport/test/test_codemap.py b/rpython/jit/backend/llsupport/test/test_codemap.py --- a/rpython/jit/backend/llsupport/test/test_codemap.py +++ b/rpython/jit/backend/llsupport/test/test_codemap.py @@ -1,10 +1,28 @@ from rpython.jit.backend.llsupport.codemap import stack_depth_at_loc -from rpython.jit.backend.llsupport.codemap import CodemapStorage,\ - ListStorageMixin, CodemapBuilder, unpack_traceback,\ - pypy_get_codemap_storage +from rpython.jit.backend.llsupport.codemap import CodemapStorage, \ + CodemapBuilder, unpack_traceback, find_codemap_at_addr -g = pypy_get_codemap_storage() +def test_register_codemap(): + codemap = CodemapStorage() + codemap.setup() + codemap.register_codemap((100, 20, [13, 14, 15])) + codemap.register_codemap((300, 30, [16, 17, 18])) + codemap.register_codemap((200, 100, [19, 20, 21, 22, 23])) + # + raw100 = find_codemap_at_addr(100) + assert find_codemap_at_addr(119) == raw100 + assert not find_codemap_at_addr(120) + # + raw200 = find_codemap_at_addr(200) + assert raw200 != raw100 + assert find_codemap_at_addr(299) == raw200 + # + raw300 = find_codemap_at_addr(329) + assert raw300 != raw100 and raw300 != raw200 + assert find_codemap_at_addr(300) == raw300 + # + codemap.free() def test_list_storage_mixin(): class X(ListStorageMixin): diff --git a/rpython/jit/backend/llsupport/test/test_skiplist.py b/rpython/jit/backend/llsupport/test/test_skiplist.py --- a/rpython/jit/backend/llsupport/test/test_skiplist.py +++ b/rpython/jit/backend/llsupport/test/test_skiplist.py @@ -13,7 +13,8 @@ skipnode_t *skiplist_malloc(uintptr_t datasize); skipnode_t *skiplist_search(skipnode_t *head, uintptr_t searchkey); void skiplist_insert(skipnode_t *head, skipnode_t *new); -void skiplist_remove(skipnode_t *head, uintptr_t exact_key); +skipnode_t *skiplist_remove(skipnode_t *head, uintptr_t exact_key); +uintptr_t skiplist_firstkey(skipnode_t *head); """) filename = os.path.join(os.path.dirname(__file__), '..', 'src', 'skiplist.c') @@ -68,11 +69,14 @@ next = following[node.key] following[prev] = next preceeding[next] = prev - lib.skiplist_remove(my_head, node.key) + res = lib.skiplist_remove(my_head, node.key) + assert res == node if prev == 0: assert lib.skiplist_search(my_head, node.key) == my_head else: assert lib.skiplist_search(my_head, node.key) == nodes[prev] + res = lib.skiplist_remove(my_head, node.key) + assert res == ffi.NULL # for i in range(100000): random_key = random.randrange(2, 10**9) From noreply at buildbot.pypy.org Thu Mar 12 11:03:16 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 12 Mar 2015 11:03:16 +0100 (CET) Subject: [pypy-commit] pypy vmprof: test_codemaps works too Message-ID: <20150312100316.286BA1C0A8A@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof Changeset: r76339:b1369ce0eb67 Date: 2015-03-12 11:03 +0100 http://bitbucket.org/pypy/pypy/changeset/b1369ce0eb67/ Log: test_codemaps works too diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py --- a/rpython/jit/backend/llsupport/codemap.py +++ b/rpython/jit/backend/llsupport/codemap.py @@ -90,24 +90,27 @@ lltype.free(items, flavor='raw', track_allocation=False) def free_asm_block(self, start, stop): - # fix up jit_addr_map - g = pypy_get_codemap_storage() - jit_adr_start = bisect_left(g.jit_addr_map, start, - g.jit_addr_map_used) - jit_adr_stop = bisect_left(g.jit_addr_map, stop, - g.jit_addr_map_used) - pypy_codemap_invalid_set(1) - self.remove('jit_addr_map', jit_adr_start, jit_adr_stop) - self.remove('jit_frame_depth_map', jit_adr_start, jit_adr_stop) - # fix up codemap - # (there should only be zero or one codemap entry in that range, - # but still we use a range to distinguish between zero and one) - codemap_adr_start = bisect_left_addr(g.jit_codemap, start, - g.jit_codemap_used) - codemap_adr_stop = bisect_left_addr(g.jit_codemap, stop, - g.jit_codemap_used) - self.remove('jit_codemap', codemap_adr_start, codemap_adr_stop) - pypy_codemap_invalid_set(0) + items = pypy_jit_codemap_del(start) + if items: + lltype.free(items, flavor='raw', track_allocation=False) + ## # fix up jit_addr_map + ## g = pypy_get_codemap_storage() + ## jit_adr_start = bisect_left(g.jit_addr_map, start, + ## g.jit_addr_map_used) + ## jit_adr_stop = bisect_left(g.jit_addr_map, stop, + ## g.jit_addr_map_used) + ## pypy_codemap_invalid_set(1) + ## self.remove('jit_addr_map', jit_adr_start, jit_adr_stop) + ## self.remove('jit_frame_depth_map', jit_adr_start, jit_adr_stop) + ## # fix up codemap + ## # (there should only be zero or one codemap entry in that range, + ## # but still we use a range to distinguish between zero and one) + ## codemap_adr_start = bisect_left_addr(g.jit_codemap, start, + ## g.jit_codemap_used) + ## codemap_adr_stop = bisect_left_addr(g.jit_codemap, stop, + ## g.jit_codemap_used) + ## self.remove('jit_codemap', codemap_adr_start, codemap_adr_stop) + ## pypy_codemap_invalid_set(0) def register_frame_depth_map(self, rawstart, frame_positions, frame_assignments): From noreply at buildbot.pypy.org Thu Mar 12 11:18:21 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 12 Mar 2015 11:18:21 +0100 (CET) Subject: [pypy-commit] pypy vmprof: test_codemap now passes Message-ID: <20150312101821.399001C0557@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof Changeset: r76340:92b353fa58de Date: 2015-03-12 11:18 +0100 http://bitbucket.org/pypy/pypy/changeset/92b353fa58de/ Log: test_codemap now passes diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py --- a/rpython/jit/backend/llsupport/codemap.py +++ b/rpython/jit/backend/llsupport/codemap.py @@ -55,6 +55,13 @@ pypy_jit_codemap_firstkey = llexternal('pypy_jit_codemap_firstkey', [], lltype.Signed) +pypy_jit_depthmap_add = llexternal('pypy_jit_depthmap_add', + [lltype.Signed, lltype.Signed, + lltype.Signed], lltype.Signed) +pypy_jit_depthmap_clear = llexternal('pypy_jit_depthmap_clear', + [lltype.Signed, lltype.Signed], + lltype.Void) + stack_depth_at_loc = llexternal('pypy_jit_stack_depth_at_loc', [lltype.Signed], lltype.Signed) find_codemap_at_addr = llexternal('pypy_find_codemap_at_addr', @@ -64,14 +71,6 @@ rffi.CArrayPtr(lltype.Signed)], lltype.Signed) - at specialize.ll() -def copy_item(source, dest, si, di, baseline=0): - TP = lltype.typeOf(dest) - if isinstance(TP.TO.OF, lltype.Struct): - rgc.copy_struct_item(source, dest, si, di) - else: - dest[di] = source[si] + baseline - class CodemapStorage(object): """ An immortal wrapper around underlaying jit codemap data @@ -93,45 +92,20 @@ items = pypy_jit_codemap_del(start) if items: lltype.free(items, flavor='raw', track_allocation=False) - ## # fix up jit_addr_map - ## g = pypy_get_codemap_storage() - ## jit_adr_start = bisect_left(g.jit_addr_map, start, - ## g.jit_addr_map_used) - ## jit_adr_stop = bisect_left(g.jit_addr_map, stop, - ## g.jit_addr_map_used) - ## pypy_codemap_invalid_set(1) - ## self.remove('jit_addr_map', jit_adr_start, jit_adr_stop) - ## self.remove('jit_frame_depth_map', jit_adr_start, jit_adr_stop) - ## # fix up codemap - ## # (there should only be zero or one codemap entry in that range, - ## # but still we use a range to distinguish between zero and one) - ## codemap_adr_start = bisect_left_addr(g.jit_codemap, start, - ## g.jit_codemap_used) - ## codemap_adr_stop = bisect_left_addr(g.jit_codemap, stop, - ## g.jit_codemap_used) - ## self.remove('jit_codemap', codemap_adr_start, codemap_adr_stop) - ## pypy_codemap_invalid_set(0) + pypy_jit_depthmap_clear(start, stop - start) - def register_frame_depth_map(self, rawstart, frame_positions, + def register_frame_depth_map(self, rawstart, rawstop, frame_positions, frame_assignments): if not frame_positions: return - pypy_codemap_invalid_set(1) - g = pypy_get_codemap_storage() - if (not g.jit_addr_map_used or - rawstart > g.jit_addr_map[g.jit_addr_map_used - 1]): - start = g.jit_addr_map_used - self.extend_with('jit_addr_map', frame_positions, - g.jit_addr_map_used, rawstart) - self.extend_with('jit_frame_depth_map', frame_assignments, - g.jit_frame_depth_map_used) - else: - start = bisect_left(g.jit_addr_map, rawstart, - g.jit_addr_map_used) - self.extend_with('jit_addr_map', frame_positions, start, rawstart) - self.extend_with('jit_frame_depth_map', frame_assignments, - start) - pypy_codemap_invalid_set(0) + assert len(frame_positions) == len(frame_assignments) + for i in range(len(frame_positions)-1, -1, -1): + pos = rawstart + frame_positions[i] + length = rawstop - pos + if length > 0: + print "ADD:", pos, length, frame_assignments[i] + pypy_jit_depthmap_add(pos, length, frame_assignments[i]) + rawstop = pos def register_codemap(self, (start, size, l)): items = lltype.malloc(INT_LIST_PTR.TO, len(l), flavor='raw', diff --git a/rpython/jit/backend/llsupport/src/codemap.c b/rpython/jit/backend/llsupport/src/codemap.c --- a/rpython/jit/backend/llsupport/src/codemap.c +++ b/rpython/jit/backend/llsupport/src/codemap.c @@ -131,6 +131,7 @@ /*** interface used from codemap.py ***/ +RPY_EXTERN long pypy_jit_depthmap_add(uintptr_t addr, unsigned int size, unsigned int stackdepth) { @@ -150,10 +151,9 @@ return 0; } +RPY_EXTERN void pypy_jit_depthmap_clear(uintptr_t addr, unsigned int size) { - abort(); -#if 0 uintptr_t search_key = addr + size - 1; if (size == 0) return; @@ -162,16 +162,17 @@ while (1) { /* search for all nodes belonging to the range, and remove them */ skipnode_t *node = skiplist_search(&jit_depthmap_head, search_key); - if (node->addr < addr) + if (node->key < addr) break; /* exhausted */ - skiplist_remove(&jit_depthmap_head, node->addr); + skiplist_remove(&jit_depthmap_head, node->key); + free(node); } pypy_codemap_invalid_set(0); -#endif } /*** interface used from pypy/module/_vmprof ***/ +RPY_EXTERN long pypy_jit_stack_depth_at_loc(long loc) { skipnode_t *depthmap = skiplist_search(&jit_depthmap_head, (uintptr_t)loc); diff --git a/rpython/jit/backend/llsupport/test/test_codemap.py b/rpython/jit/backend/llsupport/test/test_codemap.py --- a/rpython/jit/backend/llsupport/test/test_codemap.py +++ b/rpython/jit/backend/llsupport/test/test_codemap.py @@ -24,46 +24,32 @@ # codemap.free() -def test_list_storage_mixin(): - class X(ListStorageMixin): - def unpack(self): - return [g.jit_addr_map[i] for i in range(g.jit_addr_map_used)] - - x = X() - x.extend_with('jit_addr_map', [1, 2, 3], 0) - assert x.unpack() == [1, 2, 3] - x.extend_with('jit_addr_map', [4, 5, 6], 3) - assert x.unpack() == [1, 2, 3, 4, 5, 6] - x.extend_with('jit_addr_map', [7, 8, 9], 2, baseline=10) - assert x.unpack() == [1, 2, 17, 18, 19, 3, 4, 5, 6] - x.remove('jit_addr_map', 3, 6) - assert x.unpack() == [1, 2, 17, 4, 5, 6] - x.extend_with('jit_addr_map', [1] * 6, 6) - assert x.unpack() == [1, 2, 17, 4, 5, 6, 1, 1, 1, 1, 1, 1] - x.extend_with('jit_addr_map', [10] * 4, 5) - assert x.unpack() == [1, 2, 17, 4, 5, 10, 10, 10, 10, 6, - 1, 1, 1, 1, 1, 1] - x.free() - def test_find_jit_frame_depth(): codemap = CodemapStorage() codemap.setup() - codemap.register_frame_depth_map(11, [0, 5, 10], [1, 2, 3]) - codemap.register_frame_depth_map(30, [0, 5, 10], [4, 5, 6]) - codemap.register_frame_depth_map(0, [0, 5, 10], [7, 8, 9]) + codemap.register_frame_depth_map(11, 26, [0, 5, 10], [1, 2, 3]) + codemap.register_frame_depth_map(30, 41, [0, 5, 10], [4, 5, 6]) + codemap.register_frame_depth_map(0, 11, [0, 5, 10], [7, 8, 9]) assert stack_depth_at_loc(13) == 1 assert stack_depth_at_loc(-3) == -1 + assert stack_depth_at_loc(40) == 6 assert stack_depth_at_loc(41) == -1 assert stack_depth_at_loc(5) == 8 assert stack_depth_at_loc(17) == 2 assert stack_depth_at_loc(38) == 5 - codemap.free_asm_block(11, 22) - assert stack_depth_at_loc(13) == 9 + assert stack_depth_at_loc(25) == 3 + assert stack_depth_at_loc(26) == -1 + assert stack_depth_at_loc(11) == 1 + assert stack_depth_at_loc(10) == 9 + codemap.free_asm_block(11, 26) + assert stack_depth_at_loc(11) == -1 + assert stack_depth_at_loc(13) == -1 assert stack_depth_at_loc(-3) == -1 + assert stack_depth_at_loc(40) == 6 assert stack_depth_at_loc(41) == -1 assert stack_depth_at_loc(5) == 8 - assert stack_depth_at_loc(17) == 9 assert stack_depth_at_loc(38) == 5 + assert stack_depth_at_loc(10) == 9 codemap.free() def test_codemaps(): From noreply at buildbot.pypy.org Thu Mar 12 11:20:11 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 12 Mar 2015 11:20:11 +0100 (CET) Subject: [pypy-commit] pypy vmprof: Update Message-ID: <20150312102011.9C8981C0EEE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof Changeset: r76341:d5feaf337db0 Date: 2015-03-12 11:20 +0100 http://bitbucket.org/pypy/pypy/changeset/d5feaf337db0/ Log: Update diff --git a/rpython/jit/backend/llsupport/asmmemmgr.py b/rpython/jit/backend/llsupport/asmmemmgr.py --- a/rpython/jit/backend/llsupport/asmmemmgr.py +++ b/rpython/jit/backend/llsupport/asmmemmgr.py @@ -314,7 +314,8 @@ assert gcrootmap is not None for pos, mark in self.gcroot_markers: gcrootmap.register_asm_addr(rawstart + pos, mark) - cpu.codemap.register_frame_depth_map(rawstart, self.frame_positions, + cpu.codemap.register_frame_depth_map(rawstart, rawstart + size, + self.frame_positions, self.frame_assignments) self.frame_positions = None self.frame_assignments = None From noreply at buildbot.pypy.org Thu Mar 12 11:33:10 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 12 Mar 2015 11:33:10 +0100 (CET) Subject: [pypy-commit] pypy vmprof: Fixes, untested Message-ID: <20150312103310.D6E1E1C0EEE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof Changeset: r76342:de655e4feb4e Date: 2015-03-12 11:29 +0100 http://bitbucket.org/pypy/pypy/changeset/de655e4feb4e/ Log: Fixes, untested diff --git a/pypy/module/_vmprof/src/fake_pypy_api.c b/pypy/module/_vmprof/src/fake_pypy_api.c --- a/pypy/module/_vmprof/src/fake_pypy_api.c +++ b/pypy/module/_vmprof/src/fake_pypy_api.c @@ -1,25 +1,15 @@ - -long pypy_jit_start_addr(void) -{ - return 3; -} - -long pypy_jit_end_addr(void) -{ - return 3; -} long pypy_jit_stack_depth_at_loc(long x) { return 0; } -long pypy_find_codemap_at_addr(long x) +void *pypy_find_codemap_at_addr(long x) { - return 0; + return NULL; } -long pypy_yield_codemap_at_addr(long x, long y, long *a) +long pypy_yield_codemap_at_addr(void *x, long y, long *a) { return 0; } diff --git a/pypy/module/_vmprof/src/get_custom_offset.c b/pypy/module/_vmprof/src/get_custom_offset.c --- a/pypy/module/_vmprof/src/get_custom_offset.c +++ b/pypy/module/_vmprof/src/get_custom_offset.c @@ -1,12 +1,12 @@ - -long pypy_jit_start_addr(); -long pypy_jit_end_addr(); -long pypy_jit_stack_depth_at_loc(long); -void *pypy_find_codemap_at_addr(long); -long pypy_yield_codemap_at_addr(void *, long, long *); extern volatile int pypy_codemap_currently_invalid; +void *pypy_find_codemap_at_addr(long addr); +long pypy_yield_codemap_at_addr(void *codemap_raw, long addr, + long *current_pos_addr); +long pypy_jit_stack_depth_at_loc(long loc); + + void vmprof_set_tramp_range(void* start, void* end) { } @@ -18,11 +18,7 @@ static ptrdiff_t vmprof_unw_get_custom_offset(void* ip, unw_cursor_t *cp) { intptr_t ip_l = (intptr_t)ip; - - if (ip_l < pypy_jit_start_addr() || ip_l > pypy_jit_end_addr()) { - return -1; - } - return (void*)pypy_jit_stack_depth_at_loc(ip_l); + return pypy_jit_stack_depth_at_loc(ip_l); } static long vmprof_write_header_for_jit_addr(void **result, long n, From noreply at buildbot.pypy.org Thu Mar 12 11:38:16 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 12 Mar 2015 11:38:16 +0100 (CET) Subject: [pypy-commit] pypy vmprof: fix Message-ID: <20150312103816.DBEF51C0F05@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof Changeset: r76343:9b69e7c9c0de Date: 2015-03-12 11:37 +0100 http://bitbucket.org/pypy/pypy/changeset/9b69e7c9c0de/ Log: fix diff --git a/pypy/module/_vmprof/src/fake_pypy_api.c b/pypy/module/_vmprof/src/fake_pypy_api.c --- a/pypy/module/_vmprof/src/fake_pypy_api.c +++ b/pypy/module/_vmprof/src/fake_pypy_api.c @@ -6,7 +6,7 @@ void *pypy_find_codemap_at_addr(long x) { - return NULL; + return (void *)0; } long pypy_yield_codemap_at_addr(void *x, long y, long *a) From noreply at buildbot.pypy.org Thu Mar 12 12:10:39 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 12 Mar 2015 12:10:39 +0100 (CET) Subject: [pypy-commit] pypy vmprof: remove debug print Message-ID: <20150312111039.D35AC1C0221@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof Changeset: r76344:dfce721b1d22 Date: 2015-03-12 12:10 +0100 http://bitbucket.org/pypy/pypy/changeset/dfce721b1d22/ Log: remove debug print diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py --- a/rpython/jit/backend/llsupport/codemap.py +++ b/rpython/jit/backend/llsupport/codemap.py @@ -103,7 +103,7 @@ pos = rawstart + frame_positions[i] length = rawstop - pos if length > 0: - print "ADD:", pos, length, frame_assignments[i] + #print "ADD:", pos, length, frame_assignments[i] pypy_jit_depthmap_add(pos, length, frame_assignments[i]) rawstop = pos From noreply at buildbot.pypy.org Thu Mar 12 13:15:58 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 12 Mar 2015 13:15:58 +0100 (CET) Subject: [pypy-commit] pypy vmprof: Include skiplist.c with codemap.c into a single separate_module_sources; otherwise codemap.c is copied in a place that no longer has any skiplist.c Message-ID: <20150312121558.E9DAE1C00EF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof Changeset: r76345:ea87004e582f Date: 2015-03-12 13:11 +0100 http://bitbucket.org/pypy/pypy/changeset/ea87004e582f/ Log: Include skiplist.c with codemap.c into a single separate_module_sources; otherwise codemap.c is copied in a place that no longer has any skiplist.c diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py --- a/rpython/jit/backend/llsupport/codemap.py +++ b/rpython/jit/backend/llsupport/codemap.py @@ -23,7 +23,10 @@ INT_LIST_PTR = rffi.CArrayPtr(lltype.Signed) +srcdir = os.path.join(os.path.dirname(__file__), 'src') + eci = ExternalCompilationInfo(post_include_bits=[""" +#include RPY_EXTERN long pypy_jit_codemap_add(uintptr_t addr, unsigned int machine_code_size, long *bytecode_info, @@ -38,8 +41,9 @@ unsigned int stackdepth); RPY_EXTERN void pypy_jit_depthmap_clear(uintptr_t addr, unsigned int size); -"""], separate_module_files=[ - os.path.join(os.path.dirname(__file__), 'src', 'codemap.c') +"""], separate_module_sources=[ + open(os.path.join(srcdir, 'skiplist.c'), 'r').read() + + open(os.path.join(srcdir, 'codemap.c'), 'r').read() ], include_dirs=[cdir]) def llexternal(name, args, res): diff --git a/rpython/jit/backend/llsupport/src/codemap.c b/rpython/jit/backend/llsupport/src/codemap.c --- a/rpython/jit/backend/llsupport/src/codemap.c +++ b/rpython/jit/backend/llsupport/src/codemap.c @@ -1,5 +1,8 @@ #include "src/precommondefs.h" -#include "skiplist.c" + +#ifndef HAS_SKIPLIST +# error "skiplist.c needs to be included before" +#endif volatile int pypy_codemap_currently_invalid = 0; diff --git a/rpython/jit/backend/llsupport/src/skiplist.c b/rpython/jit/backend/llsupport/src/skiplist.c --- a/rpython/jit/backend/llsupport/src/skiplist.c +++ b/rpython/jit/backend/llsupport/src/skiplist.c @@ -1,7 +1,7 @@ #include #include - +#define HAS_SKIPLIST #define SKIPLIST_HEIGHT 8 typedef struct skipnode_s { From noreply at buildbot.pypy.org Thu Mar 12 13:16:00 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 12 Mar 2015 13:16:00 +0100 (CET) Subject: [pypy-commit] pypy vmprof: fix Message-ID: <20150312121600.1CFA21C00EF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof Changeset: r76346:9a0471e2b6fb Date: 2015-03-12 13:16 +0100 http://bitbucket.org/pypy/pypy/changeset/9a0471e2b6fb/ Log: fix diff --git a/rpython/translator/c/gcc/trackgcroot.py b/rpython/translator/c/gcc/trackgcroot.py --- a/rpython/translator/c/gcc/trackgcroot.py +++ b/rpython/translator/c/gcc/trackgcroot.py @@ -1069,6 +1069,7 @@ visit_leaq = FunctionGcRootTracker._visit_lea visit_xorq = FunctionGcRootTracker.binary_insn + visit_xchgl = FunctionGcRootTracker._visit_xchg visit_xchgq = FunctionGcRootTracker._visit_xchg visit_testq = FunctionGcRootTracker._visit_test From noreply at buildbot.pypy.org Thu Mar 12 14:46:53 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 12 Mar 2015 14:46:53 +0100 (CET) Subject: [pypy-commit] stmgc c8-hashtable: Remove 'modified_old_hashtables' and use a new variant Message-ID: <20150312134653.E9FD01C00EF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-hashtable Changeset: r1718:dc85ba70bb83 Date: 2015-03-12 14:47 +0100 http://bitbucket.org/pypy/stmgc/changeset/dc85ba70bb83/ Log: Remove 'modified_old_hashtables' and use a new variant of stm_undolog_s. diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -273,8 +273,14 @@ struct stm_undo_s *end = undo + cl->written_count; for (; undo < end; undo++) { if (undo->type == TYPE_POSITION_MARKER) { - fprintf(stderr, " marker %p %lu\n", - undo->marker_object, undo->marker_odd_number); + if (undo->type2 == TYPE_MODIFIED_HASHTABLE) { + fprintf(stderr, " hashtable %p\n", + undo->modif_hashtable); + } + else { + fprintf(stderr, " marker %p %lu\n", + undo->marker_object, undo->marker_odd_number); + } continue; } fprintf(stderr, " obj %p, size %d, ofs %lu: ", undo->object, @@ -376,21 +382,40 @@ struct stm_undo_s *undo = cl->written; struct stm_undo_s *end = cl->written + cl->written_count; for (; undo < end; undo++) { - if (undo->type == TYPE_POSITION_MARKER) + object_t *obj; + + if (undo->type != TYPE_POSITION_MARKER) { + /* common case: 'undo->object' was written to + in this past commit, so we must check that + it was not read by us. */ + obj = undo->object; + } + else if (undo->type2 != TYPE_MODIFIED_HASHTABLE) continue; - if (_stm_was_read(undo->object)) { - /* first reset all modified objects from the backup - copies as soon as the first conflict is detected; - then we will proceed below to update our segment from - the old (but unmodified) version to the newer version. - */ - reset_modified_from_backup_copies(my_segnum); - timing_write_read_contention(cl->written, undo); - needs_abort = true; + else { + /* the previous stm_undo_s is about a written + 'entry' object, which belongs to the hashtable + given now. Check that we haven't read the + hashtable (via stm_hashtable_list()). */ + obj = undo->modif_hashtable; + } - dprintf(("_stm_validate() failed for obj %p\n", undo->object)); - break; - } + if (LIKELY(!_stm_was_read(obj))) + continue; + + /* conflict! */ + dprintf(("_stm_validate() failed for obj %p\n", obj)); + + /* first reset all modified objects from the backup + copies as soon as the first conflict is detected; + then we will proceed below to update our segment + from the old (but unmodified) version to the newer + version. + */ + reset_modified_from_backup_copies(my_segnum); + timing_write_read_contention(cl->written, undo); + needs_abort = true; + break; } } diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -190,9 +190,15 @@ uintptr_t marker_odd_number; /* the odd number part of the marker */ object_t *marker_object; /* the object part of the marker */ }; + struct { + intptr_t type1; /* TYPE_POSITION_MARKER (again) */ + intptr_t type2; /* TYPE_MODIFIED_HASHTABLE */ + object_t *modif_hashtable; /* modified entry is previous stm_undo_s */ + }; }; }; #define TYPE_POSITION_MARKER (-1) +#define TYPE_MODIFIED_HASHTABLE (-2) #define SLICE_OFFSET(slice) ((slice) >> 16) #define SLICE_SIZE(slice) ((int)((slice) & 0xFFFF)) #define NEW_SLICE(offset, size) (((uint64_t)(offset)) << 16 | (size)) diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -457,6 +457,7 @@ struct stm_undo_s *modified = (struct stm_undo_s *)lst->items; struct stm_undo_s *end = (struct stm_undo_s *)(lst->items + lst->count); for (; modified < end; modified++) { + /* this logic also works if type2 == TYPE_MODIFIED_HASHTABLE */ if (modified->type == TYPE_POSITION_MARKER) mark_visit_possibly_new_object(modified->marker_object, pseg); } diff --git a/c8/stm/hashtable.c b/c8/stm/hashtable.c --- a/c8/stm/hashtable.c +++ b/c8/stm/hashtable.c @@ -351,27 +351,35 @@ void stm_hashtable_write_entry(object_t *hobj, stm_hashtable_entry_t *entry, object_t *nvalue) { - if (stm_write((object_t *)entry)) { + if (_STM_WRITE_CHECK_SLOWPATH((object_t *)entry)) { + + stm_write((object_t *)entry); + uintptr_t i = list_count(STM_PSEGMENT->modified_old_objects); - if (i > 0 && list_item(STM_PSEGMENT->modified_old_objects, i - 1) + if (i > 0 && list_item(STM_PSEGMENT->modified_old_objects, i - 3) == (uintptr_t)entry) { - /* 'modified_old_hashtables' is always obtained by taking - a subset of 'modified_old_objects' which contains only - stm_hashtable_entry_t objects, and then replacing the - stm_hashtable_entry_t objects with the hobj they come - from. It's possible to have duplicates in - 'modified_old_hashtables'; here we only try a bit to - avoid them --- at least the list should never be longer - than 'modified_old_objects'. */ - i = list_count(STM_PSEGMENT->modified_old_hashtables); - if (i > 0 && list_item(STM_PSEGMENT->modified_old_hashtables, i - 2) - == (uintptr_t)hobj) { - /* already added */ - } - else { - LIST_APPEND2(STM_PSEGMENT->modified_old_hashtables, - hobj, entry); - } + /* The stm_write() above recorded a write to 'entry'. Here, + we add another stm_undo_s to modified_old_objects with + TYPE_MODIFIED_HASHTABLE. It is ignored everywhere except + in _stm_validate(). + + The goal is that this TYPE_MODIFIED_HASHTABLE ends up in + the commit log's 'cl_written' array. Later, another + transaction validating that log will check two things: + + - the regular stm_undo_s entry put by stm_write() above + will make the other transaction check that it didn't + read the same 'entry' object; + + - the TYPE_MODIFIED_HASHTABLE entry we're adding now + will make the other transaction check that it didn't + do any stm_hashtable_list() on the complete hashtable. + */ + STM_PSEGMENT->modified_old_objects = list_append3( + STM_PSEGMENT->modified_old_objects, + TYPE_POSITION_MARKER, /* type1 */ + TYPE_MODIFIED_HASHTABLE, /* type2 */ + (uintptr_t)hobj); /* modif_hashtable */ } } entry->object = nvalue; diff --git a/c8/stm/marker.c b/c8/stm/marker.c --- a/c8/stm/marker.c +++ b/c8/stm/marker.c @@ -42,7 +42,8 @@ */ while (contention != start) { --contention; - if (contention->type == TYPE_POSITION_MARKER) { + if (contention->type == TYPE_POSITION_MARKER && + contention->type2 != TYPE_MODIFIED_HASHTABLE) { out_marker->odd_number = contention->marker_odd_number; out_marker->object = contention->marker_object; return; @@ -69,6 +70,9 @@ return; /* already up-to-date */ } + /* -2 is not odd */ + assert(marker.odd_number != (uintptr_t)TYPE_MODIFIED_HASHTABLE); + STM_PSEGMENT->position_markers_last = list_count(list); STM_PSEGMENT->modified_old_objects = list_append3( list, diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c --- a/c8/stm/nursery.c +++ b/c8/stm/nursery.c @@ -422,6 +422,7 @@ struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count); for (; undo < end; undo++) { + /* this logic also works if type2 == TYPE_MODIFIED_HASHTABLE */ if (undo->type == TYPE_POSITION_MARKER) minor_trace_if_young(&undo->marker_object); } diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -192,10 +192,13 @@ STM_SEGMENT->transaction_read_version; } +#define _STM_WRITE_CHECK_SLOWPATH(obj) \ + UNLIKELY(((obj)->stm_flags & _STM_GCFLAG_WRITE_BARRIER) != 0) + __attribute__((always_inline)) static inline void stm_write(object_t *obj) { - if (UNLIKELY((obj->stm_flags & _STM_GCFLAG_WRITE_BARRIER) != 0)) + if (_STM_WRITE_CHECK_SLOWPATH(obj)) _stm_write_slowpath(obj); } @@ -204,7 +207,7 @@ static inline void stm_write_card(object_t *obj, uintptr_t index) { /* if GCFLAG_WRITE_BARRIER is set, then don't do anything more. */ - if (UNLIKELY((obj->stm_flags & _STM_GCFLAG_WRITE_BARRIER) != 0)) { + if (_STM_WRITE_CHECK_SLOWPATH(obj)) { /* GCFLAG_WRITE_BARRIER is not set. This might be because it's the first time we see a given small array; or it might From noreply at buildbot.pypy.org Thu Mar 12 15:02:05 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 12 Mar 2015 15:02:05 +0100 (CET) Subject: [pypy-commit] stmgc c8-hashtable: The buggy privatization_lock() logic was still buggy Message-ID: <20150312140205.ED5CE1C0318@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-hashtable Changeset: r1719:44c106e7ae45 Date: 2015-03-12 15:02 +0100 http://bitbucket.org/pypy/stmgc/changeset/44c106e7ae45/ Log: The buggy privatization_lock() logic was still buggy diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -148,14 +148,14 @@ memcpy(nobj_seg0, initial_data, size_rounded_up); ((struct object_s *)nobj_seg0)->stm_flags = GCFLAG_WRITE_BARRIER; + acquire_privatization_lock(STM_SEGMENT->segment_num); + long j; for (j = 1; j < NB_SEGMENTS; j++) { const char *src = nobj_seg0; char *dest = get_segment_base(j) + nobj; char *end = dest + size_rounded_up; - acquire_privatization_lock(j); - while (((uintptr_t)dest) / 4096 != ((uintptr_t)end - 1) / 4096) { uintptr_t count = 4096 - ((uintptr_t)dest) / 4096; _fill_preexisting_slice(j, dest, src, count); @@ -171,9 +171,10 @@ assert(!was_read_remote(get_segment_base(j), (object_t *)nobj)); } #endif - release_privatization_lock(j); } + release_privatization_lock(STM_SEGMENT->segment_num); + write_fence(); /* make sure 'nobj' is fully initialized from all threads here */ DEBUG_EXPECT_SEGFAULT(true); @@ -298,6 +299,8 @@ } +#define TRACE_FOR_MAJOR_COLLECTION (&mark_record_trace) + static void mark_and_trace( object_t *obj, char *segment_base, /* to trace obj in */ diff --git a/c8/stm/pages.h b/c8/stm/pages.h --- a/c8/stm/pages.h +++ b/c8/stm/pages.h @@ -62,7 +62,11 @@ static inline bool get_page_status_in(long segnum, uintptr_t pagenum) { - /* reading page status requires "read"-lock: */ + /* reading page status requires "read"-lock, which is defined as + "any segment has the privatization_lock". This is enough to + prevent the "write"-lock from being acquired by somebody else + (defined as "_all_ segments have the privatization_lock"). + */ assert(STM_PSEGMENT->privatization_lock); OPT_ASSERT(segnum < 8 * sizeof(struct page_shared_s)); From noreply at buildbot.pypy.org Thu Mar 12 15:37:11 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 12 Mar 2015 15:37:11 +0100 (CET) Subject: [pypy-commit] stmgc c8-hashtable: fix test Message-ID: <20150312143711.690071C00EF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-hashtable Changeset: r1720:471ce04ba011 Date: 2015-03-12 15:37 +0100 http://bitbucket.org/pypy/stmgc/changeset/471ce04ba011/ Log: fix test diff --git a/c8/test/test_hashtable.py b/c8/test/test_hashtable.py --- a/c8/test/test_hashtable.py +++ b/c8/test/test_hashtable.py @@ -145,9 +145,15 @@ self.switch(1) self.start_transaction() tl1 = self.tls[self.current_thread] - py.test.raises(Conflict, "htset(h, 1234, lp2, tl1)") + htset(h, 1234, lp2, tl1) # self.switch(0) + self.commit_transaction() + # + py.test.raises(Conflict, self.switch, 1) + # + self.switch(0) + self.start_transaction() self.pop_root() stm_major_collect() # to get rid of the hashtable object self.commit_transaction() From noreply at buildbot.pypy.org Thu Mar 12 15:42:44 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Thu, 12 Mar 2015 15:42:44 +0100 (CET) Subject: [pypy-commit] stmgc non-zero-nursery: don't zero nursery in minor collections Message-ID: <20150312144244.72B451C0221@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: non-zero-nursery Changeset: r1721:adf3b0df8fbc Date: 2015-03-11 11:48 +0100 http://bitbucket.org/pypy/stmgc/changeset/adf3b0df8fbc/ Log: don't zero nursery in minor collections diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -862,6 +862,31 @@ } +static void touch_all_pages_of_obj(object_t *obj, size_t obj_size) +{ + int my_segnum = STM_SEGMENT->segment_num; + uintptr_t end_page, first_page = ((uintptr_t)obj) / 4096UL; + + /* get the last page containing data from the object */ + if (LIKELY(is_small_uniform(obj))) { + end_page = first_page; + } else { + end_page = (((uintptr_t)obj) + obj_size - 1) / 4096UL; + } + + acquire_privatization_lock(STM_SEGMENT->segment_num); + uintptr_t page; + for (page = first_page; page <= end_page; page++) { + if (get_page_status_in(my_segnum, page) == PAGE_NO_ACCESS) { + release_privatization_lock(STM_SEGMENT->segment_num); + volatile char *dummy = REAL_ADDRESS(STM_SEGMENT->segment_base, page * 4096UL); + *dummy; /* force segfault */ + acquire_privatization_lock(STM_SEGMENT->segment_num); + } + } + release_privatization_lock(STM_SEGMENT->segment_num); +} + __attribute__((always_inline)) static void write_slowpath_common(object_t *obj, bool mark_card) { @@ -888,29 +913,10 @@ the full obj in this segment (XXX) */ char *realobj; size_t obj_size; - int my_segnum = STM_SEGMENT->segment_num; - uintptr_t end_page, first_page = ((uintptr_t)obj) / 4096UL; - realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); obj_size = stmcb_size_rounded_up((struct object_s *)realobj); - /* get the last page containing data from the object */ - if (LIKELY(is_small_uniform(obj))) { - end_page = first_page; - } else { - end_page = (((uintptr_t)obj) + obj_size - 1) / 4096UL; - } - acquire_privatization_lock(STM_SEGMENT->segment_num); - uintptr_t page; - for (page = first_page; page <= end_page; page++) { - if (get_page_status_in(my_segnum, page) == PAGE_NO_ACCESS) { - release_privatization_lock(STM_SEGMENT->segment_num); - volatile char *dummy = REAL_ADDRESS(STM_SEGMENT->segment_base, page * 4096UL); - *dummy; /* force segfault */ - acquire_privatization_lock(STM_SEGMENT->segment_num); - } - } - release_privatization_lock(STM_SEGMENT->segment_num); + touch_all_pages_of_obj(obj, obj_size); } if (mark_card) { diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -278,6 +278,8 @@ static stm_thread_local_t *abort_with_mutex_no_longjmp(void); static void abort_data_structures_from_segment_num(int segment_num); +static void touch_all_pages_of_obj(object_t *obj, size_t obj_size); + static void synchronize_object_enqueue(object_t *obj); static void synchronize_objects_flush(void); diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c --- a/c8/stm/nursery.c +++ b/c8/stm/nursery.c @@ -451,11 +451,8 @@ #undef STM_PSEGMENT #undef STM_SEGMENT dprintf(("throw_away_nursery\n")); - /* reset the nursery by zeroing it */ + size_t nursery_used; - char *realnursery; - - realnursery = REAL_ADDRESS(pseg->pub.segment_base, _stm_nursery_start); nursery_used = pseg->pub.nursery_current - (stm_char *)_stm_nursery_start; if (nursery_used > NB_NURSERY_PAGES * 4096) { /* possible in rare cases when the program artificially advances @@ -463,11 +460,18 @@ nursery_used = NB_NURSERY_PAGES * 4096; } OPT_ASSERT((nursery_used & 7) == 0); + + +#if _STM_NURSERY_ZEROED + /* reset the nursery by zeroing it */ + char *realnursery; + realnursery = REAL_ADDRESS(pseg->pub.segment_base, _stm_nursery_start); memset(realnursery, 0, nursery_used); /* assert that the rest of the nursery still contains only zeroes */ assert_memset_zero(realnursery + nursery_used, (NURSERY_END - _stm_nursery_start) - nursery_used); +#endif pseg->pub.nursery_current = (stm_char *)_stm_nursery_start; @@ -601,6 +605,9 @@ stm_char *end = p + size_rounded_up; if ((uintptr_t)end <= NURSERY_END) { STM_SEGMENT->nursery_current = end; +#if !_STM_NURSERY_ZEROED + ((object_t *)p)->stm_flags = 0; +#endif return (object_t *)p; } @@ -626,7 +633,14 @@ tree_insert(STM_PSEGMENT->young_outside_nursery, (uintptr_t)o, 0); +#if _STM_NURSERY_ZEROED memset(REAL_ADDRESS(STM_SEGMENT->segment_base, o), 0, size_rounded_up); +#else + o->stm_flags = 0; + /* make all pages of 'o' accessible as synchronize_obj_flush() in minor + collections assumes all young objs are fully accessible. */ + touch_all_pages_of_obj(o, size_rounded_up); +#endif return o; } @@ -646,6 +660,7 @@ } #endif +__attribute__((unused)) static void assert_memset_zero(void *s, size_t n) { #ifndef NDEBUG @@ -662,9 +677,12 @@ static void check_nursery_at_transaction_start(void) { assert((uintptr_t)STM_SEGMENT->nursery_current == _stm_nursery_start); + +#if _STM_NURSERY_ZEROED assert_memset_zero(REAL_ADDRESS(STM_SEGMENT->segment_base, STM_SEGMENT->nursery_current), NURSERY_END - _stm_nursery_start); +#endif } diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -73,6 +73,10 @@ void *creating_pthread[2]; } stm_thread_local_t; +#ifndef _STM_NURSERY_ZEROED +#define _STM_NURSERY_ZEROED 0 +#endif + #define _STM_GCFLAG_WRITE_BARRIER 0x01 #define _STM_FAST_ALLOC (66*1024) #define _STM_NSE_SIGNAL_ABORT 1 @@ -254,6 +258,9 @@ if (UNLIKELY((uintptr_t)end > STM_SEGMENT->nursery_end)) return _stm_allocate_slowpath(size_rounded_up); +#if !_STM_NURSERY_ZEROED + ((object_t *)p)->stm_flags = 0; +#endif return (object_t *)p; } diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -448,6 +448,7 @@ ('STM_LARGEMALLOC_TEST', '1'), ('STM_NO_COND_WAIT', '1'), ('STM_DEBUGPRINT', '1'), + ('_STM_NURSERY_ZEROED', '1'), ('GC_N_SMALL_REQUESTS', str(GC_N_SMALL_REQUESTS)), #check ], undef_macros=['NDEBUG'], From noreply at buildbot.pypy.org Thu Mar 12 15:42:45 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Thu, 12 Mar 2015 15:42:45 +0100 (CET) Subject: [pypy-commit] stmgc non-zero-nursery: make demos initialize references with NULL after allocation Message-ID: <20150312144245.8E8AB1C0221@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: non-zero-nursery Changeset: r1722:a514aa39cef8 Date: 2015-03-11 11:49 +0100 http://bitbucket.org/pypy/stmgc/changeset/a514aa39cef8/ Log: make demos initialize references with NULL after allocation diff --git a/c8/demo/demo_random.c b/c8/demo/demo_random.c --- a/c8/demo/demo_random.c +++ b/c8/demo/demo_random.c @@ -231,10 +231,14 @@ sizeof(struct node_s) + 4096*70}; size_t size = sizes[get_rand(4)]; p = stm_allocate(size); - ((nodeptr_t)p)->sig = SIGNATURE; - ((nodeptr_t)p)->my_size = size; - ((nodeptr_t)p)->my_id = 0; - ((nodeptr_t)p)->my_hash = 0; + nodeptr_t n = (nodeptr_t)p; + n->sig = SIGNATURE; + n->my_size = size; + n->my_id = 0; + n->my_hash = 0; + nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n + n->my_size - sizeof(void*)); + n->next = NULL; + *last_next = NULL; pop_roots(); /* reload_roots not necessary, all are old after start_transaction */ break; diff --git a/c8/demo/demo_random2.c b/c8/demo/demo_random2.c --- a/c8/demo/demo_random2.c +++ b/c8/demo/demo_random2.c @@ -240,10 +240,14 @@ sizeof(struct node_s) + (get_rand(100000) & ~15)}; size_t size = sizes[get_rand(sizeof(sizes) / sizeof(size_t))]; p = stm_allocate(size); - ((nodeptr_t)p)->sig = SIGNATURE; - ((nodeptr_t)p)->my_size = size; - ((nodeptr_t)p)->my_id = 0; - ((nodeptr_t)p)->my_hash = 0; + nodeptr_t n = (nodeptr_t)p; + n->sig = SIGNATURE; + n->my_size = size; + n->my_id = 0; + n->my_hash = 0; + nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n + n->my_size - sizeof(void*)); + n->next = NULL; + *last_next = NULL; pop_roots(pushed); break; case 4: // read and validate 'p' From noreply at buildbot.pypy.org Thu Mar 12 15:42:46 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Thu, 12 Mar 2015 15:42:46 +0100 (CET) Subject: [pypy-commit] stmgc non-zero-nursery: build duhton-c8 with zeroed nursery Message-ID: <20150312144246.91F801C0221@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: non-zero-nursery Changeset: r1723:b548b42c978e Date: 2015-03-11 13:19 +0100 http://bitbucket.org/pypy/stmgc/changeset/b548b42c978e/ Log: build duhton-c8 with zeroed nursery diff --git a/duhton-c8/Makefile b/duhton-c8/Makefile --- a/duhton-c8/Makefile +++ b/duhton-c8/Makefile @@ -3,7 +3,7 @@ C8HEADERS = ../c8/stmgc.h ../c8/stm/*.h -COMMON = -pthread -lrt -g -Wall +COMMON = -pthread -lrt -g -Wall -D_STM_NURSERY_ZEROED=1 all: duhton_debug duhton duhton_release From noreply at buildbot.pypy.org Thu Mar 12 15:42:47 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Thu, 12 Mar 2015 15:42:47 +0100 (CET) Subject: [pypy-commit] stmgc non-zero-nursery: Close branch ready to be merged Message-ID: <20150312144247.8EBB01C0221@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: non-zero-nursery Changeset: r1724:72eb66355db2 Date: 2015-03-12 15:43 +0100 http://bitbucket.org/pypy/stmgc/changeset/72eb66355db2/ Log: Close branch ready to be merged From noreply at buildbot.pypy.org Thu Mar 12 15:42:48 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Thu, 12 Mar 2015 15:42:48 +0100 (CET) Subject: [pypy-commit] stmgc default: Merge non-zero-nursery Message-ID: <20150312144248.96E8B1C0221@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1725:787577b281d6 Date: 2015-03-12 15:43 +0100 http://bitbucket.org/pypy/stmgc/changeset/787577b281d6/ Log: Merge non-zero-nursery diff --git a/c8/demo/demo_random.c b/c8/demo/demo_random.c --- a/c8/demo/demo_random.c +++ b/c8/demo/demo_random.c @@ -231,10 +231,14 @@ sizeof(struct node_s) + 4096*70}; size_t size = sizes[get_rand(4)]; p = stm_allocate(size); - ((nodeptr_t)p)->sig = SIGNATURE; - ((nodeptr_t)p)->my_size = size; - ((nodeptr_t)p)->my_id = 0; - ((nodeptr_t)p)->my_hash = 0; + nodeptr_t n = (nodeptr_t)p; + n->sig = SIGNATURE; + n->my_size = size; + n->my_id = 0; + n->my_hash = 0; + nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n + n->my_size - sizeof(void*)); + n->next = NULL; + *last_next = NULL; pop_roots(); /* reload_roots not necessary, all are old after start_transaction */ break; diff --git a/c8/demo/demo_random2.c b/c8/demo/demo_random2.c --- a/c8/demo/demo_random2.c +++ b/c8/demo/demo_random2.c @@ -240,10 +240,14 @@ sizeof(struct node_s) + (get_rand(100000) & ~15)}; size_t size = sizes[get_rand(sizeof(sizes) / sizeof(size_t))]; p = stm_allocate(size); - ((nodeptr_t)p)->sig = SIGNATURE; - ((nodeptr_t)p)->my_size = size; - ((nodeptr_t)p)->my_id = 0; - ((nodeptr_t)p)->my_hash = 0; + nodeptr_t n = (nodeptr_t)p; + n->sig = SIGNATURE; + n->my_size = size; + n->my_id = 0; + n->my_hash = 0; + nodeptr_t TLPREFIX *last_next = (nodeptr_t TLPREFIX *)((stm_char*)n + n->my_size - sizeof(void*)); + n->next = NULL; + *last_next = NULL; pop_roots(pushed); break; case 4: // read and validate 'p' diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -862,6 +862,31 @@ } +static void touch_all_pages_of_obj(object_t *obj, size_t obj_size) +{ + int my_segnum = STM_SEGMENT->segment_num; + uintptr_t end_page, first_page = ((uintptr_t)obj) / 4096UL; + + /* get the last page containing data from the object */ + if (LIKELY(is_small_uniform(obj))) { + end_page = first_page; + } else { + end_page = (((uintptr_t)obj) + obj_size - 1) / 4096UL; + } + + acquire_privatization_lock(STM_SEGMENT->segment_num); + uintptr_t page; + for (page = first_page; page <= end_page; page++) { + if (get_page_status_in(my_segnum, page) == PAGE_NO_ACCESS) { + release_privatization_lock(STM_SEGMENT->segment_num); + volatile char *dummy = REAL_ADDRESS(STM_SEGMENT->segment_base, page * 4096UL); + *dummy; /* force segfault */ + acquire_privatization_lock(STM_SEGMENT->segment_num); + } + } + release_privatization_lock(STM_SEGMENT->segment_num); +} + __attribute__((always_inline)) static void write_slowpath_common(object_t *obj, bool mark_card) { @@ -888,29 +913,10 @@ the full obj in this segment (XXX) */ char *realobj; size_t obj_size; - int my_segnum = STM_SEGMENT->segment_num; - uintptr_t end_page, first_page = ((uintptr_t)obj) / 4096UL; - realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); obj_size = stmcb_size_rounded_up((struct object_s *)realobj); - /* get the last page containing data from the object */ - if (LIKELY(is_small_uniform(obj))) { - end_page = first_page; - } else { - end_page = (((uintptr_t)obj) + obj_size - 1) / 4096UL; - } - acquire_privatization_lock(STM_SEGMENT->segment_num); - uintptr_t page; - for (page = first_page; page <= end_page; page++) { - if (get_page_status_in(my_segnum, page) == PAGE_NO_ACCESS) { - release_privatization_lock(STM_SEGMENT->segment_num); - volatile char *dummy = REAL_ADDRESS(STM_SEGMENT->segment_base, page * 4096UL); - *dummy; /* force segfault */ - acquire_privatization_lock(STM_SEGMENT->segment_num); - } - } - release_privatization_lock(STM_SEGMENT->segment_num); + touch_all_pages_of_obj(obj, obj_size); } if (mark_card) { diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -278,6 +278,8 @@ static stm_thread_local_t *abort_with_mutex_no_longjmp(void); static void abort_data_structures_from_segment_num(int segment_num); +static void touch_all_pages_of_obj(object_t *obj, size_t obj_size); + static void synchronize_object_enqueue(object_t *obj); static void synchronize_objects_flush(void); diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c --- a/c8/stm/nursery.c +++ b/c8/stm/nursery.c @@ -451,11 +451,8 @@ #undef STM_PSEGMENT #undef STM_SEGMENT dprintf(("throw_away_nursery\n")); - /* reset the nursery by zeroing it */ + size_t nursery_used; - char *realnursery; - - realnursery = REAL_ADDRESS(pseg->pub.segment_base, _stm_nursery_start); nursery_used = pseg->pub.nursery_current - (stm_char *)_stm_nursery_start; if (nursery_used > NB_NURSERY_PAGES * 4096) { /* possible in rare cases when the program artificially advances @@ -463,11 +460,18 @@ nursery_used = NB_NURSERY_PAGES * 4096; } OPT_ASSERT((nursery_used & 7) == 0); + + +#if _STM_NURSERY_ZEROED + /* reset the nursery by zeroing it */ + char *realnursery; + realnursery = REAL_ADDRESS(pseg->pub.segment_base, _stm_nursery_start); memset(realnursery, 0, nursery_used); /* assert that the rest of the nursery still contains only zeroes */ assert_memset_zero(realnursery + nursery_used, (NURSERY_END - _stm_nursery_start) - nursery_used); +#endif pseg->pub.nursery_current = (stm_char *)_stm_nursery_start; @@ -601,6 +605,9 @@ stm_char *end = p + size_rounded_up; if ((uintptr_t)end <= NURSERY_END) { STM_SEGMENT->nursery_current = end; +#if !_STM_NURSERY_ZEROED + ((object_t *)p)->stm_flags = 0; +#endif return (object_t *)p; } @@ -626,7 +633,14 @@ tree_insert(STM_PSEGMENT->young_outside_nursery, (uintptr_t)o, 0); +#if _STM_NURSERY_ZEROED memset(REAL_ADDRESS(STM_SEGMENT->segment_base, o), 0, size_rounded_up); +#else + o->stm_flags = 0; + /* make all pages of 'o' accessible as synchronize_obj_flush() in minor + collections assumes all young objs are fully accessible. */ + touch_all_pages_of_obj(o, size_rounded_up); +#endif return o; } @@ -646,6 +660,7 @@ } #endif +__attribute__((unused)) static void assert_memset_zero(void *s, size_t n) { #ifndef NDEBUG @@ -662,9 +677,12 @@ static void check_nursery_at_transaction_start(void) { assert((uintptr_t)STM_SEGMENT->nursery_current == _stm_nursery_start); + +#if _STM_NURSERY_ZEROED assert_memset_zero(REAL_ADDRESS(STM_SEGMENT->segment_base, STM_SEGMENT->nursery_current), NURSERY_END - _stm_nursery_start); +#endif } diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -73,6 +73,10 @@ void *creating_pthread[2]; } stm_thread_local_t; +#ifndef _STM_NURSERY_ZEROED +#define _STM_NURSERY_ZEROED 0 +#endif + #define _STM_GCFLAG_WRITE_BARRIER 0x01 #define _STM_FAST_ALLOC (66*1024) #define _STM_NSE_SIGNAL_ABORT 1 @@ -254,6 +258,9 @@ if (UNLIKELY((uintptr_t)end > STM_SEGMENT->nursery_end)) return _stm_allocate_slowpath(size_rounded_up); +#if !_STM_NURSERY_ZEROED + ((object_t *)p)->stm_flags = 0; +#endif return (object_t *)p; } diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -448,6 +448,7 @@ ('STM_LARGEMALLOC_TEST', '1'), ('STM_NO_COND_WAIT', '1'), ('STM_DEBUGPRINT', '1'), + ('_STM_NURSERY_ZEROED', '1'), ('GC_N_SMALL_REQUESTS', str(GC_N_SMALL_REQUESTS)), #check ], undef_macros=['NDEBUG'], diff --git a/duhton-c8/Makefile b/duhton-c8/Makefile --- a/duhton-c8/Makefile +++ b/duhton-c8/Makefile @@ -3,7 +3,7 @@ C8HEADERS = ../c8/stmgc.h ../c8/stm/*.h -COMMON = -pthread -lrt -g -Wall +COMMON = -pthread -lrt -g -Wall -D_STM_NURSERY_ZEROED=1 all: duhton_debug duhton duhton_release From noreply at buildbot.pypy.org Thu Mar 12 15:43:39 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Thu, 12 Mar 2015 15:43:39 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c8: import stmgc w/o nursery zeroing Message-ID: <20150312144339.218FB1C080A@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: stmgc-c8 Changeset: r76347:06bd5b7e134d Date: 2015-03-12 09:30 +0100 http://bitbucket.org/pypy/pypy/changeset/06bd5b7e134d/ Log: import stmgc w/o nursery zeroing diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision --- a/rpython/translator/stm/src_stm/revision +++ b/rpython/translator/stm/src_stm/revision @@ -1,1 +1,1 @@ -a4e4d3ad014a +b548b42c978e+ diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c --- a/rpython/translator/stm/src_stm/stm/core.c +++ b/rpython/translator/stm/src_stm/stm/core.c @@ -862,6 +862,31 @@ } +static void touch_all_pages_of_obj(object_t *obj, size_t obj_size) +{ + int my_segnum = STM_SEGMENT->segment_num; + uintptr_t end_page, first_page = ((uintptr_t)obj) / 4096UL; + + /* get the last page containing data from the object */ + if (LIKELY(is_small_uniform(obj))) { + end_page = first_page; + } else { + end_page = (((uintptr_t)obj) + obj_size - 1) / 4096UL; + } + + acquire_privatization_lock(STM_SEGMENT->segment_num); + uintptr_t page; + for (page = first_page; page <= end_page; page++) { + if (get_page_status_in(my_segnum, page) == PAGE_NO_ACCESS) { + release_privatization_lock(STM_SEGMENT->segment_num); + volatile char *dummy = REAL_ADDRESS(STM_SEGMENT->segment_base, page * 4096UL); + *dummy; /* force segfault */ + acquire_privatization_lock(STM_SEGMENT->segment_num); + } + } + release_privatization_lock(STM_SEGMENT->segment_num); +} + __attribute__((always_inline)) static void write_slowpath_common(object_t *obj, bool mark_card) { @@ -888,29 +913,10 @@ the full obj in this segment (XXX) */ char *realobj; size_t obj_size; - int my_segnum = STM_SEGMENT->segment_num; - uintptr_t end_page, first_page = ((uintptr_t)obj) / 4096UL; - realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj); obj_size = stmcb_size_rounded_up((struct object_s *)realobj); - /* get the last page containing data from the object */ - if (LIKELY(is_small_uniform(obj))) { - end_page = first_page; - } else { - end_page = (((uintptr_t)obj) + obj_size - 1) / 4096UL; - } - acquire_privatization_lock(STM_SEGMENT->segment_num); - uintptr_t page; - for (page = first_page; page <= end_page; page++) { - if (get_page_status_in(my_segnum, page) == PAGE_NO_ACCESS) { - release_privatization_lock(STM_SEGMENT->segment_num); - volatile char *dummy = REAL_ADDRESS(STM_SEGMENT->segment_base, page * 4096UL); - *dummy; /* force segfault */ - acquire_privatization_lock(STM_SEGMENT->segment_num); - } - } - release_privatization_lock(STM_SEGMENT->segment_num); + touch_all_pages_of_obj(obj, obj_size); } if (mark_card) { diff --git a/rpython/translator/stm/src_stm/stm/core.h b/rpython/translator/stm/src_stm/stm/core.h --- a/rpython/translator/stm/src_stm/stm/core.h +++ b/rpython/translator/stm/src_stm/stm/core.h @@ -278,6 +278,8 @@ static stm_thread_local_t *abort_with_mutex_no_longjmp(void); static void abort_data_structures_from_segment_num(int segment_num); +static void touch_all_pages_of_obj(object_t *obj, size_t obj_size); + static void synchronize_object_enqueue(object_t *obj); static void synchronize_objects_flush(void); diff --git a/rpython/translator/stm/src_stm/stm/nursery.c b/rpython/translator/stm/src_stm/stm/nursery.c --- a/rpython/translator/stm/src_stm/stm/nursery.c +++ b/rpython/translator/stm/src_stm/stm/nursery.c @@ -451,11 +451,8 @@ #undef STM_PSEGMENT #undef STM_SEGMENT dprintf(("throw_away_nursery\n")); - /* reset the nursery by zeroing it */ + size_t nursery_used; - char *realnursery; - - realnursery = REAL_ADDRESS(pseg->pub.segment_base, _stm_nursery_start); nursery_used = pseg->pub.nursery_current - (stm_char *)_stm_nursery_start; if (nursery_used > NB_NURSERY_PAGES * 4096) { /* possible in rare cases when the program artificially advances @@ -463,11 +460,18 @@ nursery_used = NB_NURSERY_PAGES * 4096; } OPT_ASSERT((nursery_used & 7) == 0); + + +#if _STM_NURSERY_ZEROED + /* reset the nursery by zeroing it */ + char *realnursery; + realnursery = REAL_ADDRESS(pseg->pub.segment_base, _stm_nursery_start); memset(realnursery, 0, nursery_used); /* assert that the rest of the nursery still contains only zeroes */ assert_memset_zero(realnursery + nursery_used, (NURSERY_END - _stm_nursery_start) - nursery_used); +#endif pseg->pub.nursery_current = (stm_char *)_stm_nursery_start; @@ -601,6 +605,9 @@ stm_char *end = p + size_rounded_up; if ((uintptr_t)end <= NURSERY_END) { STM_SEGMENT->nursery_current = end; +#if !_STM_NURSERY_ZEROED + ((object_t *)p)->stm_flags = 0; +#endif return (object_t *)p; } @@ -626,7 +633,14 @@ tree_insert(STM_PSEGMENT->young_outside_nursery, (uintptr_t)o, 0); +#if _STM_NURSERY_ZEROED memset(REAL_ADDRESS(STM_SEGMENT->segment_base, o), 0, size_rounded_up); +#else + o->stm_flags = 0; + /* make all pages of 'o' accessible as synchronize_obj_flush() in minor + collections assumes all young objs are fully accessible. */ + touch_all_pages_of_obj(o, size_rounded_up); +#endif return o; } @@ -646,6 +660,7 @@ } #endif +__attribute__((unused)) static void assert_memset_zero(void *s, size_t n) { #ifndef NDEBUG @@ -662,9 +677,12 @@ static void check_nursery_at_transaction_start(void) { assert((uintptr_t)STM_SEGMENT->nursery_current == _stm_nursery_start); + +#if _STM_NURSERY_ZEROED assert_memset_zero(REAL_ADDRESS(STM_SEGMENT->segment_base, STM_SEGMENT->nursery_current), NURSERY_END - _stm_nursery_start); +#endif } diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h --- a/rpython/translator/stm/src_stm/stmgc.h +++ b/rpython/translator/stm/src_stm/stmgc.h @@ -73,6 +73,10 @@ void *creating_pthread[2]; } stm_thread_local_t; +#ifndef _STM_NURSERY_ZEROED +#define _STM_NURSERY_ZEROED 0 +#endif + #define _STM_GCFLAG_WRITE_BARRIER 0x01 #define _STM_FAST_ALLOC (66*1024) #define _STM_NSE_SIGNAL_ABORT 1 @@ -254,6 +258,9 @@ if (UNLIKELY((uintptr_t)end > STM_SEGMENT->nursery_end)) return _stm_allocate_slowpath(size_rounded_up); +#if !_STM_NURSERY_ZEROED + ((object_t *)p)->stm_flags = 0; +#endif return (object_t *)p; } From noreply at buildbot.pypy.org Thu Mar 12 15:43:40 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Thu, 12 Mar 2015 15:43:40 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c8: use stm_memclearinit instead of raw_memclear to initialize gc pointers after Message-ID: <20150312144340.41EEB1C080A@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: stmgc-c8 Changeset: r76348:04d925289b32 Date: 2015-03-12 09:32 +0100 http://bitbucket.org/pypy/pypy/changeset/04d925289b32/ Log: use stm_memclearinit instead of raw_memclear to initialize gc pointers after allocation diff --git a/rpython/memory/gc/stmgc.py b/rpython/memory/gc/stmgc.py --- a/rpython/memory/gc/stmgc.py +++ b/rpython/memory/gc/stmgc.py @@ -25,7 +25,7 @@ needs_write_barrier = "stm" prebuilt_gc_objects_are_static_roots = False ignore_immutable_static_roots = False - malloc_zero_filled = True + malloc_zero_filled = False object_minimal_size = 16 #gcflag_extra = GCFLAG_EXTRA @@ -70,7 +70,7 @@ hdr._obj.typeid16 = typeid16 hdr._obj.prebuilt_hash = prebuilt_hash - def malloc_fixedsize_clear(self, typeid16, size, + def malloc_fixedsize(self, typeid16, size, needs_finalizer=False, is_finalizer_light=False, contains_weakptr=False): @@ -84,7 +84,7 @@ return llop.stm_allocate_finalizer(llmemory.GCREF, size, typeid16) return llop.stm_allocate_tid(llmemory.GCREF, size, typeid16) - def malloc_varsize_clear(self, typeid16, length, size, itemsize, + def malloc_varsize(self, typeid16, length, size, itemsize, offset_to_length): # XXX be careful here about overflows totalsize = size + itemsize * length diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -315,7 +315,7 @@ GCClass.malloc_varsize.im_func, [s_gc, s_typeid16] + [annmodel.SomeInteger(nonneg=True) for i in range(4)], s_gcref) - + self.collect_ptr = getfn(GCClass.collect.im_func, [s_gc, annmodel.SomeInteger()], annmodel.s_None) self.can_move_ptr = getfn(GCClass.can_move.im_func, @@ -934,8 +934,13 @@ v_a = op.result v_clear_size = hop.genop('int_sub', [v_size, c_after_header], resulttype=lltype.Signed) - self.emit_raw_memclear(hop.llops, v_clear_size, None, - c_after_header, v_a) + if not self.translator.config.translation.stm: + self.emit_raw_memclear(hop.llops, v_clear_size, None, + c_after_header, v_a) + else: + self.emit_stm_memclearinit(hop.llops, v_clear_size, None, + c_after_header, v_a) + def gct_do_malloc_varsize(self, hop): # used by the JIT (see rpython.jit.backend.llsupport.gc) @@ -967,11 +972,19 @@ llmemory.sizeof(self.HDR)) v_clear_size = hop.genop('int_sub', [c_length_ofs, c_after_header], resulttype=lltype.Signed) - self.emit_raw_memclear(hop.llops, v_clear_size, None, - c_after_header, v_a) - # Clear the variable-size part - self.emit_raw_memclear(hop.llops, v_num_elem, c_itemsize, - c_basesize, v_a) + + if not self.translator.config.translation.stm: + self.emit_raw_memclear(hop.llops, v_clear_size, None, + c_after_header, v_a) + # Clear the variable-size part + self.emit_raw_memclear(hop.llops, v_num_elem, c_itemsize, + c_basesize, v_a) + else: + self.emit_stm_memclearinit(hop.llops, v_clear_size, None, + c_after_header, v_a) + self.emit_stm_memclearinit(hop.llops, v_num_elem, c_itemsize, + c_basesize, v_a) + def gct_get_write_barrier_failing_case(self, hop): op = hop.spaceop @@ -1399,7 +1412,7 @@ [v] + previous_steps + [c_name, c_null]) else: llops.genop('bare_setfield', [v, c_name, c_null]) - + return elif isinstance(TYPE, lltype.Array): ITEM = TYPE.OF @@ -1407,16 +1420,20 @@ v_size = llops.genop('getarraysize', [v], resulttype=lltype.Signed) c_size = rmodel.inputconst(lltype.Signed, llmemory.sizeof(ITEM)) - v_a = llops.genop('cast_ptr_to_adr', [v], - resulttype=llmemory.Address) c_fixedofs = rmodel.inputconst(lltype.Signed, llmemory.itemoffsetof(TYPE)) - self.emit_raw_memclear(llops, v_size, c_size, c_fixedofs, v_a) + if not self.translator.config.translation.stm: + v_a = llops.genop('cast_ptr_to_adr', [v], + resulttype=llmemory.Address) + self.emit_raw_memclear(llops, v_size, c_size, c_fixedofs, v_a) + else: + self.emit_stm_memclearinit(llops, v_size, c_size, c_fixedofs, v) return else: raise TypeError(TYPE) def emit_raw_memclear(self, llops, v_size, c_size, c_fixedofs, v_a): + assert not self.translator.config.translation.stm if c_size is None: v_totalsize = v_size else: @@ -1426,6 +1443,16 @@ resulttype=llmemory.Address) llops.genop('raw_memclear', [v_adr, v_totalsize]) + def emit_stm_memclearinit(self, llops, v_size, c_size, c_fixedofs, v_a): + assert self.translator.config.translation.stm + if c_size is None: + v_totalsize = v_size + else: + v_totalsize = llops.genop('int_mul', [v_size, c_size], + resulttype=lltype.Signed) + llops.genop('stm_memclearinit', [v_a, c_fixedofs, v_totalsize]) + + class TransformerLayoutBuilder(gctypelayout.TypeLayoutBuilder): diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py --- a/rpython/rtyper/llinterp.py +++ b/rpython/rtyper/llinterp.py @@ -1005,6 +1005,7 @@ op_stm_increment_atomic = _stm_not_implemented op_stm_decrement_atomic = _stm_not_implemented op_stm_collect = _stm_not_implemented + op_stm_memclearinit = _stm_not_implemented def op_stm_should_break_transaction(self, keep): return False diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -446,6 +446,9 @@ 'stm_stop_all_other_threads': LLOp(canmallocgc=True), 'stm_resume_all_other_threads': LLOp(), + # stm_memclearinit(gcref, offset, size) clears the memory in this address space + 'stm_memclearinit': LLOp(), + 'stm_hint_commit_soon': LLOp(canrun=True), 'stm_increment_atomic': LLOp(), diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py --- a/rpython/translator/stm/funcgen.py +++ b/rpython/translator/stm/funcgen.py @@ -294,6 +294,13 @@ typename = cdecl(funcgen.lltypename(op.result), '') return '%s = (%s)(uintptr_t)%s;' % (result, typename, arg) +def stm_memclearinit(funcgen, op): + gcref = funcgen.expr(op.args[0]) + offset = funcgen.expr(op.args[1]) + size = funcgen.expr(op.args[2]) + return 'pypy_stm_memclearinit((object_t*)%s, (size_t)%s, (size_t)%s);' % ( + gcref, offset, size) + def stm_hashtable_create(funcgen, op): _STM_HASHTABLE_ENTRY = op.args[0].concretetype.TO type_id = funcgen.db.gctransformer.get_type_id(_STM_HASHTABLE_ENTRY) diff --git a/rpython/translator/stm/src_stm/extracode.h b/rpython/translator/stm/src_stm/extracode.h --- a/rpython/translator/stm/src_stm/extracode.h +++ b/rpython/translator/stm/src_stm/extracode.h @@ -50,6 +50,12 @@ } +void pypy_stm_memclearinit(object_t *obj, size_t offset, size_t size) +{ + char *realobj = STM_SEGMENT->segment_base + (uintptr_t)obj; + memset(realobj + offset, 0, size); +} + /************************************************************/ /*** HACK: hard-coded logic to expand the marker into ***/ /*** a string, suitable for running in PyPy ***/ diff --git a/rpython/translator/stm/src_stm/stmgcintf.h b/rpython/translator/stm/src_stm/stmgcintf.h --- a/rpython/translator/stm/src_stm/stmgcintf.h +++ b/rpython/translator/stm/src_stm/stmgcintf.h @@ -24,6 +24,8 @@ void pypy_stm_register_thread_local(void); /* generated into stm_prebuilt.c */ void pypy_stm_unregister_thread_local(void); /* generated into stm_prebuilt.c */ +void pypy_stm_memclearinit(object_t *obj, size_t offset, size_t size); + void _pypy_stm_initialize_nursery_low_fill_mark(long v_counter); void _pypy_stm_inev_state(void); long _pypy_stm_start_transaction(void); From noreply at buildbot.pypy.org Thu Mar 12 15:43:41 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Thu, 12 Mar 2015 15:43:41 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c8: start fixing jit support for non-zero nursery Message-ID: <20150312144341.613D81C080A@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: stmgc-c8 Changeset: r76349:f7607bffb63b Date: 2015-03-12 09:33 +0100 http://bitbucket.org/pypy/pypy/changeset/f7607bffb63b/ Log: start fixing jit support for non-zero nursery diff --git a/rpython/jit/backend/llsupport/test/test_stmrewrite.py b/rpython/jit/backend/llsupport/test/test_stmrewrite.py --- a/rpython/jit/backend/llsupport/test/test_stmrewrite.py +++ b/rpython/jit/backend/llsupport/test/test_stmrewrite.py @@ -54,10 +54,11 @@ self.gc_ll_descr.write_barrier_descr.has_write_barrier_from_array = ( lambda cpu: True) self.gc_ll_descr.minimal_size_in_nursery = 16 + self.gc_ll_descr.malloc_zero_filled = False # class FakeCPU(BaseFakeCPU): def sizeof(self, STRUCT): - descr = SizeDescrWithVTable(104) + descr = SizeDescrWithVTable(104, gc_fielddescrs=[]) descr.tid = 9315 return descr @@ -73,7 +74,7 @@ for name, value in self.gc_ll_descr.__dict__.items(): if name.endswith('descr') and name[1] == '2' and len(name) == 8: namespace[name] = value # "X2Ydescr" - self.gc_ll_descr.malloc_zero_filled = True + self.gc_ll_descr.malloc_zero_filled = False RewriteTests.check_rewrite(self, frm_operations, to_operations, **namespace) @@ -226,6 +227,7 @@ setfield_gc(p2, %(tdescr.tid)d, descr=tiddescr) cond_call_gc_wb(p3, descr=wbdescr) setfield_gc(p3, p1, descr=tzdescr) + zero_ptr_field(p2, %(tdescr.gc_fielddescrs[0].offset)s) jump(p2) """) @@ -243,6 +245,7 @@ p3 = call_malloc_nursery(%(tdescr.size)d) setfield_gc(p3, %(tdescr.tid)d, descr=tiddescr) p4 = getfield_gc(p1, descr=tzdescr) + zero_ptr_field(p3, %(tdescr.gc_fielddescrs[0].offset)s) jump(p2) """) @@ -271,6 +274,7 @@ p2 = call_malloc_nursery(%(tdescr.size)d) setfield_gc(p2, %(tdescr.tid)d, descr=tiddescr) p1 = getfield_gc(p2, descr=tzdescr) + zero_ptr_field(p2, %(tdescr.gc_fielddescrs[0].offset)s) jump(p1) """) @@ -605,6 +609,7 @@ [i2, i3] p1 = call_malloc_nursery_varsize(1, 1, i3, descr=strdescr) setfield_gc(p1, i3, descr=strlendescr) + setfield_gc(p1, 0, descr=strhashdescr) cond_call_gc_wb(p1, descr=wbdescr) strsetitem(p1, i2, i3) unicodesetitem(p1, i2, i3) @@ -688,6 +693,7 @@ [p1, i1, i2, i3] p2 = call_malloc_nursery_varsize(1, 1, i3, descr=strdescr) setfield_gc(p2, i3, descr=strlendescr) + setfield_gc(p2, 0, descr=strhashdescr) cond_call_gc_wb(p2, descr=wbdescr) copystrcontent(p1, p2, i1, i2, i3) jump() @@ -892,9 +898,12 @@ %(sdescr.size + tdescr.size + sdescr.size)d) setfield_gc(p0, 1234, descr=tiddescr) p1 = int_add(p0, %(sdescr.size)d) + setfield_gc(p1, 0, descr=stmflagsdescr) setfield_gc(p1, 5678, descr=tiddescr) p2 = int_add(p1, %(tdescr.size)d) + setfield_gc(p2, 0, descr=stmflagsdescr) setfield_gc(p2, 1234, descr=tiddescr) + zero_ptr_field(p1, %(tdescr.gc_fielddescrs[0].offset)s) jump() """) @@ -1005,7 +1014,7 @@ [i0] p0 = call_malloc_nursery_varsize(1, 1, i0, descr=strdescr) setfield_gc(p0, i0, descr=strlendescr) - + setfield_gc(p0, 0, descr=strhashdescr) jump(i0) """) @@ -1123,18 +1132,22 @@ unicodedescr.basesize + 10 * unicodedescr.itemsize)d) setfield_gc(p0, %(strdescr.tid)d, descr=tiddescr) setfield_gc(p0, 14, descr=strlendescr) + setfield_gc(p0, 0, descr=strhashdescr) p1 = int_add(p0, %(strdescr.basesize + 16 * strdescr.itemsize)d) setfield_gc(p1, %(unicodedescr.tid)d, descr=tiddescr) setfield_gc(p1, 10, descr=unicodelendescr) + setfield_gc(p1, 0, descr=unicodehashdescr) p2 = call_malloc_nursery_varsize(2, 4, i2, \ descr=unicodedescr) setfield_gc(p2, i2, descr=unicodelendescr) + setfield_gc(p2, 0, descr=unicodehashdescr) p3 = call_malloc_nursery_varsize(1, 1, i2, \ descr=strdescr) setfield_gc(p3, i2, descr=strlendescr) + setfield_gc(p3, 0, descr=strhashdescr) jump() """) @@ -1142,7 +1155,7 @@ def test_label_makes_size_unknown(self): self.check_rewrite(""" [i2, p3] - p1 = new_array(5, descr=cdescr) + p1 = new_array_clear(5, descr=cdescr) label(p1, i2, p3) setarrayitem_gc(p1, i2, p3, descr=cdescr) """, """ @@ -1151,6 +1164,7 @@ %(cdescr.basesize + 5 * cdescr.itemsize)d) setfield_gc(p1, 8111, descr=tiddescr) setfield_gc(p1, 5, descr=clendescr) + zero_array(p1, 0, 5, descr=cdescr) label(p1, i2, p3) cond_call_gc_wb_array(p1, i2, descr=wbdescr) setarrayitem_gc(p1, i2, p3, descr=cdescr) @@ -1303,6 +1317,7 @@ p2 = call_malloc_nursery(%(tdescr.size)d) setfield_gc(p2, %(tdescr.tid)d, descr=tiddescr) i1 = stm_should_break_transaction() + zero_ptr_field(p2, %(tdescr.gc_fielddescrs[0].offset)s) jump(i1) """) @@ -1331,6 +1346,7 @@ [] p2 = call_malloc_nursery(%(tdescr.size)d) setfield_gc(p2, %(tdescr.tid)d, descr=tiddescr) + zero_ptr_field(p2, %(tdescr.gc_fielddescrs[0].offset)s) label() i1 = stm_should_break_transaction() $DUMMYALLOC @@ -1351,3 +1367,17 @@ guard_not_forced_2() [] finish() """) + + def test_zero_before_maymalloc(self): + self.check_rewrite(""" + [] + p2 = new(descr=tdescr) + escape() + """, """ + [] + p2 = call_malloc_nursery(%(tdescr.size)d) + setfield_gc(p2, %(tdescr.tid)d, descr=tiddescr) + zero_ptr_field(p2, %(tdescr.gc_fielddescrs[0].offset)s) + $INEV + escape() + """) diff --git a/rpython/jit/metainterp/gc.py b/rpython/jit/metainterp/gc.py --- a/rpython/jit/metainterp/gc.py +++ b/rpython/jit/metainterp/gc.py @@ -29,7 +29,7 @@ malloc_zero_filled = False class GC_stmgc(GcDescription): - malloc_zero_filled = True + malloc_zero_filled = False def get_description(config): name = config.translation.gc From noreply at buildbot.pypy.org Thu Mar 12 15:43:42 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Thu, 12 Mar 2015 15:43:42 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c8: zero stm_flags in the JIT Message-ID: <20150312144342.8B59A1C080A@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: stmgc-c8 Changeset: r76350:b31739603c93 Date: 2015-03-12 10:30 +0100 http://bitbucket.org/pypy/pypy/changeset/b31739603c93/ Log: zero stm_flags in the JIT diff --git a/TODO b/TODO --- a/TODO +++ b/TODO @@ -1,7 +1,12 @@ +------------------------------------------------------------ + +fuse the two 32bit setfield_gc for stmflags & tid in the jit + ------------------------------------------------------------ better heuristic of when to break transactions? e.g., we should rarely break if there are not threads running in parallel. +But we need to break sometimes in order to run finalizers... ------------------------------------------------------------ diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py --- a/rpython/jit/backend/llsupport/descr.py +++ b/rpython/jit/backend/llsupport/descr.py @@ -173,8 +173,10 @@ def build_stm_tid_field_descr(): from rpython.rlib import rstm - return FieldDescr('tid', rstm.tid_offset, rffi.sizeof(rstm.TID), - get_type_flag(rstm.TID), False, True) + return (FieldDescr('stmflags', rstm.stmflags_offset, rffi.sizeof(rstm.STMFLAGS), + get_type_flag(rstm.STMFLAGS), False, True), + FieldDescr('tid', rstm.tid_offset, rffi.sizeof(rstm.TID), + get_type_flag(rstm.TID), False, True)) def get_type_flag(TYPE): if isinstance(TYPE, lltype.Ptr): diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py --- a/rpython/jit/backend/llsupport/gc.py +++ b/rpython/jit/backend/llsupport/gc.py @@ -487,6 +487,8 @@ def _initialize_for_tests(self): self.layoutbuilder = None self.fielddescr_tid = AbstractDescr() + if self.stm: + self.fielddescr_stmflags = AbstractDescr() self.max_size_of_young_obj = 1000 self.GCClass = None self.gcheaderbuilder = None @@ -531,7 +533,7 @@ self.fielddescr_tid = get_field_descr(self, self.GCClass.HDR, 'tid') else: - self.fielddescr_tid = build_stm_tid_field_descr() + self.fielddescr_stmflags, self.fielddescr_tid = build_stm_tid_field_descr() frame_tid = self.layoutbuilder.get_type_id(jitframe.JITFRAME) self.translator._jit2gc['frame_tid'] = frame_tid diff --git a/rpython/jit/backend/llsupport/stmrewrite.py b/rpython/jit/backend/llsupport/stmrewrite.py --- a/rpython/jit/backend/llsupport/stmrewrite.py +++ b/rpython/jit/backend/llsupport/stmrewrite.py @@ -134,6 +134,17 @@ def must_apply_write_barrier(self, val, v=None): return val not in self.write_barrier_applied + def gen_initialize_tid(self, v_newgcobj, tid): + # Also emit a setfield that zeroes the stm_flags field. + # This is necessary since we merge some allocations together and + # stmgc assumes flags to be cleared. + assert self.gc_ll_descr.fielddescr_stmflags is not None + + op = ResOperation(rop.SETFIELD_GC, + [v_newgcobj, ConstInt(0)], None, + descr=self.gc_ll_descr.fielddescr_stmflags) + self.newop(op) + return GcRewriterAssembler.gen_initialize_tid(self, v_newgcobj, tid) @specialize.arg(1) def _do_stm_call(self, funcname, args, result): diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py --- a/rpython/jit/backend/llsupport/test/test_rewrite.py +++ b/rpython/jit/backend/llsupport/test/test_rewrite.py @@ -73,6 +73,8 @@ register_known_gctype(self.cpu, o_vtable, O) # tiddescr = self.gc_ll_descr.fielddescr_tid + if self.gc_ll_descr.stm: + stmflagsdescr = self.gc_ll_descr.fielddescr_stmflags wbdescr = self.gc_ll_descr.write_barrier_descr WORD = globals()['WORD'] # diff --git a/rpython/jit/backend/llsupport/test/test_stmrewrite.py b/rpython/jit/backend/llsupport/test/test_stmrewrite.py --- a/rpython/jit/backend/llsupport/test/test_stmrewrite.py +++ b/rpython/jit/backend/llsupport/test/test_stmrewrite.py @@ -55,6 +55,7 @@ lambda cpu: True) self.gc_ll_descr.minimal_size_in_nursery = 16 self.gc_ll_descr.malloc_zero_filled = False + assert self.gc_ll_descr.stm # class FakeCPU(BaseFakeCPU): def sizeof(self, STRUCT): @@ -224,6 +225,7 @@ cond_call_gc_wb(p3, descr=wbdescr) setfield_gc(p3, p1, descr=tzdescr) p2 = call_malloc_nursery(%(tdescr.size)d) + setfield_gc(p2, 0, descr=stmflagsdescr) setfield_gc(p2, %(tdescr.tid)d, descr=tiddescr) cond_call_gc_wb(p3, descr=wbdescr) setfield_gc(p3, p1, descr=tzdescr) @@ -243,6 +245,7 @@ p2 = getfield_gc(p1, descr=tzdescr) stm_read(p1) p3 = call_malloc_nursery(%(tdescr.size)d) + setfield_gc(p3, 0, descr=stmflagsdescr) setfield_gc(p3, %(tdescr.tid)d, descr=tiddescr) p4 = getfield_gc(p1, descr=tzdescr) zero_ptr_field(p3, %(tdescr.gc_fielddescrs[0].offset)s) @@ -258,6 +261,7 @@ """, """ [p1] p2 = call_malloc_nursery(%(tdescr.size)d) + setfield_gc(p2, 0, descr=stmflagsdescr) setfield_gc(p2, %(tdescr.tid)d, descr=tiddescr) setfield_gc(p2, p1, descr=tzdescr) jump(p2) @@ -272,6 +276,7 @@ """, """ [] p2 = call_malloc_nursery(%(tdescr.size)d) + setfield_gc(p2, 0, descr=stmflagsdescr) setfield_gc(p2, %(tdescr.tid)d, descr=tiddescr) p1 = getfield_gc(p2, descr=tzdescr) zero_ptr_field(p2, %(tdescr.gc_fielddescrs[0].offset)s) @@ -779,6 +784,7 @@ [i0, f0] i1 = getfield_raw(ConstClass(frame_info), descr=jfi_frame_depth) p1 = call_malloc_nursery_varsize_frame(i1) + setfield_gc(p1, 0, descr=stmflagsdescr) setfield_gc(p1, 0, descr=tiddescr) setfield_gc(p1, i1, descr=framelendescr) setfield_gc(p1, ConstClass(frame_info), descr=jf_frame_info) @@ -809,6 +815,7 @@ i1 = getfield_raw(ConstClass(frame_info), descr=jfi_frame_depth) p5 = call_malloc_nursery_varsize_frame(i1) + setfield_gc(p5, 0, descr=stmflagsdescr) setfield_gc(p5, 0, descr=tiddescr) setfield_gc(p5, i1, descr=framelendescr) setfield_gc(p5, ConstClass(frame_info), descr=jf_frame_info) @@ -881,6 +888,7 @@ """, """ [p1] p0 = call_malloc_nursery(%(sdescr.size)d) + setfield_gc(p0, 0, descr=stmflagsdescr) setfield_gc(p0, 1234, descr=tiddescr) jump() """) @@ -896,6 +904,7 @@ [] p0 = call_malloc_nursery( \ %(sdescr.size + tdescr.size + sdescr.size)d) + setfield_gc(p0, 0, descr=stmflagsdescr) setfield_gc(p0, 1234, descr=tiddescr) p1 = int_add(p0, %(sdescr.size)d) setfield_gc(p1, 0, descr=stmflagsdescr) @@ -916,6 +925,7 @@ [] p0 = call_malloc_nursery( \ %(adescr.basesize + 10 * adescr.itemsize)d) + setfield_gc(p0, 0, descr=stmflagsdescr) setfield_gc(p0, 4321, descr=tiddescr) setfield_gc(p0, 10, descr=alendescr) jump() @@ -932,8 +942,10 @@ p0 = call_malloc_nursery( \ %(sdescr.size + \ adescr.basesize + 10 * adescr.itemsize)d) + setfield_gc(p0, 0, descr=stmflagsdescr) setfield_gc(p0, 1234, descr=tiddescr) p1 = int_add(p0, %(sdescr.size)d) + setfield_gc(p1, 0, descr=stmflagsdescr) setfield_gc(p1, 4321, descr=tiddescr) setfield_gc(p1, 10, descr=alendescr) jump() @@ -947,6 +959,7 @@ """, """ [] p0 = call_malloc_nursery(%(bdescr.basesize + 8)d) + setfield_gc(p0, 0, descr=stmflagsdescr) setfield_gc(p0, 8765, descr=tiddescr) setfield_gc(p0, 6, descr=blendescr) jump() @@ -963,15 +976,19 @@ """, """ [] p0 = call_malloc_nursery(%(4 * (bdescr.basesize + 8))d) + setfield_gc(p0, 0, descr=stmflagsdescr) setfield_gc(p0, 8765, descr=tiddescr) setfield_gc(p0, 5, descr=blendescr) p1 = int_add(p0, %(bdescr.basesize + 8)d) + setfield_gc(p1, 0, descr=stmflagsdescr) setfield_gc(p1, 8765, descr=tiddescr) setfield_gc(p1, 5, descr=blendescr) p2 = int_add(p1, %(bdescr.basesize + 8)d) + setfield_gc(p2, 0, descr=stmflagsdescr) setfield_gc(p2, 8765, descr=tiddescr) setfield_gc(p2, 5, descr=blendescr) p3 = int_add(p2, %(bdescr.basesize + 8)d) + setfield_gc(p3, 0, descr=stmflagsdescr) setfield_gc(p3, 8765, descr=tiddescr) setfield_gc(p3, 5, descr=blendescr) jump() @@ -986,8 +1003,10 @@ """, """ [] p0 = call_malloc_nursery(%(4*WORD)d) + setfield_gc(p0, 0, descr=stmflagsdescr) setfield_gc(p0, 9000, descr=tiddescr) p1 = int_add(p0, %(2*WORD)d) + setfield_gc(p1, 0, descr=stmflagsdescr) setfield_gc(p1, 9000, descr=tiddescr) jump() """) @@ -1068,13 +1087,16 @@ [] p0 = call_malloc_nursery( \ %(2 * (bdescr.basesize + 104))d) + setfield_gc(p0, 0, descr=stmflagsdescr) setfield_gc(p0, 8765, descr=tiddescr) setfield_gc(p0, 101, descr=blendescr) p1 = int_add(p0, %(bdescr.basesize + 104)d) + setfield_gc(p1, 0, descr=stmflagsdescr) setfield_gc(p1, 8765, descr=tiddescr) setfield_gc(p1, 102, descr=blendescr) p2 = call_malloc_nursery( \ %(bdescr.basesize + 104)d) + setfield_gc(p2, 0, descr=stmflagsdescr) setfield_gc(p2, 8765, descr=tiddescr) setfield_gc(p2, 103, descr=blendescr) """) @@ -1101,6 +1123,7 @@ """, """ [p1] p0 = call_malloc_nursery(104) # rounded up + setfield_gc(p0, 0, descr=stmflagsdescr) setfield_gc(p0, 9315, descr=tiddescr) setfield_gc(p0, ConstClass(o_vtable), descr=vtable_descr) """) @@ -1130,11 +1153,13 @@ p0 = call_malloc_nursery( \ %(strdescr.basesize + 16 * strdescr.itemsize + \ unicodedescr.basesize + 10 * unicodedescr.itemsize)d) + setfield_gc(p0, 0, descr=stmflagsdescr) setfield_gc(p0, %(strdescr.tid)d, descr=tiddescr) setfield_gc(p0, 14, descr=strlendescr) setfield_gc(p0, 0, descr=strhashdescr) p1 = int_add(p0, %(strdescr.basesize + 16 * strdescr.itemsize)d) + setfield_gc(p1, 0, descr=stmflagsdescr) setfield_gc(p1, %(unicodedescr.tid)d, descr=tiddescr) setfield_gc(p1, 10, descr=unicodelendescr) setfield_gc(p1, 0, descr=unicodehashdescr) @@ -1162,6 +1187,7 @@ [i2, p3] p1 = call_malloc_nursery( \ %(cdescr.basesize + 5 * cdescr.itemsize)d) + setfield_gc(p1, 0, descr=stmflagsdescr) setfield_gc(p1, 8111, descr=tiddescr) setfield_gc(p1, 5, descr=clendescr) zero_array(p1, 0, 5, descr=cdescr) @@ -1190,9 +1216,11 @@ [i0, f0] p0 = call_malloc_nursery( \ %(2 * (bdescr.basesize + 8))d) + setfield_gc(p0, 0, descr=stmflagsdescr) setfield_gc(p0, 8765, descr=tiddescr) setfield_gc(p0, 5, descr=blendescr) p1 = int_add(p0, %(bdescr.basesize + 8)d) + setfield_gc(p1, 0, descr=stmflagsdescr) setfield_gc(p1, 8765, descr=tiddescr) setfield_gc(p1, 5, descr=blendescr) @@ -1201,6 +1229,7 @@ p2 = call_malloc_nursery( \ %(bdescr.basesize + 8)d) + setfield_gc(p2, 0, descr=stmflagsdescr) setfield_gc(p2, 8765, descr=tiddescr) setfield_gc(p2, 5, descr=blendescr) """, calldescr2=calldescr2) @@ -1272,6 +1301,7 @@ [i0, f0] i1 = getfield_raw(ConstClass(frame_info), descr=jfi_frame_depth) {54} p1 = call_malloc_nursery_varsize_frame(i1) {54} + setfield_gc(p1, 0, descr=stmflagsdescr) setfield_gc(p1, 0, descr=tiddescr) {54} setfield_gc(p1, i1, descr=framelendescr) {54} setfield_gc(p1, ConstClass(frame_info), descr=jf_frame_info) {54} @@ -1315,6 +1345,7 @@ """, """ [] p2 = call_malloc_nursery(%(tdescr.size)d) + setfield_gc(p2, 0, descr=stmflagsdescr) setfield_gc(p2, %(tdescr.tid)d, descr=tiddescr) i1 = stm_should_break_transaction() zero_ptr_field(p2, %(tdescr.gc_fielddescrs[0].offset)s) @@ -1345,6 +1376,7 @@ """, """ [] p2 = call_malloc_nursery(%(tdescr.size)d) + setfield_gc(p2, 0, descr=stmflagsdescr) setfield_gc(p2, %(tdescr.tid)d, descr=tiddescr) zero_ptr_field(p2, %(tdescr.gc_fielddescrs[0].offset)s) label() @@ -1376,6 +1408,7 @@ """, """ [] p2 = call_malloc_nursery(%(tdescr.size)d) + setfield_gc(p2, 0, descr=stmflagsdescr) setfield_gc(p2, %(tdescr.tid)d, descr=tiddescr) zero_ptr_field(p2, %(tdescr.gc_fielddescrs[0].offset)s) $INEV diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -2738,6 +2738,10 @@ assert IS_X86_64 self.mc.MOV32(mem(self.SEGMENT_GC, eax, rstm.tid_offset), imm(arraydescr.tid)) + # also zero stm_flags: + self.mc.MOV32(mem(self.SEGMENT_GC, eax, rstm.stmflags_offset), + imm(0)) + else: self.mc.MOV(mem(self.SEGMENT_GC, eax, 0), imm(arraydescr.tid)) # while we're at it, this line is not needed if we've done the CALL diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py --- a/rpython/rlib/rstm.py +++ b/rpython/rlib/rstm.py @@ -13,7 +13,9 @@ TID = rffi.UINT +STMFLAGS = rffi.UINT tid_offset = CFlexSymbolic('offsetof(struct rpyobj_s, tid)') +stmflags_offset = CFlexSymbolic('offsetof(struct rpyobj_s, lib)') stm_nb_segments = CFlexSymbolic('STM_NB_SEGMENTS') adr_nursery_free = CFlexSymbolic('((long)&STM_SEGMENT->nursery_current)') adr_nursery_top = CFlexSymbolic('((long)&STM_SEGMENT->nursery_end)') From noreply at buildbot.pypy.org Thu Mar 12 15:43:43 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Thu, 12 Mar 2015 15:43:43 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c8: fix memset usage in the jit for %gs prefix'd address Message-ID: <20150312144343.AE2791C080A@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: stmgc-c8 Changeset: r76351:4dbf5a92f9f6 Date: 2015-03-12 11:36 +0100 http://bitbucket.org/pypy/pypy/changeset/4dbf5a92f9f6/ Log: fix memset usage in the jit for %gs prefix'd address diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -1102,7 +1102,7 @@ self.mc.LEA(result, addr_add(self.SEGMENT_NO, frm, sizereg, baseofs, scale)) - def convert_addresses_to_linear(self, reg1, reg2): + def convert_addresses_to_linear(self, reg1, reg2=None): if not self.cpu.gc_ll_descr.stm: # stm-only return if not IS_X86_64: @@ -1111,7 +1111,8 @@ assert rx86.fits_in_32bits(sb_adr) # because it is in the 2nd page self.mc.MOV_rj(X86_64_SCRATCH_REG.value, (self.SEGMENT_GC, sb_adr)) self.mc.ADD(reg1, X86_64_SCRATCH_REG) - self.mc.ADD(reg2, X86_64_SCRATCH_REG) + if reg2 is not None: + self.mc.ADD(reg2, X86_64_SCRATCH_REG) def _unaryop(asmop): def genop_unary(self, op, arglocs, resloc): diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -1461,6 +1461,8 @@ dstaddr_loc, startindex_loc, itemsize_loc, base_loc, imm(baseofs)) self.assembler.mc.LEA(dstaddr_loc, dst_addr) + # for stm: convert the address from %gs-based to linear + self.assembler.convert_addresses_to_linear(dstaddr_loc) # if constbytes >= 0: length_loc = imm(constbytes) From noreply at buildbot.pypy.org Thu Mar 12 15:43:44 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Thu, 12 Mar 2015 15:43:44 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c8: fix for missing manual jitframe clearing Message-ID: <20150312144344.BD35B1C080A@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: stmgc-c8 Changeset: r76352:64f973219f79 Date: 2015-03-12 14:37 +0100 http://bitbucket.org/pypy/pypy/changeset/64f973219f79/ Log: fix for missing manual jitframe clearing diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -328,6 +328,24 @@ self.newop(op0) self.gen_malloc_nursery_varsize_frame(length_box, frame) self.gen_initialize_tid(frame, descrs.arraydescr.tid) + # we need to explicitely zero all the gc fields, because + # of the unusal malloc pattern + extra_ops = [ + ResOperation(rop.SETFIELD_GC, [frame, self.c_zero], + None, descr=descrs.jf_extra_stack_depth), + ResOperation(rop.SETFIELD_GC, [frame, self.c_null], + None, descr=descrs.jf_savedata), + ResOperation(rop.SETFIELD_GC, [frame, self.c_null], + None, descr=descrs.jf_force_descr), + ResOperation(rop.SETFIELD_GC, [frame, self.c_null], + None, descr=descrs.jf_descr), + ResOperation(rop.SETFIELD_GC, [frame, self.c_null], + None, descr=descrs.jf_guard_exc), + ResOperation(rop.SETFIELD_GC, [frame, self.c_null], + None, descr=descrs.jf_forward), + ] + for extra_op in extra_ops: + self.newop(extra_op) self.gen_initialize_len(frame, length_box, descrs.arraydescr.lendescr) diff --git a/rpython/jit/backend/llsupport/stmrewrite.py b/rpython/jit/backend/llsupport/stmrewrite.py --- a/rpython/jit/backend/llsupport/stmrewrite.py +++ b/rpython/jit/backend/llsupport/stmrewrite.py @@ -141,7 +141,7 @@ assert self.gc_ll_descr.fielddescr_stmflags is not None op = ResOperation(rop.SETFIELD_GC, - [v_newgcobj, ConstInt(0)], None, + [v_newgcobj, self.c_zero], None, descr=self.gc_ll_descr.fielddescr_stmflags) self.newop(op) return GcRewriterAssembler.gen_initialize_tid(self, v_newgcobj, tid) diff --git a/rpython/jit/backend/llsupport/test/test_stmrewrite.py b/rpython/jit/backend/llsupport/test/test_stmrewrite.py --- a/rpython/jit/backend/llsupport/test/test_stmrewrite.py +++ b/rpython/jit/backend/llsupport/test/test_stmrewrite.py @@ -786,6 +786,14 @@ p1 = call_malloc_nursery_varsize_frame(i1) setfield_gc(p1, 0, descr=stmflagsdescr) setfield_gc(p1, 0, descr=tiddescr) + + setfield_gc(p1, 0, descr=jf_extra_stack_depth) + setfield_gc(p1, NULL, descr=jf_savedata) + setfield_gc(p1, NULL, descr=jf_force_descr) + setfield_gc(p1, NULL, descr=jf_descr) + setfield_gc(p1, NULL, descr=jf_guard_exc) + setfield_gc(p1, NULL, descr=jf_forward) + setfield_gc(p1, i1, descr=framelendescr) setfield_gc(p1, ConstClass(frame_info), descr=jf_frame_info) setarrayitem_gc(p1, 0, i0, descr=signedframedescr) @@ -817,6 +825,12 @@ p5 = call_malloc_nursery_varsize_frame(i1) setfield_gc(p5, 0, descr=stmflagsdescr) setfield_gc(p5, 0, descr=tiddescr) + setfield_gc(p5, 0, descr=jf_extra_stack_depth) + setfield_gc(p5, NULL, descr=jf_savedata) + setfield_gc(p5, NULL, descr=jf_force_descr) + setfield_gc(p5, NULL, descr=jf_descr) + setfield_gc(p5, NULL, descr=jf_guard_exc) + setfield_gc(p5, NULL, descr=jf_forward) setfield_gc(p5, i1, descr=framelendescr) setfield_gc(p5, ConstClass(frame_info), descr=jf_frame_info) setarrayitem_gc(p5, 0, i0, descr=signedframedescr) @@ -1303,6 +1317,12 @@ p1 = call_malloc_nursery_varsize_frame(i1) {54} setfield_gc(p1, 0, descr=stmflagsdescr) setfield_gc(p1, 0, descr=tiddescr) {54} + setfield_gc(p1, 0, descr=jf_extra_stack_depth) + setfield_gc(p1, NULL, descr=jf_savedata) + setfield_gc(p1, NULL, descr=jf_force_descr) + setfield_gc(p1, NULL, descr=jf_descr) + setfield_gc(p1, NULL, descr=jf_guard_exc) + setfield_gc(p1, NULL, descr=jf_forward) setfield_gc(p1, i1, descr=framelendescr) {54} setfield_gc(p1, ConstClass(frame_info), descr=jf_frame_info) {54} setarrayitem_gc(p1, 0, i0, descr=signedframedescr) {54} From noreply at buildbot.pypy.org Thu Mar 12 15:45:36 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 12 Mar 2015 15:45:36 +0100 (CET) Subject: [pypy-commit] stmgc c8-hashtable: fix test Message-ID: <20150312144536.B625A1C00EF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-hashtable Changeset: r1726:92ed176fe541 Date: 2015-03-12 15:46 +0100 http://bitbucket.org/pypy/stmgc/changeset/92ed176fe541/ Log: fix test diff --git a/c8/test/test_hashtable.py b/c8/test/test_hashtable.py --- a/c8/test/test_hashtable.py +++ b/c8/test/test_hashtable.py @@ -297,19 +297,27 @@ h = self.allocate_hashtable() self.push_root(h) self.commit_transaction() + # + self.start_transaction() h = self.pop_root() + self.push_root(h) + tl0 = self.tls[self.current_thread] + htset(h, 10, stm_allocate(32), tl0) # + self.switch(1) self.start_transaction() assert htlen(h) == 0 # - self.switch(1) - self.start_transaction() - tl0 = self.tls[self.current_thread] - htset(h, 10, stm_allocate(32), tl0) - py.test.raises(Conflict, self.commit_transaction) + self.switch(0) + self.commit_transaction() + # + py.test.raises(Conflict, self.switch, 1) # self.switch(0) + self.start_transaction() + self.pop_root() stm_major_collect() # to get rid of the hashtable object + self.commit_transaction() def test_grow_without_conflict(self): self.start_transaction() From noreply at buildbot.pypy.org Thu Mar 12 15:47:07 2015 From: noreply at buildbot.pypy.org (mjacob) Date: Thu, 12 Mar 2015 15:47:07 +0100 (CET) Subject: [pypy-commit] pypy default: Update version number in RPython documentation and copyright years for both RPython and PyPy. Message-ID: <20150312144707.DF1E01C00EF@cobra.cs.uni-duesseldorf.de> Author: Manuel Jacob Branch: Changeset: r76353:706f110bc5ad Date: 2015-03-12 15:47 +0100 http://bitbucket.org/pypy/pypy/changeset/706f110bc5ad/ Log: Update version number in RPython documentation and copyright years for both RPython and PyPy. We don't really have versions for RPython, but showing version 2.0-beta1 in the RPython documentation is confusing. diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -58,7 +58,7 @@ # General information about the project. project = u'PyPy' -copyright = u'2014, The PyPy Project' +copyright = u'2015, The PyPy Project' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/rpython/doc/conf.py b/rpython/doc/conf.py --- a/rpython/doc/conf.py +++ b/rpython/doc/conf.py @@ -59,16 +59,16 @@ # General information about the project. project = u'RPython' -copyright = u'2013, The PyPy Project' +copyright = u'2015, The PyPy Project' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '2.0' +version = '2.5' # The full version, including alpha/beta/rc tags. -release = '2.0-beta1' +release = '2.5.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. From noreply at buildbot.pypy.org Thu Mar 12 16:28:36 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 12 Mar 2015 16:28:36 +0100 (CET) Subject: [pypy-commit] stmgc c8-hashtable: major gc: remove from 'modified_old_objects' all old hashtables that die Message-ID: <20150312152836.D7A6A1C0EEE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-hashtable Changeset: r1727:591a48b66365 Date: 2015-03-12 16:29 +0100 http://bitbucket.org/pypy/stmgc/changeset/591a48b66365/ Log: major gc: remove from 'modified_old_objects' all old hashtables that die diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -460,8 +460,8 @@ struct stm_undo_s *modified = (struct stm_undo_s *)lst->items; struct stm_undo_s *end = (struct stm_undo_s *)(lst->items + lst->count); for (; modified < end; modified++) { - /* this logic also works if type2 == TYPE_MODIFIED_HASHTABLE */ - if (modified->type == TYPE_POSITION_MARKER) + if (modified->type == TYPE_POSITION_MARKER && + modified->type2 != TYPE_MODIFIED_HASHTABLE) mark_visit_possibly_new_object(modified->marker_object, pseg); } } @@ -594,6 +594,31 @@ list_set_item(lst, n, list_pop_item(lst)); } } + + /* Remove from 'modified_old_objects' all old hashtables that die */ + { + lst = pseg->modified_old_objects; + uintptr_t j, k = 0, limit = list_count(lst); + for (j = 0; j < limit; j += 3) { + uintptr_t e0 = list_item(lst, j + 0); + uintptr_t e1 = list_item(lst, j + 1); + uintptr_t e2 = list_item(lst, j + 2); + if (e0 == TYPE_POSITION_MARKER && + e1 == TYPE_MODIFIED_HASHTABLE && + !mark_visited_test((object_t *)e2)) { + /* hashtable object dies */ + } + else { + if (j != k) { + list_set_item(lst, k + 0, e0); + list_set_item(lst, k + 1, e1); + list_set_item(lst, k + 2, e2); + } + k += 3; + } + } + lst->count = k; + } } #pragma pop_macro("STM_SEGMENT") #pragma pop_macro("STM_PSEGMENT") From noreply at buildbot.pypy.org Thu Mar 12 16:32:17 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 12 Mar 2015 16:32:17 +0100 (CET) Subject: [pypy-commit] stmgc c8-hashtable: argh Message-ID: <20150312153217.C33FE1C0F05@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-hashtable Changeset: r1728:dee3d648763d Date: 2015-03-12 16:32 +0100 http://bitbucket.org/pypy/stmgc/changeset/dee3d648763d/ Log: argh diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -157,7 +157,7 @@ char *end = dest + size_rounded_up; while (((uintptr_t)dest) / 4096 != ((uintptr_t)end - 1) / 4096) { - uintptr_t count = 4096 - ((uintptr_t)dest) / 4096; + uintptr_t count = 4096 - (((uintptr_t)dest) & 4095); _fill_preexisting_slice(j, dest, src, count); src += count; dest += count; From noreply at buildbot.pypy.org Thu Mar 12 17:07:16 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 12 Mar 2015 17:07:16 +0100 (CET) Subject: [pypy-commit] stmgc c8-hashtable: Fix _stm_segfault_expected to start at 1 instead of incrementing it Message-ID: <20150312160716.6E0111C123B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-hashtable Changeset: r1729:3b9fbd70b11f Date: 2015-03-12 17:04 +0100 http://bitbucket.org/pypy/stmgc/changeset/3b9fbd70b11f/ Log: Fix _stm_segfault_expected to start at 1 instead of incrementing it in stm_register_thread_local(). The latter ends up having all the tests run with a large value in _stm_segfault_expected, which is thus never <= 0. diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -142,13 +142,12 @@ uintptr_t nobj = (uintptr_t)np; dprintf(("allocate_preexisting: %p\n", (object_t *)nobj)); - DEBUG_EXPECT_SEGFAULT(false); - char *nobj_seg0 = stm_object_pages + nobj; memcpy(nobj_seg0, initial_data, size_rounded_up); ((struct object_s *)nobj_seg0)->stm_flags = GCFLAG_WRITE_BARRIER; acquire_privatization_lock(STM_SEGMENT->segment_num); + DEBUG_EXPECT_SEGFAULT(false); long j; for (j = 1; j < NB_SEGMENTS; j++) { @@ -173,11 +172,11 @@ #endif } + DEBUG_EXPECT_SEGFAULT(true); release_privatization_lock(STM_SEGMENT->segment_num); write_fence(); /* make sure 'nobj' is fully initialized from all threads here */ - DEBUG_EXPECT_SEGFAULT(true); return (object_t *)nobj; } diff --git a/c8/stm/hashtable.c b/c8/stm/hashtable.c --- a/c8/stm/hashtable.c +++ b/c8/stm/hashtable.c @@ -150,7 +150,7 @@ uintptr_t biggercount, int remove_unread_from_seg) { - dprintf(("rehash %p to %ld, remove_unread_from_seg=%d\n", + dprintf(("rehash %p to size %ld, remove_unread_from_seg=%d\n", hashtable, biggercount, remove_unread_from_seg)); size_t size = (offsetof(stm_hashtable_table_t, items) diff --git a/c8/stm/setup.c b/c8/stm/setup.c --- a/c8/stm/setup.c +++ b/c8/stm/setup.c @@ -250,8 +250,6 @@ set_gs_register(get_segment_base(num + 1)); s_mutex_unlock(); - DEBUG_EXPECT_SEGFAULT(true); - if (num == 0) { dprintf(("STM_GC_NURSERY: %d\n", STM_GC_NURSERY)); dprintf(("NB_PAGES: %d\n", NB_PAGES)); diff --git a/c8/stm/setup.h b/c8/stm/setup.h --- a/c8/stm/setup.h +++ b/c8/stm/setup.h @@ -3,7 +3,7 @@ static pthread_t *_get_cpth(stm_thread_local_t *); #ifndef NDEBUG -static __thread long _stm_segfault_expected = 0; +static __thread long _stm_segfault_expected = 1; #define DEBUG_EXPECT_SEGFAULT(v) do {if (v) _stm_segfault_expected++; else _stm_segfault_expected--;} while (0) #else #define DEBUG_EXPECT_SEGFAULT(v) {} From noreply at buildbot.pypy.org Thu Mar 12 17:07:17 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 12 Mar 2015 17:07:17 +0100 (CET) Subject: [pypy-commit] stmgc c8-hashtable: extra assert Message-ID: <20150312160717.6FFF61C123B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-hashtable Changeset: r1730:351aff2b4699 Date: 2015-03-12 17:07 +0100 http://bitbucket.org/pypy/stmgc/changeset/351aff2b4699/ Log: extra assert diff --git a/c8/stm/setup.h b/c8/stm/setup.h --- a/c8/stm/setup.h +++ b/c8/stm/setup.h @@ -4,7 +4,7 @@ #ifndef NDEBUG static __thread long _stm_segfault_expected = 1; -#define DEBUG_EXPECT_SEGFAULT(v) do {if (v) _stm_segfault_expected++; else _stm_segfault_expected--;} while (0) +#define DEBUG_EXPECT_SEGFAULT(v) do {if (v) _stm_segfault_expected++; else _stm_segfault_expected--; assert(_stm_segfault_expected <= 1);} while (0) #else #define DEBUG_EXPECT_SEGFAULT(v) {} #endif From noreply at buildbot.pypy.org Thu Mar 12 17:40:47 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 12 Mar 2015 17:40:47 +0100 (CET) Subject: [pypy-commit] stmgc c8-hashtable: Fix. Now test_hashtable seems to pass. Message-ID: <20150312164047.516D61C00EF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: c8-hashtable Changeset: r1731:0b32167a8194 Date: 2015-03-12 17:41 +0100 http://bitbucket.org/pypy/stmgc/changeset/0b32167a8194/ Log: Fix. Now test_hashtable seems to pass. diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -258,6 +258,14 @@ return stm_object_pages + segment_num * (NB_PAGES * 4096UL); } +static inline long get_num_segment_containing_address(char *addr) +{ + uintptr_t delta = addr - stm_object_pages; + uintptr_t result = delta / (NB_PAGES * 4096UL); + assert(result < NB_SEGMENTS); + return result; +} + static inline struct stm_segment_info_s *get_segment(long segment_num) { return (struct stm_segment_info_s *)REAL_ADDRESS( diff --git a/c8/stm/hashtable.c b/c8/stm/hashtable.c --- a/c8/stm/hashtable.c +++ b/c8/stm/hashtable.c @@ -148,10 +148,10 @@ static void _stm_rehash_hashtable(stm_hashtable_t *hashtable, uintptr_t biggercount, - int remove_unread_from_seg) + char *segment_base) { - dprintf(("rehash %p to size %ld, remove_unread_from_seg=%d\n", - hashtable, biggercount, remove_unread_from_seg)); + dprintf(("rehash %p to size %ld, segment_base=%p\n", + hashtable, biggercount, segment_base)); size_t size = (offsetof(stm_hashtable_table_t, items) + biggercount * sizeof(stm_hashtable_entry_t *)); @@ -169,12 +169,11 @@ uintptr_t j, mask = table->mask; uintptr_t rc = biggertable->resize_counter; - char *segment_base = get_segment_base(remove_unread_from_seg); for (j = 0; j <= mask; j++) { stm_hashtable_entry_t *entry = table->items[j]; if (entry == NULL) continue; - if (remove_unread_from_seg != 0) { + if (segment_base != NULL) { if (((struct stm_hashtable_entry_s *) REAL_ADDRESS(segment_base, entry))->object == NULL && !_stm_was_read_by_anybody((object_t *)entry)) { @@ -184,7 +183,7 @@ } uintptr_t eindex; - if (remove_unread_from_seg == 0) + if (segment_base == NULL) eindex = entry->index; /* read from STM_SEGMENT */ else eindex = ((struct stm_hashtable_entry_s *) @@ -289,7 +288,6 @@ entry->userdata = stm_hashtable_entry_userdata; entry->index = index; entry->object = NULL; - hashtable->additions = STM_SEGMENT->segment_num; } else { /* for a non-nursery 'hashtableobj', we pretend that the @@ -320,7 +318,7 @@ entry = (stm_hashtable_entry_t *) stm_allocate_preexisting(sizeof(stm_hashtable_entry_t), (char *)&initial.header); - hashtable->additions += 0x100; + hashtable->additions++; } table->items[i] = entry; write_fence(); /* make sure 'table->items' is written here */ @@ -335,7 +333,7 @@ biggercount *= 4; else biggercount *= 2; - _stm_rehash_hashtable(hashtable, biggercount, /*remove_unread=*/0); + _stm_rehash_hashtable(hashtable, biggercount, /*segment_base=*/NULL); goto restart; } } @@ -460,16 +458,27 @@ return nresult; } -static void _stm_compact_hashtable(stm_hashtable_t *hashtable) +static void _stm_compact_hashtable(struct object_s *hobj, + stm_hashtable_t *hashtable) { stm_hashtable_table_t *table = hashtable->table; uintptr_t rc = table->resize_counter; assert(!IS_EVEN(rc)); - if ((hashtable->additions >> 8) * 4 > table->mask) { - int segment_num = (hashtable->additions & 0xFF); - if (!segment_num) segment_num = 1; - hashtable->additions = segment_num; + if (hashtable->additions * 4 > table->mask) { + hashtable->additions = 0; + + /* If 'hobj' was created in some current transaction, i.e. if it is + now an overflow object, then we have the risk that some of its + entry objects were not created with stm_allocate_preexisting(). + In that situation, a valid workaround is to read all entry + objects in the segment of the running transaction. Otherwise, + the base case is to read them all from segment zero. + */ + long segnum = get_num_segment_containing_address((char *)hobj); + if (!IS_OVERFLOW_OBJ(get_priv_segment(segnum), hobj)) + segnum = 0; + uintptr_t initial_rc = (table->mask + 1) * 4 + 1; uintptr_t num_entries_times_6 = initial_rc - rc; uintptr_t count = INITIAL_HASHTABLE_SIZE; @@ -480,7 +489,7 @@ assert(count <= table->mask + 1); dprintf(("compact with %ld items:\n", num_entries_times_6 / 6)); - _stm_rehash_hashtable(hashtable, count, /*remove_unread=*/segment_num); + _stm_rehash_hashtable(hashtable, count, get_segment_base(segnum)); } table = hashtable->table; @@ -503,10 +512,11 @@ } } -void stm_hashtable_tracefn(stm_hashtable_t *hashtable, void trace(object_t **)) +void stm_hashtable_tracefn(struct object_s *hobj, stm_hashtable_t *hashtable, + void trace(object_t **)) { if (trace == TRACE_FOR_MAJOR_COLLECTION) - _stm_compact_hashtable(hashtable); + _stm_compact_hashtable(hobj, hashtable); stm_hashtable_table_t *table; table = VOLATILE_HASHTABLE(hashtable)->table; diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -492,7 +492,8 @@ long stm_hashtable_list(object_t *, stm_hashtable_t *, stm_hashtable_entry_t **results); extern uint32_t stm_hashtable_entry_userdata; -void stm_hashtable_tracefn(stm_hashtable_t *, void (object_t **)); +void stm_hashtable_tracefn(struct object_s *, stm_hashtable_t *, + void (object_t **)); struct stm_hashtable_entry_s { struct object_s header; diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -210,7 +210,8 @@ long stm_hashtable_list(object_t *, stm_hashtable_t *, stm_hashtable_entry_t **results); uint32_t stm_hashtable_entry_userdata; -void stm_hashtable_tracefn(stm_hashtable_t *, void (object_t **)); +void stm_hashtable_tracefn(struct object_s *, stm_hashtable_t *, + void trace(object_t **)); void _set_hashtable(object_t *obj, stm_hashtable_t *h); stm_hashtable_t *_get_hashtable(object_t *obj); @@ -442,7 +443,7 @@ if (myobj->type_id == 421419) { /* hashtable */ stm_hashtable_t *h = *((stm_hashtable_t **)(myobj + 1)); - stm_hashtable_tracefn(h, visit); + stm_hashtable_tracefn(obj, h, visit); return; } if (myobj->type_id == 421418) { From noreply at buildbot.pypy.org Thu Mar 12 17:43:05 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Thu, 12 Mar 2015 17:43:05 +0100 (CET) Subject: [pypy-commit] stmgc default: we actually hit these asserts, because there is stuff running in the safe-point Message-ID: <20150312164305.96A171C0318@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1732:df5b285c6f57 Date: 2015-03-12 17:44 +0100 http://bitbucket.org/pypy/stmgc/changeset/df5b285c6f57/ Log: we actually hit these asserts, because there is stuff running in the safe-point diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -1107,16 +1107,6 @@ STM_PSEGMENT->shadowstack_at_start_of_transaction = tl->shadowstack; STM_PSEGMENT->threadlocal_at_start_of_transaction = tl->thread_local_obj; - enter_safe_point_if_requested(); - dprintf(("> start_transaction\n")); - - s_mutex_unlock(); // XXX it's probably possible to not acquire this here - - uint8_t old_rv = STM_SEGMENT->transaction_read_version; - STM_SEGMENT->transaction_read_version = old_rv + 1; - if (UNLIKELY(old_rv == 0xff)) { - reset_transaction_read_version(); - } assert(list_is_empty(STM_PSEGMENT->modified_old_objects)); assert(list_is_empty(STM_PSEGMENT->large_overflow_objects)); @@ -1135,6 +1125,19 @@ check_nursery_at_transaction_start(); + /* Warning: this safe-point may run light finalizers and register + commit/abort callbacks if a major GC is triggered here */ + enter_safe_point_if_requested(); + dprintf(("> start_transaction\n")); + + s_mutex_unlock(); // XXX it's probably possible to not acquire this here + + uint8_t old_rv = STM_SEGMENT->transaction_read_version; + STM_SEGMENT->transaction_read_version = old_rv + 1; + if (UNLIKELY(old_rv == 0xff)) { + reset_transaction_read_version(); + } + stm_validate(); } From noreply at buildbot.pypy.org Thu Mar 12 17:56:36 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 12 Mar 2015 17:56:36 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c8-hashtable: A branch to reintroduce hashtables Message-ID: <20150312165636.6FFC01C00EF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c8-hashtable Changeset: r76354:93fe72206074 Date: 2015-03-12 17:45 +0100 http://bitbucket.org/pypy/pypy/changeset/93fe72206074/ Log: A branch to reintroduce hashtables From noreply at buildbot.pypy.org Thu Mar 12 17:56:37 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 12 Mar 2015 17:56:37 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c8-hashtable: import stmgc (branch c8-hashtable) Message-ID: <20150312165637.9D2061C00EF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c8-hashtable Changeset: r76355:1f14bce8027f Date: 2015-03-12 17:46 +0100 http://bitbucket.org/pypy/pypy/changeset/1f14bce8027f/ Log: import stmgc (branch c8-hashtable) diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision --- a/rpython/translator/stm/src_stm/revision +++ b/rpython/translator/stm/src_stm/revision @@ -1,1 +1,1 @@ -a4e4d3ad014a +0b32167a8194 diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c --- a/rpython/translator/stm/src_stm/stm/core.c +++ b/rpython/translator/stm/src_stm/stm/core.c @@ -273,8 +273,14 @@ struct stm_undo_s *end = undo + cl->written_count; for (; undo < end; undo++) { if (undo->type == TYPE_POSITION_MARKER) { - fprintf(stderr, " marker %p %lu\n", - undo->marker_object, undo->marker_odd_number); + if (undo->type2 == TYPE_MODIFIED_HASHTABLE) { + fprintf(stderr, " hashtable %p\n", + undo->modif_hashtable); + } + else { + fprintf(stderr, " marker %p %lu\n", + undo->marker_object, undo->marker_odd_number); + } continue; } fprintf(stderr, " obj %p, size %d, ofs %lu: ", undo->object, @@ -376,21 +382,40 @@ struct stm_undo_s *undo = cl->written; struct stm_undo_s *end = cl->written + cl->written_count; for (; undo < end; undo++) { - if (undo->type == TYPE_POSITION_MARKER) + object_t *obj; + + if (undo->type != TYPE_POSITION_MARKER) { + /* common case: 'undo->object' was written to + in this past commit, so we must check that + it was not read by us. */ + obj = undo->object; + } + else if (undo->type2 != TYPE_MODIFIED_HASHTABLE) continue; - if (_stm_was_read(undo->object)) { - /* first reset all modified objects from the backup - copies as soon as the first conflict is detected; - then we will proceed below to update our segment from - the old (but unmodified) version to the newer version. - */ - reset_modified_from_backup_copies(my_segnum); - timing_write_read_contention(cl->written, undo); - needs_abort = true; + else { + /* the previous stm_undo_s is about a written + 'entry' object, which belongs to the hashtable + given now. Check that we haven't read the + hashtable (via stm_hashtable_list()). */ + obj = undo->modif_hashtable; + } - dprintf(("_stm_validate() failed for obj %p\n", undo->object)); - break; - } + if (LIKELY(!_stm_was_read(obj))) + continue; + + /* conflict! */ + dprintf(("_stm_validate() failed for obj %p\n", obj)); + + /* first reset all modified objects from the backup + copies as soon as the first conflict is detected; + then we will proceed below to update our segment + from the old (but unmodified) version to the newer + version. + */ + reset_modified_from_backup_copies(my_segnum); + timing_write_read_contention(cl->written, undo); + needs_abort = true; + break; } } diff --git a/rpython/translator/stm/src_stm/stm/core.h b/rpython/translator/stm/src_stm/stm/core.h --- a/rpython/translator/stm/src_stm/stm/core.h +++ b/rpython/translator/stm/src_stm/stm/core.h @@ -190,9 +190,15 @@ uintptr_t marker_odd_number; /* the odd number part of the marker */ object_t *marker_object; /* the object part of the marker */ }; + struct { + intptr_t type1; /* TYPE_POSITION_MARKER (again) */ + intptr_t type2; /* TYPE_MODIFIED_HASHTABLE */ + object_t *modif_hashtable; /* modified entry is previous stm_undo_s */ + }; }; }; #define TYPE_POSITION_MARKER (-1) +#define TYPE_MODIFIED_HASHTABLE (-2) #define SLICE_OFFSET(slice) ((slice) >> 16) #define SLICE_SIZE(slice) ((int)((slice) & 0xFFFF)) #define NEW_SLICE(offset, size) (((uint64_t)(offset)) << 16 | (size)) @@ -252,6 +258,14 @@ return stm_object_pages + segment_num * (NB_PAGES * 4096UL); } +static inline long get_num_segment_containing_address(char *addr) +{ + uintptr_t delta = addr - stm_object_pages; + uintptr_t result = delta / (NB_PAGES * 4096UL); + assert(result < NB_SEGMENTS); + return result; +} + static inline struct stm_segment_info_s *get_segment(long segment_num) { return (struct stm_segment_info_s *)REAL_ADDRESS( @@ -284,6 +298,17 @@ static void _signal_handler(int sig, siginfo_t *siginfo, void *context); static bool _stm_validate(); +static inline bool was_read_remote(char *base, object_t *obj) +{ + uint8_t other_transaction_read_version = + ((struct stm_segment_info_s *)REAL_ADDRESS(base, STM_PSEGMENT)) + ->transaction_read_version; + uint8_t rm = ((struct stm_read_marker_s *) + (base + (((uintptr_t)obj) >> 4)))->rm; + assert(rm <= other_transaction_read_version); + return rm == other_transaction_read_version; +} + static inline void _duck(void) { /* put a call to _duck() between two instructions that set 0 into a %gs-prefixed address and that may otherwise be replaced with diff --git a/rpython/translator/stm/src_stm/stm/gcpage.c b/rpython/translator/stm/src_stm/stm/gcpage.c --- a/rpython/translator/stm/src_stm/stm/gcpage.c +++ b/rpython/translator/stm/src_stm/stm/gcpage.c @@ -127,6 +127,58 @@ return o; } +static void _fill_preexisting_slice(long segnum, char *dest, + const char *src, uintptr_t size) +{ + uintptr_t np = dest - get_segment_base(segnum); + if (get_page_status_in(segnum, np / 4096) != PAGE_NO_ACCESS) + memcpy(dest, src, size); +} + +object_t *stm_allocate_preexisting(ssize_t size_rounded_up, + const char *initial_data) +{ + stm_char *np = allocate_outside_nursery_large(size_rounded_up); + uintptr_t nobj = (uintptr_t)np; + dprintf(("allocate_preexisting: %p\n", (object_t *)nobj)); + + char *nobj_seg0 = stm_object_pages + nobj; + memcpy(nobj_seg0, initial_data, size_rounded_up); + ((struct object_s *)nobj_seg0)->stm_flags = GCFLAG_WRITE_BARRIER; + + acquire_privatization_lock(STM_SEGMENT->segment_num); + DEBUG_EXPECT_SEGFAULT(false); + + long j; + for (j = 1; j < NB_SEGMENTS; j++) { + const char *src = nobj_seg0; + char *dest = get_segment_base(j) + nobj; + char *end = dest + size_rounded_up; + + while (((uintptr_t)dest) / 4096 != ((uintptr_t)end - 1) / 4096) { + uintptr_t count = 4096 - (((uintptr_t)dest) & 4095); + _fill_preexisting_slice(j, dest, src, count); + src += count; + dest += count; + } + _fill_preexisting_slice(j, dest, src, end - dest); + +#ifdef STM_TESTS + /* can't really enable this check outside tests, because there is + a change that the transaction_state changes in parallel */ + if (get_priv_segment(j)->transaction_state != TS_NONE) { + assert(!was_read_remote(get_segment_base(j), (object_t *)nobj)); + } +#endif + } + + DEBUG_EXPECT_SEGFAULT(true); + release_privatization_lock(STM_SEGMENT->segment_num); + + write_fence(); /* make sure 'nobj' is fully initialized from + all threads here */ + return (object_t *)nobj; +} /************************************************************/ @@ -246,6 +298,8 @@ } +#define TRACE_FOR_MAJOR_COLLECTION (&mark_record_trace) + static void mark_and_trace( object_t *obj, char *segment_base, /* to trace obj in */ @@ -405,7 +459,8 @@ struct stm_undo_s *modified = (struct stm_undo_s *)lst->items; struct stm_undo_s *end = (struct stm_undo_s *)(lst->items + lst->count); for (; modified < end; modified++) { - if (modified->type == TYPE_POSITION_MARKER) + if (modified->type == TYPE_POSITION_MARKER && + modified->type2 != TYPE_MODIFIED_HASHTABLE) mark_visit_possibly_new_object(modified->marker_object, pseg); } } @@ -538,6 +593,31 @@ list_set_item(lst, n, list_pop_item(lst)); } } + + /* Remove from 'modified_old_objects' all old hashtables that die */ + { + lst = pseg->modified_old_objects; + uintptr_t j, k = 0, limit = list_count(lst); + for (j = 0; j < limit; j += 3) { + uintptr_t e0 = list_item(lst, j + 0); + uintptr_t e1 = list_item(lst, j + 1); + uintptr_t e2 = list_item(lst, j + 2); + if (e0 == TYPE_POSITION_MARKER && + e1 == TYPE_MODIFIED_HASHTABLE && + !mark_visited_test((object_t *)e2)) { + /* hashtable object dies */ + } + else { + if (j != k) { + list_set_item(lst, k + 0, e0); + list_set_item(lst, k + 1, e1); + list_set_item(lst, k + 2, e2); + } + k += 3; + } + } + lst->count = k; + } } #pragma pop_macro("STM_SEGMENT") #pragma pop_macro("STM_PSEGMENT") diff --git a/rpython/translator/stm/src_stm/stm/hashtable.c b/rpython/translator/stm/src_stm/stm/hashtable.c new file mode 100644 --- /dev/null +++ b/rpython/translator/stm/src_stm/stm/hashtable.c @@ -0,0 +1,532 @@ +/* Imported by rpython/translator/stm/import_stmgc.py */ +/* +Design of stmgc's "hashtable" objects +===================================== +A "hashtable" is theoretically a lazily-filled array of objects of +length 2**64. Initially it is full of NULLs. It's obviously +implemented as a dictionary in which NULL objects are not needed. + +A real dictionary can be implemented on top of it, by using the index +`hash(key)` in the hashtable, and storing a list of `(key, value)` +pairs at that index (usually only one, unless there is a hash +collision). + +The main operations on a hashtable are reading or writing an object at a +given index. It also supports fetching the list of non-NULL entries. + +There are two markers for every index (a read and a write marker). +This is unlike regular arrays, which have only two markers in total. + +Additionally, we use the read marker for the hashtable object itself +to mean "we have read the complete list of keys". This plays the role +of a "global" read marker: when any thread adds a new key/value object +to the hashtable, this new object's read marker is initialized with a +copy of the "global" read marker --- in all segments. + + +Implementation +-------------- + +First idea: have the hashtable in raw memory, pointing to "entry" +objects (which are regular, GC- and STM-managed objects). The entry +objects themselves point to the user-specified objects. The entry +objects hold the read/write markers. Every entry object, once +created, stays around. It is only removed by the next major GC if it +points to NULL and its read/write markers are not set in any +currently-running transaction. + +References +---------- + +Inspired by: http://ppl.stanford.edu/papers/podc011-bronson.pdf +*/ + + +uint32_t stm_hashtable_entry_userdata; + + +#define INITIAL_HASHTABLE_SIZE 8 +#define PERTURB_SHIFT 5 +#define RESIZING_LOCK 0 + +typedef struct { + uintptr_t mask; + + /* 'resize_counter' start at an odd value, and is decremented (by + 6) for every new item put in 'items'. When it crosses 0, we + instead allocate a bigger table and change 'resize_counter' to + be a regular pointer to it (which is then even). The whole + structure is immutable then. + + The field 'resize_counter' also works as a write lock: changes + go via the intermediate value RESIZING_LOCK (0). + */ + uintptr_t resize_counter; + + stm_hashtable_entry_t *items[INITIAL_HASHTABLE_SIZE]; +} stm_hashtable_table_t; + +#define IS_EVEN(p) (((p) & 1) == 0) + +struct stm_hashtable_s { + stm_hashtable_table_t *table; + stm_hashtable_table_t initial_table; + uint64_t additions; +}; + + +static inline void init_table(stm_hashtable_table_t *table, uintptr_t itemcount) +{ + table->mask = itemcount - 1; + table->resize_counter = itemcount * 4 + 1; + memset(table->items, 0, itemcount * sizeof(stm_hashtable_entry_t *)); +} + +stm_hashtable_t *stm_hashtable_create(void) +{ + stm_hashtable_t *hashtable = malloc(sizeof(stm_hashtable_t)); + assert(hashtable); + hashtable->table = &hashtable->initial_table; + hashtable->additions = 0; + init_table(&hashtable->initial_table, INITIAL_HASHTABLE_SIZE); + return hashtable; +} + +void stm_hashtable_free(stm_hashtable_t *hashtable) +{ + uintptr_t rc = hashtable->initial_table.resize_counter; + free(hashtable); + while (IS_EVEN(rc)) { + assert(rc != RESIZING_LOCK); + + stm_hashtable_table_t *table = (stm_hashtable_table_t *)rc; + rc = table->resize_counter; + free(table); + } +} + +static bool _stm_was_read_by_anybody(object_t *obj) +{ + /* can only be safely called during major GC, when all other threads + are suspended */ + long i; + for (i = 1; i < NB_SEGMENTS; i++) { + if (get_priv_segment(i)->transaction_state == TS_NONE) + continue; + if (was_read_remote(get_segment_base(i), obj)) + return true; + } + return false; +} + +#define VOLATILE_HASHTABLE(p) ((volatile stm_hashtable_t *)(p)) +#define VOLATILE_TABLE(p) ((volatile stm_hashtable_table_t *)(p)) + +static void _insert_clean(stm_hashtable_table_t *table, + stm_hashtable_entry_t *entry, + uintptr_t index) +{ + uintptr_t mask = table->mask; + uintptr_t i = index & mask; + if (table->items[i] == NULL) { + table->items[i] = entry; + return; + } + + uintptr_t perturb = index; + while (1) { + i = (i << 2) + i + perturb + 1; + i &= mask; + if (table->items[i] == NULL) { + table->items[i] = entry; + return; + } + + perturb >>= PERTURB_SHIFT; + } +} + +static void _stm_rehash_hashtable(stm_hashtable_t *hashtable, + uintptr_t biggercount, + char *segment_base) +{ + dprintf(("rehash %p to size %ld, segment_base=%p\n", + hashtable, biggercount, segment_base)); + + size_t size = (offsetof(stm_hashtable_table_t, items) + + biggercount * sizeof(stm_hashtable_entry_t *)); + stm_hashtable_table_t *biggertable = malloc(size); + assert(biggertable); // XXX + + stm_hashtable_table_t *table = hashtable->table; + table->resize_counter = (uintptr_t)biggertable; + /* ^^^ this unlocks the table by writing a non-zero value to + table->resize_counter, but the new value is a pointer to the + new bigger table, so IS_EVEN() is still true */ + assert(IS_EVEN(table->resize_counter)); + + init_table(biggertable, biggercount); + + uintptr_t j, mask = table->mask; + uintptr_t rc = biggertable->resize_counter; + for (j = 0; j <= mask; j++) { + stm_hashtable_entry_t *entry = table->items[j]; + if (entry == NULL) + continue; + if (segment_base != NULL) { + if (((struct stm_hashtable_entry_s *) + REAL_ADDRESS(segment_base, entry))->object == NULL && + !_stm_was_read_by_anybody((object_t *)entry)) { + dprintf((" removing dead %p\n", entry)); + continue; + } + } + + uintptr_t eindex; + if (segment_base == NULL) + eindex = entry->index; /* read from STM_SEGMENT */ + else + eindex = ((struct stm_hashtable_entry_s *) + REAL_ADDRESS(segment_base, entry))->index; + + dprintf((" insert_clean %p at index=%ld\n", + entry, eindex)); + _insert_clean(biggertable, entry, eindex); + assert(rc > 6); + rc -= 6; + } + biggertable->resize_counter = rc; + + write_fence(); /* make sure that 'biggertable' is valid here, + and make sure 'table->resize_counter' is updated + ('table' must be immutable from now on). */ + VOLATILE_HASHTABLE(hashtable)->table = biggertable; +} + +stm_hashtable_entry_t *stm_hashtable_lookup(object_t *hashtableobj, + stm_hashtable_t *hashtable, + uintptr_t index) +{ + stm_hashtable_table_t *table; + uintptr_t mask; + uintptr_t i; + stm_hashtable_entry_t *entry; + + restart: + /* classical dict lookup logic */ + table = VOLATILE_HASHTABLE(hashtable)->table; + mask = table->mask; /* read-only field */ + i = index & mask; + entry = VOLATILE_TABLE(table)->items[i]; + if (entry != NULL) { + if (entry->index == index) + return entry; /* found at the first try */ + + uintptr_t perturb = index; + while (1) { + i = (i << 2) + i + perturb + 1; + i &= mask; + entry = VOLATILE_TABLE(table)->items[i]; + if (entry != NULL) { + if (entry->index == index) + return entry; /* found */ + } + else + break; + perturb >>= PERTURB_SHIFT; + } + } + /* here, we didn't find the 'entry' with the correct index. Note + that even if the same 'table' is modified or resized by other + threads concurrently, any new item found from a race condition + would anyway contain NULL in the present segment (ensured by + the first write_fence() below). If the 'table' grows an entry + just after we checked above, then we go ahead and lock the + table; but after we get the lock, we will notice the new entry + (ensured by the second write_fence() below) and restart the + whole process. + */ + + uintptr_t rc = VOLATILE_TABLE(table)->resize_counter; + + /* if rc is RESIZING_LOCK (which is 0, so even), a concurrent thread + is writing to the hashtable. Or, if rc is another even number, it is + actually a pointer to the next version of the table, installed + just now. In both cases, this thread must simply spin loop. + */ + if (IS_EVEN(rc)) { + spin_loop(); + goto restart; + } + /* in the other cases, we need to grab the RESIZING_LOCK. + */ + if (!__sync_bool_compare_and_swap(&table->resize_counter, + rc, RESIZING_LOCK)) { + goto restart; + } + /* we now have the lock. The only table with a non-even value of + 'resize_counter' should be the last one in the chain, so if we + succeeded in locking it, check this. */ + assert(table == hashtable->table); + + /* Check that 'table->items[i]' is still NULL, + i.e. hasn't been populated under our feet. + */ + if (table->items[i] != NULL) { + table->resize_counter = rc; /* unlock */ + goto restart; + } + /* if rc is greater than 6, there is enough room for a new + item in the current table. + */ + if (rc > 6) { + /* we can only enter here once! If we allocate stuff, we may + run the GC, and so 'hashtableobj' might move afterwards. */ + if (_is_in_nursery(hashtableobj)) { + entry = (stm_hashtable_entry_t *) + stm_allocate(sizeof(stm_hashtable_entry_t)); + entry->userdata = stm_hashtable_entry_userdata; + entry->index = index; + entry->object = NULL; + } + else { + /* for a non-nursery 'hashtableobj', we pretend that the + 'entry' object we're about to return was already + existing all along, with NULL in all segments. If the + caller of this function is going to modify the 'object' + field, it will call stm_write(entry) first, which will + correctly schedule 'entry' for write propagation. We + do that even if 'hashtableobj' was created by the + running transaction: the new 'entry' object is created + as if it was older than the transaction. + + Note the following difference: if 'hashtableobj' is + still in the nursery (case above), the 'entry' object + is also allocated from the nursery, and after a minor + collection it ages as an old-but-created-by-the- + current-transaction object. We could try to emulate + this here, or to create young 'entry' objects, but + doing either of these would require careful + synchronization with other pieces of the code that may + change. + */ + struct stm_hashtable_entry_s initial = { + .userdata = stm_hashtable_entry_userdata, + .index = index, + .object = NULL + }; + entry = (stm_hashtable_entry_t *) + stm_allocate_preexisting(sizeof(stm_hashtable_entry_t), + (char *)&initial.header); + hashtable->additions++; + } + table->items[i] = entry; + write_fence(); /* make sure 'table->items' is written here */ + VOLATILE_TABLE(table)->resize_counter = rc - 6; /* unlock */ + return entry; + } + else { + /* if rc is smaller than 6, we must allocate a new bigger table. + */ + uintptr_t biggercount = table->mask + 1; + if (biggercount < 50000) + biggercount *= 4; + else + biggercount *= 2; + _stm_rehash_hashtable(hashtable, biggercount, /*segment_base=*/NULL); + goto restart; + } +} + +object_t *stm_hashtable_read(object_t *hobj, stm_hashtable_t *hashtable, + uintptr_t key) +{ + stm_hashtable_entry_t *e = stm_hashtable_lookup(hobj, hashtable, key); + stm_read((object_t *)e); + return e->object; +} + +void stm_hashtable_write_entry(object_t *hobj, stm_hashtable_entry_t *entry, + object_t *nvalue) +{ + if (_STM_WRITE_CHECK_SLOWPATH((object_t *)entry)) { + + stm_write((object_t *)entry); + + uintptr_t i = list_count(STM_PSEGMENT->modified_old_objects); + if (i > 0 && list_item(STM_PSEGMENT->modified_old_objects, i - 3) + == (uintptr_t)entry) { + /* The stm_write() above recorded a write to 'entry'. Here, + we add another stm_undo_s to modified_old_objects with + TYPE_MODIFIED_HASHTABLE. It is ignored everywhere except + in _stm_validate(). + + The goal is that this TYPE_MODIFIED_HASHTABLE ends up in + the commit log's 'cl_written' array. Later, another + transaction validating that log will check two things: + + - the regular stm_undo_s entry put by stm_write() above + will make the other transaction check that it didn't + read the same 'entry' object; + + - the TYPE_MODIFIED_HASHTABLE entry we're adding now + will make the other transaction check that it didn't + do any stm_hashtable_list() on the complete hashtable. + */ + STM_PSEGMENT->modified_old_objects = list_append3( + STM_PSEGMENT->modified_old_objects, + TYPE_POSITION_MARKER, /* type1 */ + TYPE_MODIFIED_HASHTABLE, /* type2 */ + (uintptr_t)hobj); /* modif_hashtable */ + } + } + entry->object = nvalue; +} + +void stm_hashtable_write(object_t *hobj, stm_hashtable_t *hashtable, + uintptr_t key, object_t *nvalue, + stm_thread_local_t *tl) +{ + STM_PUSH_ROOT(*tl, nvalue); + STM_PUSH_ROOT(*tl, hobj); + stm_hashtable_entry_t *e = stm_hashtable_lookup(hobj, hashtable, key); + STM_POP_ROOT(*tl, hobj); + STM_POP_ROOT(*tl, nvalue); + stm_hashtable_write_entry(hobj, e, nvalue); +} + +long stm_hashtable_length_upper_bound(stm_hashtable_t *hashtable) +{ + stm_hashtable_table_t *table; + uintptr_t rc; + + restart: + table = VOLATILE_HASHTABLE(hashtable)->table; + rc = VOLATILE_TABLE(table)->resize_counter; + if (IS_EVEN(rc)) { + spin_loop(); + goto restart; + } + + uintptr_t initial_rc = (table->mask + 1) * 4 + 1; + uintptr_t num_entries_times_6 = initial_rc - rc; + return num_entries_times_6 / 6; +} + +long stm_hashtable_list(object_t *hobj, stm_hashtable_t *hashtable, + stm_hashtable_entry_t **results) +{ + /* Set the read marker. It will be left as long as we're running + the same transaction. + */ + stm_read(hobj); + + /* Get the table. No synchronization is needed: we may miss some + entries that are being added, but they would contain NULL in + this segment anyway. */ + stm_hashtable_table_t *table = VOLATILE_HASHTABLE(hashtable)->table; + + /* Read all entries, check which ones are not NULL, count them, + and optionally list them in 'results'. + */ + uintptr_t i, mask = table->mask; + stm_hashtable_entry_t *entry; + long nresult = 0; + + if (results != NULL) { + /* collect the results in the provided list */ + for (i = 0; i <= mask; i++) { + entry = VOLATILE_TABLE(table)->items[i]; + if (entry != NULL) { + stm_read((object_t *)entry); + if (entry->object != NULL) + results[nresult++] = entry; + } + } + } + else { + /* don't collect, just get the exact number of results */ + for (i = 0; i <= mask; i++) { + entry = VOLATILE_TABLE(table)->items[i]; + if (entry != NULL) { + stm_read((object_t *)entry); + if (entry->object != NULL) + nresult++; + } + } + } + return nresult; +} + +static void _stm_compact_hashtable(struct object_s *hobj, + stm_hashtable_t *hashtable) +{ + stm_hashtable_table_t *table = hashtable->table; + uintptr_t rc = table->resize_counter; + assert(!IS_EVEN(rc)); + + if (hashtable->additions * 4 > table->mask) { + hashtable->additions = 0; + + /* If 'hobj' was created in some current transaction, i.e. if it is + now an overflow object, then we have the risk that some of its + entry objects were not created with stm_allocate_preexisting(). + In that situation, a valid workaround is to read all entry + objects in the segment of the running transaction. Otherwise, + the base case is to read them all from segment zero. + */ + long segnum = get_num_segment_containing_address((char *)hobj); + if (!IS_OVERFLOW_OBJ(get_priv_segment(segnum), hobj)) + segnum = 0; + + uintptr_t initial_rc = (table->mask + 1) * 4 + 1; + uintptr_t num_entries_times_6 = initial_rc - rc; + uintptr_t count = INITIAL_HASHTABLE_SIZE; + while (count * 4 < num_entries_times_6) + count *= 2; + /* sanity-check: 'num_entries_times_6 < initial_rc', and so 'count' + can never grow larger than the current table size. */ + assert(count <= table->mask + 1); + + dprintf(("compact with %ld items:\n", num_entries_times_6 / 6)); + _stm_rehash_hashtable(hashtable, count, get_segment_base(segnum)); + } + + table = hashtable->table; + assert(!IS_EVEN(table->resize_counter)); + + if (table != &hashtable->initial_table) { + uintptr_t rc = hashtable->initial_table.resize_counter; + while (1) { + assert(IS_EVEN(rc)); + assert(rc != RESIZING_LOCK); + + stm_hashtable_table_t *old_table = (stm_hashtable_table_t *)rc; + if (old_table == table) + break; + rc = old_table->resize_counter; + free(old_table); + } + hashtable->initial_table.resize_counter = (uintptr_t)table; + assert(IS_EVEN(hashtable->initial_table.resize_counter)); + } +} + +void stm_hashtable_tracefn(struct object_s *hobj, stm_hashtable_t *hashtable, + void trace(object_t **)) +{ + if (trace == TRACE_FOR_MAJOR_COLLECTION) + _stm_compact_hashtable(hobj, hashtable); + + stm_hashtable_table_t *table; + table = VOLATILE_HASHTABLE(hashtable)->table; + + uintptr_t j, mask = table->mask; + for (j = 0; j <= mask; j++) { + stm_hashtable_entry_t *volatile *pentry; + pentry = &VOLATILE_TABLE(table)->items[j]; + if (*pentry != NULL) { + trace((object_t **)pentry); + } + } +} diff --git a/rpython/translator/stm/src_stm/stm/marker.c b/rpython/translator/stm/src_stm/stm/marker.c --- a/rpython/translator/stm/src_stm/stm/marker.c +++ b/rpython/translator/stm/src_stm/stm/marker.c @@ -42,7 +42,8 @@ */ while (contention != start) { --contention; - if (contention->type == TYPE_POSITION_MARKER) { + if (contention->type == TYPE_POSITION_MARKER && + contention->type2 != TYPE_MODIFIED_HASHTABLE) { out_marker->odd_number = contention->marker_odd_number; out_marker->object = contention->marker_object; return; @@ -69,6 +70,9 @@ return; /* already up-to-date */ } + /* -2 is not odd */ + assert(marker.odd_number != (uintptr_t)TYPE_MODIFIED_HASHTABLE); + STM_PSEGMENT->position_markers_last = list_count(list); STM_PSEGMENT->modified_old_objects = list_append3( list, diff --git a/rpython/translator/stm/src_stm/stm/misc.c b/rpython/translator/stm/src_stm/stm/misc.c --- a/rpython/translator/stm/src_stm/stm/misc.c +++ b/rpython/translator/stm/src_stm/stm/misc.c @@ -31,10 +31,7 @@ bool _stm_was_read(object_t *obj) { - uint8_t rm = ((struct stm_read_marker_s *) - (STM_SEGMENT->segment_base + (((uintptr_t)obj) >> 4)))->rm; - assert(rm <= STM_SEGMENT->transaction_read_version); - return rm == STM_SEGMENT->transaction_read_version; + return was_read_remote(STM_SEGMENT->segment_base, obj); } bool _stm_was_written(object_t *obj) diff --git a/rpython/translator/stm/src_stm/stm/nursery.c b/rpython/translator/stm/src_stm/stm/nursery.c --- a/rpython/translator/stm/src_stm/stm/nursery.c +++ b/rpython/translator/stm/src_stm/stm/nursery.c @@ -422,6 +422,7 @@ struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count); for (; undo < end; undo++) { + /* this logic also works if type2 == TYPE_MODIFIED_HASHTABLE */ if (undo->type == TYPE_POSITION_MARKER) minor_trace_if_young(&undo->marker_object); } diff --git a/rpython/translator/stm/src_stm/stm/pages.h b/rpython/translator/stm/src_stm/stm/pages.h --- a/rpython/translator/stm/src_stm/stm/pages.h +++ b/rpython/translator/stm/src_stm/stm/pages.h @@ -62,7 +62,11 @@ static inline bool get_page_status_in(long segnum, uintptr_t pagenum) { - /* reading page status requires "read"-lock: */ + /* reading page status requires "read"-lock, which is defined as + "any segment has the privatization_lock". This is enough to + prevent the "write"-lock from being acquired by somebody else + (defined as "_all_ segments have the privatization_lock"). + */ assert(STM_PSEGMENT->privatization_lock); OPT_ASSERT(segnum < 8 * sizeof(struct page_shared_s)); diff --git a/rpython/translator/stm/src_stm/stm/setup.c b/rpython/translator/stm/src_stm/stm/setup.c --- a/rpython/translator/stm/src_stm/stm/setup.c +++ b/rpython/translator/stm/src_stm/stm/setup.c @@ -250,8 +250,6 @@ set_gs_register(get_segment_base(num + 1)); s_mutex_unlock(); - DEBUG_EXPECT_SEGFAULT(true); - if (num == 0) { dprintf(("STM_GC_NURSERY: %d\n", STM_GC_NURSERY)); dprintf(("NB_PAGES: %d\n", NB_PAGES)); diff --git a/rpython/translator/stm/src_stm/stm/setup.h b/rpython/translator/stm/src_stm/stm/setup.h --- a/rpython/translator/stm/src_stm/stm/setup.h +++ b/rpython/translator/stm/src_stm/stm/setup.h @@ -3,8 +3,8 @@ static void setup_protection_settings(void); static pthread_t *_get_cpth(stm_thread_local_t *); #ifndef NDEBUG -static __thread long _stm_segfault_expected = 0; -#define DEBUG_EXPECT_SEGFAULT(v) do {if (v) _stm_segfault_expected++; else _stm_segfault_expected--;} while (0) +static __thread long _stm_segfault_expected = 1; +#define DEBUG_EXPECT_SEGFAULT(v) do {if (v) _stm_segfault_expected++; else _stm_segfault_expected--; assert(_stm_segfault_expected <= 1);} while (0) #else #define DEBUG_EXPECT_SEGFAULT(v) {} #endif diff --git a/rpython/translator/stm/src_stm/stmgc.c b/rpython/translator/stm/src_stm/stmgc.c --- a/rpython/translator/stm/src_stm/stmgc.c +++ b/rpython/translator/stm/src_stm/stmgc.c @@ -39,3 +39,4 @@ #include "stm/prof.c" #include "stm/rewind_setjmp.c" #include "stm/finalizer.c" +#include "stm/hashtable.c" diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h --- a/rpython/translator/stm/src_stm/stmgc.h +++ b/rpython/translator/stm/src_stm/stmgc.h @@ -192,10 +192,13 @@ STM_SEGMENT->transaction_read_version; } +#define _STM_WRITE_CHECK_SLOWPATH(obj) \ + UNLIKELY(((obj)->stm_flags & _STM_GCFLAG_WRITE_BARRIER) != 0) + __attribute__((always_inline)) static inline void stm_write(object_t *obj) { - if (UNLIKELY((obj->stm_flags & _STM_GCFLAG_WRITE_BARRIER) != 0)) + if (_STM_WRITE_CHECK_SLOWPATH(obj)) _stm_write_slowpath(obj); } @@ -204,7 +207,7 @@ static inline void stm_write_card(object_t *obj, uintptr_t index) { /* if GCFLAG_WRITE_BARRIER is set, then don't do anything more. */ - if (UNLIKELY((obj->stm_flags & _STM_GCFLAG_WRITE_BARRIER) != 0)) { + if (_STM_WRITE_CHECK_SLOWPATH(obj)) { /* GCFLAG_WRITE_BARRIER is not set. This might be because it's the first time we see a given small array; or it might @@ -467,6 +470,38 @@ /* dummies for now: */ static inline void stm_flush_timing(stm_thread_local_t *tl, int verbose) {} + +/* Hashtables. Keys are 64-bit unsigned integers, values are + 'object_t *'. Note that the type 'stm_hashtable_t' is not an + object type at all; you need to allocate and free it explicitly. + If you want to embed the hashtable inside an 'object_t' you + probably need a light finalizer to do the freeing. */ +typedef struct stm_hashtable_s stm_hashtable_t; +typedef TLPREFIX struct stm_hashtable_entry_s stm_hashtable_entry_t; + +stm_hashtable_t *stm_hashtable_create(void); +void stm_hashtable_free(stm_hashtable_t *); +stm_hashtable_entry_t *stm_hashtable_lookup(object_t *, stm_hashtable_t *, + uintptr_t key); +object_t *stm_hashtable_read(object_t *, stm_hashtable_t *, uintptr_t key); +void stm_hashtable_write(object_t *, stm_hashtable_t *, uintptr_t key, + object_t *nvalue, stm_thread_local_t *); +void stm_hashtable_write_entry(object_t *hobj, stm_hashtable_entry_t *entry, + object_t *nvalue); +long stm_hashtable_length_upper_bound(stm_hashtable_t *); +long stm_hashtable_list(object_t *, stm_hashtable_t *, + stm_hashtable_entry_t **results); +extern uint32_t stm_hashtable_entry_userdata; +void stm_hashtable_tracefn(struct object_s *, stm_hashtable_t *, + void (object_t **)); + +struct stm_hashtable_entry_s { + struct object_s header; + uint32_t userdata; + uintptr_t index; + object_t *object; +}; + /* ==================== END ==================== */ static void (*stmcb_expand_marker)(char *segment_base, uintptr_t odd_number, From noreply at buildbot.pypy.org Thu Mar 12 17:56:38 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 12 Mar 2015 17:56:38 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c8-hashtable: Re-introduce this part from stmgc-c7 Message-ID: <20150312165638.B6B901C00EF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c8-hashtable Changeset: r76356:8dead8bff1d7 Date: 2015-03-12 17:49 +0100 http://bitbucket.org/pypy/pypy/changeset/8dead8bff1d7/ Log: Re-introduce this part from stmgc-c7 diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py --- a/rpython/rlib/rstm.py +++ b/rpython/rlib/rstm.py @@ -256,8 +256,7 @@ 'freelist': _ll_hashtable_freelist, 'lookup': _ll_hashtable_lookup, 'writeobj': _ll_hashtable_writeobj}) -# NULL_HASHTABLE = lltype.nullptr(_HASHTABLE_OBJ) -NULL_HASHTABLE = None +NULL_HASHTABLE = lltype.nullptr(_HASHTABLE_OBJ) def _ll_hashtable_trace(gc, obj, callback, arg): from rpython.memory.gctransform.stmframework import get_visit_function @@ -277,67 +276,23 @@ def create_hashtable(): if not we_are_translated(): return HashtableForTest() # for tests - return HashtableEmulation() - # rgc.register_custom_light_finalizer(_HASHTABLE_OBJ, lambda_hashtable_finlz) - # rgc.register_custom_trace_hook(_HASHTABLE_OBJ, lambda_hashtable_trace) - # # Pass a null pointer to _STM_HASHTABLE_ENTRY to stm_hashtable_create(). - # # Make sure we see a malloc() of it, so that its typeid is correctly - # # initialized. It can be done in a NonConstant(False) path so that - # # the C compiler will actually drop it. - # if _false: - # p = lltype.malloc(_STM_HASHTABLE_ENTRY) - # else: - # p = lltype.nullptr(_STM_HASHTABLE_ENTRY) - # h = lltype.malloc(_HASHTABLE_OBJ) - # h.ll_raw_hashtable = lltype.nullptr(_STM_HASHTABLE_P.TO) - # h.ll_raw_hashtable = llop.stm_hashtable_create(_STM_HASHTABLE_P, p) - # return h + rgc.register_custom_light_finalizer(_HASHTABLE_OBJ, lambda_hashtable_finlz) + rgc.register_custom_trace_hook(_HASHTABLE_OBJ, lambda_hashtable_trace) + # Pass a null pointer to _STM_HASHTABLE_ENTRY to stm_hashtable_create(). + # Make sure we see a malloc() of it, so that its typeid is correctly + # initialized. It can be done in a NonConstant(False) path so that + # the C compiler will actually drop it. + if _false: + p = lltype.malloc(_STM_HASHTABLE_ENTRY) + else: + p = lltype.nullptr(_STM_HASHTABLE_ENTRY) + h = lltype.malloc(_HASHTABLE_OBJ) + h.ll_raw_hashtable = lltype.nullptr(_STM_HASHTABLE_P.TO) + h.ll_raw_hashtable = llop.stm_hashtable_create(_STM_HASHTABLE_P, p) + return h NULL_GCREF = lltype.nullptr(llmemory.GCREF.TO) -class HashtableEmulation(object): - def __init__(self): - self._content = {} # dict {integer: GCREF} - - def get(self, key): - return self._content.get(key, NULL_GCREF) - - def set(self, key, value): - if value: - self._content[key] = value - else: - try: - del self._content[key] - except KeyError: - pass - - def len(self): - return len(self._content) - - def list(self): - items = [] - for key in self._content.keys(): - items.append(self.lookup(key)) - count = len(items) - return items, count - - def freelist(self, array): - pass - - def lookup(self, key): - return EntryObjectEmulation(self, key) - - def writeobj(self, entry, nvalue): - self.set(entry.key, nvalue) - -class EntryObjectEmulation(object): - def __init__(self, hashtable, key): - self.hashtable = hashtable - self.key = key - self.index = r_uint(key) - self.object = hashtable.get(key) - - class HashtableForTest(object): def __init__(self): self._content = {} # dict {integer: GCREF} From noreply at buildbot.pypy.org Thu Mar 12 17:56:39 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 12 Mar 2015 17:56:39 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c8-hashtable: Pass the extra arg to stm_hashtable_tracefn(). Add some casts somewhere Message-ID: <20150312165639.CA6801C00EF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c8-hashtable Changeset: r76357:9e912907fe1c Date: 2015-03-12 17:56 +0100 http://bitbucket.org/pypy/pypy/changeset/9e912907fe1c/ Log: Pass the extra arg to stm_hashtable_tracefn(). Add some casts somewhere else in funcgen.py to silence warnings. diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py --- a/rpython/rlib/rstm.py +++ b/rpython/rlib/rstm.py @@ -262,7 +262,7 @@ from rpython.memory.gctransform.stmframework import get_visit_function visit_fn = get_visit_function(callback, arg) addr = obj + llmemory.offsetof(_HASHTABLE_OBJ, 'll_raw_hashtable') - llop.stm_hashtable_tracefn(lltype.Void, addr.address[0], visit_fn) + llop.stm_hashtable_tracefn(lltype.Void, obj, addr.address[0], visit_fn) lambda_hashtable_trace = lambda: _ll_hashtable_trace def _ll_hashtable_finalizer(h): diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py --- a/rpython/translator/stm/funcgen.py +++ b/rpython/translator/stm/funcgen.py @@ -334,8 +334,9 @@ arg1 = funcgen.expr(op.args[1]) arg2 = funcgen.expr(op.args[2]) result = funcgen.expr(op.result) - return '%s = stm_hashtable_lookup((object_t *)%s, %s, %s);' % ( - result, arg0, arg1, arg2) + typename = cdecl(funcgen.lltypename(op.result), '') + return '%s = (%s)stm_hashtable_lookup((object_t *)%s, %s, %s);' % ( + result, typename, arg0, arg1, arg2) def stm_hashtable_length_upper_bound(funcgen, op): arg0 = funcgen.expr(op.args[0]) @@ -348,10 +349,12 @@ arg1 = funcgen.expr(op.args[1]) arg2 = funcgen.expr(op.args[2]) result = funcgen.expr(op.result) - return '%s = stm_hashtable_list((object_t *)%s, %s, %s);' % ( - result, arg0, arg1, arg2) + return ('%s = stm_hashtable_list((object_t *)%s, %s, ' + '(stm_hashtable_entry_t **)%s);' % (result, arg0, arg1, arg2)) def stm_hashtable_tracefn(funcgen, op): arg0 = funcgen.expr(op.args[0]) arg1 = funcgen.expr(op.args[1]) - return 'stm_hashtable_tracefn((stm_hashtable_t *)%s, %s);' % (arg0, arg1) + arg2 = funcgen.expr(op.args[2]) + return ('stm_hashtable_tracefn(%s, (stm_hashtable_t *)%s, ' + ' (void(*)(object_t**))%s);' % (arg0, arg1, arg2)) From noreply at buildbot.pypy.org Thu Mar 12 18:02:50 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 12 Mar 2015 18:02:50 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c8-hashtable: hg merge c8ea66315864 (I made the branch slightly too early) Message-ID: <20150312170250.01C0F1C0473@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c8-hashtable Changeset: r76358:090025f6da69 Date: 2015-03-12 18:02 +0100 http://bitbucket.org/pypy/pypy/changeset/090025f6da69/ Log: hg merge c8ea66315864 (I made the branch slightly too early) diff --git a/pypy/stm/print_stm_log.py b/pypy/stm/print_stm_log.py --- a/pypy/stm/print_stm_log.py +++ b/pypy/stm/print_stm_log.py @@ -91,7 +91,10 @@ self.cpu_time_committed = 0.0 self.cpu_time_aborted = 0.0 self.cpu_time_paused = 0.0 + self.cpu_time_gc_minor = 0.0 + self.cpu_time_gc_major = 0.0 self._prev = (0.0, "stop") + self._in_major_coll = None self.reset_counters() def reset_counters(self): @@ -99,6 +102,7 @@ self._transaction_pause_time = 0.0 self._transaction_aborting = False self._transaction_inev = None + self._in_minor_coll = None def transaction_start(self, entry): self.reset_counters() @@ -167,9 +171,29 @@ def wait_for_other_inev(self, wait_time, out_conflicts): c = self.get_conflict(self._transaction_inev[0], out_conflicts) - assert wait_time >= 0 + assert wait_time >= 0.0 c.paused_time += wait_time + def gc_minor_start(self, event): + self._in_minor_coll = event.timestamp + + def gc_minor_done(self, event): + if self._in_minor_coll is not None: + gc_time = event.timestamp - self._in_minor_coll + assert gc_time >= 0.0 + self.cpu_time_gc_minor += gc_time + self._in_minor_coll = None + + def gc_major_start(self, event): + self._in_major_coll = event.timestamp + + def gc_major_done(self, event): + if self._in_major_coll is not None: + gc_time = event.timestamp - self._in_major_coll + assert gc_time >= 0.0 + self.cpu_time_gc_major += gc_time + self._in_major_coll = None + class ConflictSummary(object): def __init__(self, event, marker): @@ -250,6 +274,14 @@ t.transaction_pause(entry) elif entry.event == STM_WAIT_DONE: t.transaction_unpause(entry, conflicts) + elif entry.event == STM_GC_MINOR_START: + t.gc_minor_start(entry) + elif entry.event == STM_GC_MINOR_DONE: + t.gc_minor_done(entry) + elif entry.event == STM_GC_MAJOR_START: + t.gc_major_start(entry) + elif entry.event == STM_GC_MAJOR_DONE: + t.gc_major_done(entry) # if cnt == 0: raise Exception("empty file") @@ -264,7 +296,7 @@ start_time = stmlog.start_time total_time = stmlog.total_time print - print 'Total real time: %.3fs' % (total_time,) + print 'Total real time: %9.3fs' % (total_time,) # total_cpu_time_committed = stmlog.get_total_cpu_time_committed() total_cpu_time_aborted = stmlog.get_total_cpu_time_aborted() @@ -272,14 +304,20 @@ total_cpu_time_total = (total_cpu_time_committed + total_cpu_time_aborted + total_cpu_time_paused) - print 'CPU time in STM mode: %.3fs (%s) committed' % ( + total_cpu_time_gc_minor = stmlog.get_total_cpu_time_gc_minor() + total_cpu_time_gc_major = stmlog.get_total_cpu_time_gc_major() + print 'CPU time in STM mode:%9.3fs (%4s) committed' % ( total_cpu_time_committed, percent(total_cpu_time_committed, total_time)) - print ' %.3fs (%s) aborted' % ( + print ' %9.3fs (%4s) aborted' % ( total_cpu_time_aborted, percent(total_cpu_time_aborted, total_time)) - print ' %.3fs (%s) paused' % ( + print ' %9.3fs (%4s) paused' % ( total_cpu_time_paused, percent(total_cpu_time_paused, total_time)) - print ' %.3fs (%s) total' % ( + print ' %9.3fs (%4s) TOTAL' % ( total_cpu_time_total, percent(total_cpu_time_total, total_time)) + print ' including %9.3fs (%4s) minor GC collections' % ( + total_cpu_time_gc_minor, percent(total_cpu_time_gc_minor, total_time)) + print ' and %9.3fs (%4s) major GC collections' % ( + total_cpu_time_gc_major, percent(total_cpu_time_gc_major, total_time)) print # values = stmlog.get_conflicts() @@ -308,6 +346,12 @@ def get_total_cpu_time_paused(self): return sum([v.cpu_time_paused for v in self.threads.values()]) + def get_total_cpu_time_gc_minor(self): + return sum([v.cpu_time_gc_minor for v in self.threads.values()]) + + def get_total_cpu_time_gc_major(self): + return sum([v.cpu_time_gc_major for v in self.threads.values()]) + def get_conflicts(self): values = self.conflicts.values() values.sort(key=ConflictSummary.sortkey) diff --git a/rpython/translator/stm/test/test_inevitable.py b/rpython/translator/stm/test/test_inevitable.py --- a/rpython/translator/stm/test/test_inevitable.py +++ b/rpython/translator/stm/test/test_inevitable.py @@ -6,7 +6,7 @@ from rpython.translator.stm.inevitable import insert_turn_inevitable from rpython.translator.stm import inevitable from rpython.conftest import option - +import py CATEGORIES = [inevitable.ALWAYS_ALLOW_OPERATIONS, inevitable.CALLS, @@ -38,8 +38,7 @@ class LLSTMInevFrame(LLFrame): def op_stm_become_inevitable(self, info): assert info is not None - if self.llinterpreter.inevitable_cause is None: - self.llinterpreter.inevitable_cause = info + self.llinterpreter.inevitable_cause.append(info) def op_gc_dump_rpy_heap(self): pass # for test_unsupported_op @@ -65,7 +64,7 @@ if option.view: self.translator.view() # - interp.inevitable_cause = None + interp.inevitable_cause = [] result = interp.eval_graph(self.graph, args) return interp.inevitable_cause @@ -79,7 +78,7 @@ x1.foo = n res = self.interpret_inevitable(f1, [4]) - assert res is None + assert res == [] def test_unsupported_op(self): X = lltype.Struct('X', ('foo', lltype.Signed)) @@ -89,7 +88,7 @@ llop.gc_dump_rpy_heap(lltype.Void) res = self.interpret_inevitable(f1, []) - assert res == 'gc_dump_rpy_heap' + assert res == ['gc_dump_rpy_heap'] def test_raw_getfield(self): X = lltype.Struct('X', ('foo', lltype.Signed)) @@ -100,7 +99,7 @@ return x1.foo res = self.interpret_inevitable(f1, []) - assert res == 'getfield' + assert res == ['getfield'] def test_raw_getfield_immutable(self): X = lltype.Struct('X', ('foo', lltype.Signed), @@ -112,7 +111,7 @@ return x1.foo res = self.interpret_inevitable(f1, []) - assert res is None + assert res == [] def test_raw_getfield_with_hint(self): X = lltype.Struct('X', ('foo', lltype.Signed), @@ -124,7 +123,7 @@ return x1.foo res = self.interpret_inevitable(f1, []) - assert res is None + assert res == [] def test_raw_setfield(self): X = lltype.Struct('X', ('foo', lltype.Signed)) @@ -135,7 +134,7 @@ x1.foo = n res = self.interpret_inevitable(f1, [43]) - assert res == 'setfield' + assert res == ['setfield'] def test_malloc_no_inevitable(self): X = lltype.GcStruct('X', ('foo', lltype.Signed)) @@ -144,7 +143,7 @@ return lltype.malloc(X) res = self.interpret_inevitable(f1, []) - assert res is None + assert res == [] def test_raw_malloc_1(self): X = lltype.Struct('X', ('foo', lltype.Signed)) @@ -154,7 +153,7 @@ lltype.free(p, flavor='raw') res = self.interpret_inevitable(f1, []) - assert res is None + assert res == [] def test_raw_malloc_2(self): X = lltype.Struct('X', ('foo', lltype.Signed)) @@ -164,7 +163,7 @@ llmemory.raw_free(addr) res = self.interpret_inevitable(f1, []) - assert res is None + assert res == [] def test_unknown_raw_free(self): X = lltype.Struct('X', ('foo', lltype.Signed)) @@ -172,7 +171,7 @@ lltype.free(p, flavor='raw') res = self.interpret_inevitable(f2, [lltype.malloc(X, flavor='raw')]) - assert res is None + assert res == [] def test_ext_direct_call_safe(self): @@ -185,7 +184,7 @@ extfunc() res = self.interpret_inevitable(f1, []) - assert res is None + assert res == [] def test_ext_direct_call_unsafe(self): @@ -197,7 +196,7 @@ extfunc() res = self.interpret_inevitable(f1, []) - assert res == 'extfunc()' + assert res == ['extfunc()'] def test_rpy_direct_call(self): def f2(): @@ -206,7 +205,7 @@ f2() res = self.interpret_inevitable(f1, []) - assert res is None + assert res == [] def test_rpy_indirect_call(self): def f2(): @@ -221,7 +220,7 @@ f() res = self.interpret_inevitable(f1, [True]) - assert res is None + assert res == [] def test_ext_indirect_call(self): TYPE = lltype.FuncType([], lltype.Void) @@ -240,7 +239,7 @@ f() res = self.interpret_inevitable(f1, [True]) - assert res == 'indirect_call' + assert res == ['indirect_call'] def test_instantiate_indirect_call(self): # inits are necessary to generate indirect_call @@ -259,7 +258,7 @@ c() res = self.interpret_inevitable(f1, [True]) - assert res is None + assert res == [] def test_raw_class_hint(self): class A: @@ -278,7 +277,7 @@ return i res = self.interpret_inevitable(f, [2]) - assert res is None # not setfield or getfield or free + assert res == [] # not setfield or getfield or free def test_do_malloc_llops(self): def f(i): @@ -288,7 +287,7 @@ return i res = self.interpret_inevitable(f, [2]) - assert res is None + assert res == [] def test_raw_load_nonpure(self): X = lltype.Struct('X', ('foo', lltype.Signed)) @@ -300,7 +299,7 @@ lltype.Signed, llmemory.cast_ptr_to_adr(x1), 0, False) res = self.interpret_inevitable(f1, []) - assert res == 'raw_load' + assert res == ['raw_load'] def test_raw_load_pure(self): X = lltype.Struct('X', ('foo', lltype.Signed)) @@ -312,7 +311,7 @@ lltype.Signed, llmemory.cast_ptr_to_adr(x1), 0, True) res = self.interpret_inevitable(f1, []) - assert res is None + assert res == [] def test_threadlocal(self): from rpython.rlib.rthread import ThreadLocalField @@ -330,4 +329,84 @@ #assert x == 42 res = self.interpret_inevitable(f1, []) - assert res is None + assert res == [] + + + + def test_only_one_inev(self): + py.test.skip("not yet") + X = lltype.Struct('X', ('foo', lltype.Signed)) + x1 = lltype.malloc(X, immortal=True) + x1.foo = 42 + + def f1(): + r = 0 + r += x1.foo + r += x1.foo + return r + + res = self.interpret_inevitable(f1, []) + assert res == ['getfield'] + + def test_only_one_inev2(self): + py.test.skip("not yet") + X = lltype.Struct('X', ('foo', lltype.Signed)) + x1 = lltype.malloc(X, immortal=True) + x1.foo = 42 + + def f1(i): + r = 0 + if i: + r += x1.foo + r += x1.foo + return r + + res = self.interpret_inevitable(f1, [1]) + assert res == ['getfield'] + + + def test_not_for_local_raw(self): + py.test.skip("not yet") + X = lltype.Struct('X', ('foo', lltype.Signed)) + + def f1(i): + x1 = lltype.malloc(X, flavor='raw') + x1.foo = 42 + r = x1.foo + lltype.free(x1, flavor='raw') + return r + + res = self.interpret_inevitable(f1, [1]) + assert res == [] + + + def test_for_unknown_raw(self): + py.test.skip("not yet") + X = lltype.Struct('X', ('foo', lltype.Signed)) + + def f1(i): + x1 = lltype.malloc(X, flavor='raw') + x1.foo = 42 + r = x1.foo + if i: + lltype.free(x1, flavor='raw') + return r + + res = self.interpret_inevitable(f1, [1]) + assert res == ['setfield', 'getfield'] + + + def test_local_raw_in_same_transaction(self): + py.test.skip("not yet") + X = lltype.Struct('X', ('foo', lltype.Signed)) + + def f1(i): + x1 = lltype.malloc(X, flavor='raw') + x1.foo = 42 + r = x1.foo + func() # gil-release, non-gil-release, random-gc-effects???? + lltype.free(x1, flavor='raw') + return r + + res = self.interpret_inevitable(f1, [1]) + assert res == [] From noreply at buildbot.pypy.org Thu Mar 12 18:54:12 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 12 Mar 2015 18:54:12 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: Motivation to write a blog post Message-ID: <20150312175412.E35A01C0557@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r5511:1e0e3a4e555f Date: 2015-03-12 18:54 +0100 http://bitbucket.org/pypy/extradoc/changeset/1e0e3a4e555f/ Log: Motivation to write a blog post diff --git a/blog/draft/stm-mar2015.txt b/blog/draft/stm-mar2015.txt new file mode 100644 --- /dev/null +++ b/blog/draft/stm-mar2015.txt @@ -0,0 +1,10 @@ +============= +Status of STM +============= + +- stmgc-c7 reasonably good results, now porting to stmgc-c8 + +- stmgc-c7 results (with download link), explain the new + features + +- stmgc-c8 in two words From noreply at buildbot.pypy.org Fri Mar 13 08:47:56 2015 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 13 Mar 2015 08:47:56 +0100 (CET) Subject: [pypy-commit] pypy optresult: hack until we pass test_p123_nested Message-ID: <20150313074756.33E391C0173@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult Changeset: r76359:49727a67d83a Date: 2015-03-12 12:04 +0200 http://bitbucket.org/pypy/pypy/changeset/49727a67d83a/ Log: hack until we pass test_p123_nested diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py --- a/rpython/jit/metainterp/optimizeopt/info.py +++ b/rpython/jit/metainterp/optimizeopt/info.py @@ -20,7 +20,6 @@ INFO_UNKNOWN = 2 FLAG_VIRTUAL = 1 -FLAG_DIRTY = 2 class AbstractInfo(AbstractValue): is_info_class = True @@ -68,8 +67,7 @@ op.set_forwarded(newop) newop.set_forwarded(self) self.flags &= ~FLAG_VIRTUAL # clean the virtual flag - if self._force_elements(newop, optforce): - self.flags |= FLAG_DIRTY + self._force_elements(newop, optforce) return newop return op @@ -83,14 +81,13 @@ self._fields = [None] * len(descr.all_fielddescrs) def clear_cache(self): - assert self.flags & (FLAG_DIRTY | FLAG_VIRTUAL) == FLAG_DIRTY + assert self.flags & FLAG_VIRTUAL == 0 self.flags = 0 self._fields = [None] * len(self._fields) def setfield(self, descr, op, optheap=None): if not self.is_virtual(): - if self.flags & FLAG_DIRTY == 0: - self.flags |= FLAG_DIRTY + if self._fields[descr.index] is not None: assert optheap is not None # we should only call it with virtuals without optheap optheap.register_dirty_field(descr, self) @@ -111,8 +108,7 @@ setfieldop = ResOperation(rop.SETFIELD_GC, [op, subbox], descr=flddescr) optforce.emit_operation(setfieldop) - if optforce.optheap: - optforce.optheap.register_dirty_field(flddescr, self) + optforce.optheap.register_dirty_field(flddescr, self) count += 1 return count From noreply at buildbot.pypy.org Fri Mar 13 08:47:57 2015 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 13 Mar 2015 08:47:57 +0100 (CET) Subject: [pypy-commit] pypy optresult: implement heapcache on consts Message-ID: <20150313074757.7CDD91C0173@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult Changeset: r76360:719110b9c792 Date: 2015-03-13 09:47 +0200 http://bitbucket.org/pypy/pypy/changeset/719110b9c792/ Log: implement heapcache on consts diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -202,6 +202,8 @@ def setup(self): self.optimizer.optheap = self + # mapping const value -> info corresponding to it's heap cache + self.const_infos = self.optimizer.cpu.ts.new_ref_dict() def force_at_end_of_preamble(self): self.cached_dict_reads.clear() @@ -483,7 +485,7 @@ def optimize_GETFIELD_GC_I(self, op): opinfo = self.ensure_ptr_info_arg0(op) - fld = opinfo.getfield(op.getdescr()) + fld = opinfo.getfield(op.getdescr(), self) if fld is not None: self.make_equal_to(op, fld) return diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py --- a/rpython/jit/metainterp/optimizeopt/info.py +++ b/rpython/jit/metainterp/optimizeopt/info.py @@ -93,7 +93,7 @@ optheap.register_dirty_field(descr, self) self._fields[descr.index] = op - def getfield(self, descr): + def getfield(self, descr, optheap=None): return self._fields[descr.index] def _force_elements(self, op, optforce): @@ -213,6 +213,23 @@ def __init__(self, const): self._const = const + def _get_info(self, descr, optheap): + ref = self._const.getref_base() + info = optheap.const_infos.get(ref, None) + if info is None: + info = StructPtrInfo() + info.init_fields(descr.parent_descr) + optheap.const_infos[ref] = info + return info + + def getfield(self, descr, optheap=None): + info = self._get_info(descr, optheap) + return info.getfield(descr) + + def setfield(self, descr, op, optheap): + info = self._get_info(descr, optheap) + info.setfield(descr, op, optheap) + def is_null(self): return not bool(self._const.getref_base()) diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -588,22 +588,13 @@ if is_object: opinfo = info.InstancePtrInfo() else: - xxx + opinfo = info.StructPtrInfo() opinfo.init_fields(op.getdescr().parent_descr) else: yyy arg0.set_forwarded(opinfo) return opinfo - def make_ptr_info(self, op, mode): - op = self.get_box_replacement(op) - if op.is_constant(): - return info.ConstPtrInfo(op) - opinfo = op.get_forwarded() - if isinstance(opinfo, info.AbstractVirtualPtrInfo): - return opinfo - xxx - def new_const(self, fieldofs): if fieldofs.is_pointer_field(): return self.cpu.ts.CONST_NULL diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py --- a/rpython/jit/metainterp/resume.py +++ b/rpython/jit/metainterp/resume.py @@ -221,8 +221,7 @@ tagged = liveboxes[box] else: if box.type == 'r': - xxx - info = optimizer.getinfo(box, create=False) + info = optimizer.getptrinfo(box) is_virtual = (info is not None and info.is_virtual()) else: is_virtual = False From noreply at buildbot.pypy.org Fri Mar 13 09:44:48 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 13 Mar 2015 09:44:48 +0100 (CET) Subject: [pypy-commit] cffi default: Py3 syntax Message-ID: <20150313084448.C9BA41C0173@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1667:16bbcade234c Date: 2015-03-13 09:43 +0100 http://bitbucket.org/cffi/cffi/changeset/16bbcade234c/ Log: Py3 syntax diff --git a/testing/test_zintegration.py b/testing/test_zintegration.py --- a/testing/test_zintegration.py +++ b/testing/test_zintegration.py @@ -26,7 +26,10 @@ if e.errno in (errno.ENOTDIR, errno.EINVAL): shutil.copy(src, dst) else: - print 'got errno',e.errno,'not',errno.ENOTDIR + print('got errno') + print(e.errno) + print('not') + print(errno.ENOTDIR) raise site_packages = None From noreply at buildbot.pypy.org Fri Mar 13 09:44:47 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 13 Mar 2015 09:44:47 +0100 (CET) Subject: [pypy-commit] cffi default: Forbid from_buffer(x) even if x is a buffer or memoryview that itself Message-ID: <20150313084447.BEE961C0173@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1666:bec74625ec6b Date: 2015-03-13 09:12 +0100 http://bitbucket.org/cffi/cffi/changeset/bec74625ec6b/ Log: Forbid from_buffer(x) even if x is a buffer or memoryview that itself refers to a string or bytearray object diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -5176,6 +5176,33 @@ return 0; } +static int invalid_input_buffer_type(PyObject *x) +{ + if (PyBuffer_Check(x)) { + /* XXX fish fish fish in an inofficial way */ + typedef struct { + PyObject_HEAD + PyObject *b_base; + } _my_PyBufferObject; + + _my_PyBufferObject *b = (_my_PyBufferObject *)x; + x = b->b_base; + if (x == NULL) + return 0; + } + else if (PyMemoryView_Check(x)) { + x = PyMemoryView_GET_BASE(x); + if (x == NULL) + return 0; + } + + if (PyBytes_Check(x) || PyUnicode_Check(x)) + return 1; + if (PyByteArray_Check(x)) /* <= this one here for PyPy compatibility */ + return 1; + return 0; +} + static PyObject *b_from_buffer(PyObject *self, PyObject *args) { CTypeDescrObject *ct; @@ -5191,8 +5218,7 @@ return NULL; } - if (PyBytes_Check(x) || PyText_Check(x) || - PyByteArray_Check(x) /* <= this one here for PyPy compatibility */ ) { + if (invalid_input_buffer_type(x)) { PyErr_SetString(PyExc_TypeError, "from_buffer() cannot return the address of the " "raw string within a "STR_OR_BYTES" or unicode or " diff --git a/c/test_c.py b/c/test_c.py --- a/c/test_c.py +++ b/c/test_c.py @@ -3258,6 +3258,20 @@ cast(p, c)[1] += 500 assert list(a) == [10000, 20500, 30000] +def test_from_buffer_not_str_unicode_bytearray(): + from __builtin__ import buffer + BChar = new_primitive_type("char") + BCharP = new_pointer_type(BChar) + BCharA = new_array_type(BCharP, None) + py.test.raises(TypeError, from_buffer, BCharA, b"foo") + py.test.raises(TypeError, from_buffer, BCharA, u"foo") + py.test.raises(TypeError, from_buffer, BCharA, bytearray(b"foo")) + py.test.raises(TypeError, from_buffer, BCharA, buffer(b"foo")) + py.test.raises(TypeError, from_buffer, BCharA, buffer(u"foo")) + py.test.raises(TypeError, from_buffer, BCharA, buffer(bytearray(b"foo"))) + py.test.raises(TypeError, from_buffer, BCharA, memoryview(b"foo")) + py.test.raises(TypeError, from_buffer, BCharA, memoryview(bytearray(b"fo"))) + def test_from_buffer_more_cases(): try: from _cffi_backend import _testbuff From noreply at buildbot.pypy.org Fri Mar 13 09:44:46 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 13 Mar 2015 09:44:46 +0100 (CET) Subject: [pypy-commit] cffi default: Test and fix for from_buffer() receiving read-only buffer objects as Message-ID: <20150313084446.9AF431C0173@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1665:18a20248b9f7 Date: 2015-03-13 09:02 +0100 http://bitbucket.org/cffi/cffi/changeset/18a20248b9f7/ Log: Test and fix for from_buffer() receiving read-only buffer objects as argument. diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -5142,10 +5142,13 @@ 'view->obj'. */ PyBufferProcs *pb = x->ob_type->tp_as_buffer; if (pb && !pb->bf_releasebuffer) { - /* try all three in some vaguely sensible order */ - readbufferproc proc = (readbufferproc)pb->bf_getwritebuffer; + /* we used to try all three in some vaguely sensible order, + i.e. first the write. But trying to call the write on a + read-only buffer fails with TypeError. So we use a less- + sensible order now. See test_from_buffer_more_cases. */ + readbufferproc proc = (readbufferproc)pb->bf_getreadbuffer; if (!proc) proc = (readbufferproc)pb->bf_getcharbuffer; - if (!proc) proc = (readbufferproc)pb->bf_getreadbuffer; + if (!proc) proc = (readbufferproc)pb->bf_getwritebuffer; if (proc && pb->bf_getsegcount) { if ((*pb->bf_getsegcount)(x, NULL) != 1) { PyErr_SetString(PyExc_TypeError, @@ -5441,6 +5444,63 @@ return PyLong_FromVoidPtr(f); } +static Py_ssize_t _test_segcountproc(PyObject *o, Py_ssize_t *ignored) +{ + return 1; +} +static Py_ssize_t _test_getreadbuf(PyObject *o, Py_ssize_t i, void **r) +{ + static char buf[] = "RDB"; + *r = buf; + return 3; +} +static Py_ssize_t _test_getwritebuf(PyObject *o, Py_ssize_t i, void **r) +{ + static char buf[] = "WRB"; + *r = buf; + return 3; +} +static Py_ssize_t _test_getcharbuf(PyObject *o, Py_ssize_t i, char **r) +{ + static char buf[] = "CHB"; + *r = buf; + return 3; +} +static int _test_getbuf(PyObject *self, Py_buffer *view, int flags) +{ + static char buf[] = "GTB"; + return PyBuffer_FillInfo(view, self, buf, 3, /*readonly=*/0, flags); +} +static int _test_getbuf_ro(PyObject *self, Py_buffer *view, int flags) +{ + static char buf[] = "ROB"; + return PyBuffer_FillInfo(view, self, buf, 3, /*readonly=*/1, flags); +} + + +static PyObject *b__testbuff(PyObject *self, PyObject *args) +{ + /* for testing only */ + int methods; + PyTypeObject *obj; + if (!PyArg_ParseTuple(args, "O!i|_testbuff", &PyType_Type, &obj, &methods)) + return NULL; + + assert(obj->tp_as_buffer != NULL); + + obj->tp_as_buffer->bf_getsegcount = &_test_segcountproc; + obj->tp_flags |= Py_TPFLAGS_HAVE_GETCHARBUFFER; + obj->tp_flags |= Py_TPFLAGS_HAVE_NEWBUFFER; + if (methods & 1) obj->tp_as_buffer->bf_getreadbuffer = &_test_getreadbuf; + if (methods & 2) obj->tp_as_buffer->bf_getwritebuffer = &_test_getwritebuf; + if (methods & 4) obj->tp_as_buffer->bf_getcharbuffer = &_test_getcharbuf; + if (methods & 8) obj->tp_as_buffer->bf_getbuffer = &_test_getbuf; + if (methods & 16) obj->tp_as_buffer->bf_getbuffer = &_test_getbuf_ro; + + Py_INCREF(Py_None); + return Py_None; +} + static PyMethodDef FFIBackendMethods[] = { {"load_library", b_load_library, METH_VARARGS}, {"new_primitive_type", b_new_primitive_type, METH_VARARGS}, @@ -5473,6 +5533,7 @@ #endif {"_get_types", b__get_types, METH_NOARGS}, {"_testfunc", b__testfunc, METH_VARARGS}, + {"_testbuff", b__testbuff, METH_VARARGS}, {NULL, NULL} /* Sentinel */ }; diff --git a/c/test_c.py b/c/test_c.py --- a/c/test_c.py +++ b/c/test_c.py @@ -3258,6 +3258,56 @@ cast(p, c)[1] += 500 assert list(a) == [10000, 20500, 30000] +def test_from_buffer_more_cases(): + try: + from _cffi_backend import _testbuff + except ImportError: + py.test.skip("not for pypy") + BChar = new_primitive_type("char") + BCharP = new_pointer_type(BChar) + BCharA = new_array_type(BCharP, None) + # + def check1(bufobj, expected): + c = from_buffer(BCharA, bufobj) + assert typeof(c) is BCharA + assert list(c) == list(expected) + # + def check(methods, expected, expeted_for_memoryview=None): + from __builtin__ import buffer, memoryview + class X(object): + pass + _testbuff(X, methods) + bufobj = X() + check1(bufobj, expected) + try: + bufobjb = buffer(bufobj) + except TypeError: + pass + else: + check1(bufobjb, expected) + try: + bufobjm = memoryview(bufobj) + except TypeError: + pass + else: + check1(bufobjm, expeted_for_memoryview or expected) + # + check(1, "RDB") + check(2, "WRB") + check(4, "CHB") + check(8, "GTB") + check(16, "ROB") + # + check(1 | 2, "RDB") + check(1 | 4, "RDB") + check(2 | 4, "CHB") + check(1 | 8, "RDB", "GTB") + check(1 | 16, "RDB", "ROB") + check(2 | 8, "WRB", "GTB") + check(2 | 16, "WRB", "ROB") + check(4 | 8, "CHB", "GTB") + check(4 | 16, "CHB", "ROB") + def test_version(): # this test is here mostly for PyPy assert __version__ == "0.9.1" From noreply at buildbot.pypy.org Fri Mar 13 09:44:49 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 13 Mar 2015 09:44:49 +0100 (CET) Subject: [pypy-commit] cffi default: Support for Python 3 too Message-ID: <20150313084449.CEBB31C0173@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1668:47d41a504ce6 Date: 2015-03-13 09:43 +0100 http://bitbucket.org/cffi/cffi/changeset/47d41a504ce6/ Log: Support for Python 3 too diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -5178,6 +5178,7 @@ static int invalid_input_buffer_type(PyObject *x) { +#if PY_MAJOR_VERSION < 3 if (PyBuffer_Check(x)) { /* XXX fish fish fish in an inofficial way */ typedef struct { @@ -5190,11 +5191,17 @@ if (x == NULL) return 0; } - else if (PyMemoryView_Check(x)) { + else +#endif +#if PY_MAJOR_VERSION > 2 || PY_MINOR_VERSION > 6 + if (PyMemoryView_Check(x)) { x = PyMemoryView_GET_BASE(x); if (x == NULL) return 0; } + else +#endif + ; if (PyBytes_Check(x) || PyUnicode_Check(x)) return 1; @@ -5470,6 +5477,7 @@ return PyLong_FromVoidPtr(f); } +#if PY_MAJOR_VERSION < 3 static Py_ssize_t _test_segcountproc(PyObject *o, Py_ssize_t *ignored) { return 1; @@ -5492,6 +5500,7 @@ *r = buf; return 3; } +#endif static int _test_getbuf(PyObject *self, Py_buffer *view, int flags) { static char buf[] = "GTB"; @@ -5514,12 +5523,14 @@ assert(obj->tp_as_buffer != NULL); +#if PY_MAJOR_VERSION < 3 obj->tp_as_buffer->bf_getsegcount = &_test_segcountproc; obj->tp_flags |= Py_TPFLAGS_HAVE_GETCHARBUFFER; obj->tp_flags |= Py_TPFLAGS_HAVE_NEWBUFFER; if (methods & 1) obj->tp_as_buffer->bf_getreadbuffer = &_test_getreadbuf; if (methods & 2) obj->tp_as_buffer->bf_getwritebuffer = &_test_getwritebuf; if (methods & 4) obj->tp_as_buffer->bf_getcharbuffer = &_test_getcharbuf; +#endif if (methods & 8) obj->tp_as_buffer->bf_getbuffer = &_test_getbuf; if (methods & 16) obj->tp_as_buffer->bf_getbuffer = &_test_getbuf_ro; diff --git a/c/test_c.py b/c/test_c.py --- a/c/test_c.py +++ b/c/test_c.py @@ -3259,18 +3259,29 @@ assert list(a) == [10000, 20500, 30000] def test_from_buffer_not_str_unicode_bytearray(): - from __builtin__ import buffer BChar = new_primitive_type("char") BCharP = new_pointer_type(BChar) BCharA = new_array_type(BCharP, None) py.test.raises(TypeError, from_buffer, BCharA, b"foo") py.test.raises(TypeError, from_buffer, BCharA, u"foo") py.test.raises(TypeError, from_buffer, BCharA, bytearray(b"foo")) - py.test.raises(TypeError, from_buffer, BCharA, buffer(b"foo")) - py.test.raises(TypeError, from_buffer, BCharA, buffer(u"foo")) - py.test.raises(TypeError, from_buffer, BCharA, buffer(bytearray(b"foo"))) - py.test.raises(TypeError, from_buffer, BCharA, memoryview(b"foo")) - py.test.raises(TypeError, from_buffer, BCharA, memoryview(bytearray(b"fo"))) + try: + from __builtin__ import buffer + except ImportError: + pass + else: + py.test.raises(TypeError, from_buffer, BCharA, buffer(b"foo")) + py.test.raises(TypeError, from_buffer, BCharA, buffer(u"foo")) + py.test.raises(TypeError, from_buffer, BCharA, + buffer(bytearray(b"foo"))) + try: + from __builtin__ import memoryview + except ImportError: + pass + else: + py.test.raises(TypeError, from_buffer, BCharA, memoryview(b"foo")) + py.test.raises(TypeError, from_buffer, BCharA, + memoryview(bytearray(b"foo"))) def test_from_buffer_more_cases(): try: @@ -3284,27 +3295,34 @@ def check1(bufobj, expected): c = from_buffer(BCharA, bufobj) assert typeof(c) is BCharA + if sys.version_info >= (3,): + expected = [bytes(c, "ascii") for c in expected] assert list(c) == list(expected) # - def check(methods, expected, expeted_for_memoryview=None): - from __builtin__ import buffer, memoryview + def check(methods, expected, expected_for_memoryview=None): + if sys.version_info >= (3,): + if methods <= 7: + return + if expected_for_memoryview is not None: + expected = expected_for_memoryview class X(object): pass _testbuff(X, methods) bufobj = X() check1(bufobj, expected) try: + from __builtin__ import buffer bufobjb = buffer(bufobj) - except TypeError: + except (TypeError, ImportError): pass else: check1(bufobjb, expected) try: bufobjm = memoryview(bufobj) - except TypeError: + except (TypeError, NameError): pass else: - check1(bufobjm, expeted_for_memoryview or expected) + check1(bufobjm, expected_for_memoryview or expected) # check(1, "RDB") check(2, "WRB") @@ -3312,14 +3330,14 @@ check(8, "GTB") check(16, "ROB") # - check(1 | 2, "RDB") - check(1 | 4, "RDB") - check(2 | 4, "CHB") - check(1 | 8, "RDB", "GTB") + check(1 | 2, "RDB") + check(1 | 4, "RDB") + check(2 | 4, "CHB") + check(1 | 8, "RDB", "GTB") check(1 | 16, "RDB", "ROB") - check(2 | 8, "WRB", "GTB") + check(2 | 8, "WRB", "GTB") check(2 | 16, "WRB", "ROB") - check(4 | 8, "CHB", "GTB") + check(4 | 8, "CHB", "GTB") check(4 | 16, "CHB", "ROB") def test_version(): From noreply at buildbot.pypy.org Fri Mar 13 09:59:40 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Fri, 13 Mar 2015 09:59:40 +0100 (CET) Subject: [pypy-commit] stmgc default: missing check for accessibility of the start of the obj to check a gcflag Message-ID: <20150313085940.A50591C1DC6@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1733:7b58657a2575 Date: 2015-03-13 10:00 +0100 http://bitbucket.org/pypy/stmgc/changeset/7b58657a2575/ Log: missing check for accessibility of the start of the obj to check a gcflag diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -48,6 +48,7 @@ assert(IMPLY(from_segnum >= 0, get_priv_segment(from_segnum)->modification_lock)); assert(STM_PSEGMENT->modification_lock); + long my_segnum = STM_SEGMENT->segment_num; DEBUG_EXPECT_SEGFAULT(false); for (; undo < end; undo++) { if (undo->type == TYPE_POSITION_MARKER) @@ -57,18 +58,19 @@ uintptr_t current_page_num = ((uintptr_t)oslice) / 4096; if (pagenum == -1) { - if (get_page_status_in(STM_SEGMENT->segment_num, - current_page_num) == PAGE_NO_ACCESS) + if (get_page_status_in(my_segnum, current_page_num) == PAGE_NO_ACCESS) continue; - } - else { - if (current_page_num != pagenum) - continue; + } else if (pagenum != current_page_num) { + continue; } - if (from_segnum == -2 && _stm_was_read(obj) && (obj->stm_flags & GCFLAG_WB_EXECUTED)) { + if (from_segnum == -2 + && _stm_was_read(obj) + && (get_page_status_in(my_segnum, (uintptr_t)obj / 4096) == PAGE_ACCESSIBLE) + && (obj->stm_flags & GCFLAG_WB_EXECUTED)) { /* called from stm_validate(): > if not was_read(), we certainly didn't modify + > if obj->stm_flags is not accessible, WB_EXECUTED cannot be set > if not WB_EXECUTED, we may have read from the obj in a different page but did not modify it (should not occur right now, but future proof!) only the WB_EXECUTED alone is not enough, since we may have imported from a @@ -79,8 +81,7 @@ /* XXX: if the next assert is always true, we should never get a segfault in this function at all. So the DEBUG_EXPECT_SEGFAULT is correct. */ - assert((get_page_status_in(STM_SEGMENT->segment_num, - current_page_num) != PAGE_NO_ACCESS)); + assert((get_page_status_in(my_segnum, current_page_num) != PAGE_NO_ACCESS)); /* dprintf(("import slice seg=%d obj=%p off=%lu sz=%d pg=%lu\n", */ /* from_segnum, obj, SLICE_OFFSET(undo->slice), */ @@ -95,7 +96,8 @@ if (src_segment_base == NULL && SLICE_OFFSET(undo->slice) == 0) { /* check that restored obj doesn't have WB_EXECUTED */ - assert(!(obj->stm_flags & GCFLAG_WB_EXECUTED)); + assert((get_page_status_in(my_segnum, (uintptr_t)obj / 4096) == PAGE_NO_ACCESS) + || !(obj->stm_flags & GCFLAG_WB_EXECUTED)); } } DEBUG_EXPECT_SEGFAULT(true); From noreply at buildbot.pypy.org Fri Mar 13 10:02:43 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 13 Mar 2015 10:02:43 +0100 (CET) Subject: [pypy-commit] cffi release-0.9: hg merge default Message-ID: <20150313090243.C19B41C1DC6@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: release-0.9 Changeset: r1669:7e52019c7511 Date: 2015-03-13 10:00 +0100 http://bitbucket.org/cffi/cffi/changeset/7e52019c7511/ Log: hg merge default diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -5142,10 +5142,13 @@ 'view->obj'. */ PyBufferProcs *pb = x->ob_type->tp_as_buffer; if (pb && !pb->bf_releasebuffer) { - /* try all three in some vaguely sensible order */ - readbufferproc proc = (readbufferproc)pb->bf_getwritebuffer; + /* we used to try all three in some vaguely sensible order, + i.e. first the write. But trying to call the write on a + read-only buffer fails with TypeError. So we use a less- + sensible order now. See test_from_buffer_more_cases. */ + readbufferproc proc = (readbufferproc)pb->bf_getreadbuffer; if (!proc) proc = (readbufferproc)pb->bf_getcharbuffer; - if (!proc) proc = (readbufferproc)pb->bf_getreadbuffer; + if (!proc) proc = (readbufferproc)pb->bf_getwritebuffer; if (proc && pb->bf_getsegcount) { if ((*pb->bf_getsegcount)(x, NULL) != 1) { PyErr_SetString(PyExc_TypeError, @@ -5173,6 +5176,40 @@ return 0; } +static int invalid_input_buffer_type(PyObject *x) +{ +#if PY_MAJOR_VERSION < 3 + if (PyBuffer_Check(x)) { + /* XXX fish fish fish in an inofficial way */ + typedef struct { + PyObject_HEAD + PyObject *b_base; + } _my_PyBufferObject; + + _my_PyBufferObject *b = (_my_PyBufferObject *)x; + x = b->b_base; + if (x == NULL) + return 0; + } + else +#endif +#if PY_MAJOR_VERSION > 2 || PY_MINOR_VERSION > 6 + if (PyMemoryView_Check(x)) { + x = PyMemoryView_GET_BASE(x); + if (x == NULL) + return 0; + } + else +#endif + ; + + if (PyBytes_Check(x) || PyUnicode_Check(x)) + return 1; + if (PyByteArray_Check(x)) /* <= this one here for PyPy compatibility */ + return 1; + return 0; +} + static PyObject *b_from_buffer(PyObject *self, PyObject *args) { CTypeDescrObject *ct; @@ -5188,8 +5225,7 @@ return NULL; } - if (PyBytes_Check(x) || PyText_Check(x) || - PyByteArray_Check(x) /* <= this one here for PyPy compatibility */ ) { + if (invalid_input_buffer_type(x)) { PyErr_SetString(PyExc_TypeError, "from_buffer() cannot return the address of the " "raw string within a "STR_OR_BYTES" or unicode or " @@ -5441,6 +5477,67 @@ return PyLong_FromVoidPtr(f); } +#if PY_MAJOR_VERSION < 3 +static Py_ssize_t _test_segcountproc(PyObject *o, Py_ssize_t *ignored) +{ + return 1; +} +static Py_ssize_t _test_getreadbuf(PyObject *o, Py_ssize_t i, void **r) +{ + static char buf[] = "RDB"; + *r = buf; + return 3; +} +static Py_ssize_t _test_getwritebuf(PyObject *o, Py_ssize_t i, void **r) +{ + static char buf[] = "WRB"; + *r = buf; + return 3; +} +static Py_ssize_t _test_getcharbuf(PyObject *o, Py_ssize_t i, char **r) +{ + static char buf[] = "CHB"; + *r = buf; + return 3; +} +#endif +static int _test_getbuf(PyObject *self, Py_buffer *view, int flags) +{ + static char buf[] = "GTB"; + return PyBuffer_FillInfo(view, self, buf, 3, /*readonly=*/0, flags); +} +static int _test_getbuf_ro(PyObject *self, Py_buffer *view, int flags) +{ + static char buf[] = "ROB"; + return PyBuffer_FillInfo(view, self, buf, 3, /*readonly=*/1, flags); +} + + +static PyObject *b__testbuff(PyObject *self, PyObject *args) +{ + /* for testing only */ + int methods; + PyTypeObject *obj; + if (!PyArg_ParseTuple(args, "O!i|_testbuff", &PyType_Type, &obj, &methods)) + return NULL; + + assert(obj->tp_as_buffer != NULL); + +#if PY_MAJOR_VERSION < 3 + obj->tp_as_buffer->bf_getsegcount = &_test_segcountproc; + obj->tp_flags |= Py_TPFLAGS_HAVE_GETCHARBUFFER; + obj->tp_flags |= Py_TPFLAGS_HAVE_NEWBUFFER; + if (methods & 1) obj->tp_as_buffer->bf_getreadbuffer = &_test_getreadbuf; + if (methods & 2) obj->tp_as_buffer->bf_getwritebuffer = &_test_getwritebuf; + if (methods & 4) obj->tp_as_buffer->bf_getcharbuffer = &_test_getcharbuf; +#endif + if (methods & 8) obj->tp_as_buffer->bf_getbuffer = &_test_getbuf; + if (methods & 16) obj->tp_as_buffer->bf_getbuffer = &_test_getbuf_ro; + + Py_INCREF(Py_None); + return Py_None; +} + static PyMethodDef FFIBackendMethods[] = { {"load_library", b_load_library, METH_VARARGS}, {"new_primitive_type", b_new_primitive_type, METH_VARARGS}, @@ -5473,6 +5570,7 @@ #endif {"_get_types", b__get_types, METH_NOARGS}, {"_testfunc", b__testfunc, METH_VARARGS}, + {"_testbuff", b__testbuff, METH_VARARGS}, {NULL, NULL} /* Sentinel */ }; diff --git a/c/test_c.py b/c/test_c.py --- a/c/test_c.py +++ b/c/test_c.py @@ -3258,6 +3258,88 @@ cast(p, c)[1] += 500 assert list(a) == [10000, 20500, 30000] +def test_from_buffer_not_str_unicode_bytearray(): + BChar = new_primitive_type("char") + BCharP = new_pointer_type(BChar) + BCharA = new_array_type(BCharP, None) + py.test.raises(TypeError, from_buffer, BCharA, b"foo") + py.test.raises(TypeError, from_buffer, BCharA, u"foo") + py.test.raises(TypeError, from_buffer, BCharA, bytearray(b"foo")) + try: + from __builtin__ import buffer + except ImportError: + pass + else: + py.test.raises(TypeError, from_buffer, BCharA, buffer(b"foo")) + py.test.raises(TypeError, from_buffer, BCharA, buffer(u"foo")) + py.test.raises(TypeError, from_buffer, BCharA, + buffer(bytearray(b"foo"))) + try: + from __builtin__ import memoryview + except ImportError: + pass + else: + py.test.raises(TypeError, from_buffer, BCharA, memoryview(b"foo")) + py.test.raises(TypeError, from_buffer, BCharA, + memoryview(bytearray(b"foo"))) + +def test_from_buffer_more_cases(): + try: + from _cffi_backend import _testbuff + except ImportError: + py.test.skip("not for pypy") + BChar = new_primitive_type("char") + BCharP = new_pointer_type(BChar) + BCharA = new_array_type(BCharP, None) + # + def check1(bufobj, expected): + c = from_buffer(BCharA, bufobj) + assert typeof(c) is BCharA + if sys.version_info >= (3,): + expected = [bytes(c, "ascii") for c in expected] + assert list(c) == list(expected) + # + def check(methods, expected, expected_for_memoryview=None): + if sys.version_info >= (3,): + if methods <= 7: + return + if expected_for_memoryview is not None: + expected = expected_for_memoryview + class X(object): + pass + _testbuff(X, methods) + bufobj = X() + check1(bufobj, expected) + try: + from __builtin__ import buffer + bufobjb = buffer(bufobj) + except (TypeError, ImportError): + pass + else: + check1(bufobjb, expected) + try: + bufobjm = memoryview(bufobj) + except (TypeError, NameError): + pass + else: + check1(bufobjm, expected_for_memoryview or expected) + # + check(1, "RDB") + check(2, "WRB") + check(4, "CHB") + check(8, "GTB") + check(16, "ROB") + # + check(1 | 2, "RDB") + check(1 | 4, "RDB") + check(2 | 4, "CHB") + check(1 | 8, "RDB", "GTB") + check(1 | 16, "RDB", "ROB") + check(2 | 8, "WRB", "GTB") + check(2 | 16, "WRB", "ROB") + check(4 | 8, "CHB", "GTB") + check(4 | 16, "CHB", "ROB") + def test_version(): # this test is here mostly for PyPy assert __version__ == "0.9.1" diff --git a/testing/test_zdistutils.py b/testing/test_zdistutils.py --- a/testing/test_zdistutils.py +++ b/testing/test_zdistutils.py @@ -164,7 +164,8 @@ assert lib.sin(12.3) == math.sin(12.3) v = ffi.verifier ext = v.get_extension() - assert 'distutils.extension.Extension' in str(ext.__class__) + assert 'distutils.extension.Extension' in str(ext.__class__) or \ + 'setuptools.extension.Extension' in str(ext.__class__) assert ext.sources == [maybe_relative_path(v.sourcefilename)] assert ext.name == v.get_module_name() assert ext.define_macros == [('TEST_EXTENSION_OBJECT', '1')] @@ -193,7 +194,8 @@ assert lib.test1eoes(7.0) == 42.0 v = ffi.verifier ext = v.get_extension() - assert 'distutils.extension.Extension' in str(ext.__class__) + assert 'distutils.extension.Extension' in str(ext.__class__) or \ + 'setuptools.extension.Extension' in str(ext.__class__) assert ext.sources == [maybe_relative_path(v.sourcefilename), extra_source] assert ext.name == v.get_module_name() diff --git a/testing/test_zintegration.py b/testing/test_zintegration.py --- a/testing/test_zintegration.py +++ b/testing/test_zintegration.py @@ -3,6 +3,9 @@ import subprocess from testing.udir import udir +if sys.platform == 'win32': + py.test.skip('snippets do not run on win32') + def create_venv(name): tmpdir = udir.join(name) try: @@ -12,6 +15,23 @@ except OSError as e: py.test.skip("Cannot execute virtualenv: %s" % (e,)) + try: + deepcopy = os.symlink + except: + import shutil, errno + def deepcopy(src, dst): + try: + shutil.copytree(src, dst) + except OSError as e: + if e.errno in (errno.ENOTDIR, errno.EINVAL): + shutil.copy(src, dst) + else: + print('got errno') + print(e.errno) + print('not') + print(errno.ENOTDIR) + raise + site_packages = None for dirpath, dirnames, filenames in os.walk(str(tmpdir)): if os.path.basename(dirpath) == 'site-packages': @@ -31,7 +51,7 @@ modules += ('ply',) # needed for older versions of pycparser for module in modules: target = imp.find_module(module)[1] - os.symlink(target, os.path.join(site_packages, + deepcopy(target, os.path.join(site_packages, os.path.basename(target))) return tmpdir @@ -50,7 +70,11 @@ python_f.write(py.code.Source(python_snippet)) try: os.chdir(str(SNIPPET_DIR.join(dirname))) - vp = str(venv_dir.join('bin/python')) + if os.name == 'nt': + bindir = 'Scripts' + else: + bindir = 'bin' + vp = str(venv_dir.join(bindir).join('python')) subprocess.check_call((vp, 'setup.py', 'clean')) subprocess.check_call((vp, 'setup.py', 'install')) subprocess.check_call((vp, str(python_f))) From noreply at buildbot.pypy.org Fri Mar 13 10:02:44 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 13 Mar 2015 10:02:44 +0100 (CET) Subject: [pypy-commit] cffi release-0.9: Prepare release 0.9.2 Message-ID: <20150313090244.E1C3D1C1DC6@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: release-0.9 Changeset: r1670:54ab50a4b22c Date: 2015-03-13 10:01 +0100 http://bitbucket.org/cffi/cffi/changeset/54ab50a4b22c/ Log: Prepare release 0.9.2 diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -5787,7 +5787,7 @@ if (v == NULL || PyModule_AddObject(m, "_C_API", v) < 0) INITERROR; - v = PyText_FromString("0.9.1"); + v = PyText_FromString("0.9.2"); if (v == NULL || PyModule_AddObject(m, "__version__", v) < 0) INITERROR; diff --git a/c/test_c.py b/c/test_c.py --- a/c/test_c.py +++ b/c/test_c.py @@ -3342,4 +3342,4 @@ def test_version(): # this test is here mostly for PyPy - assert __version__ == "0.9.1" + assert __version__ == "0.9.2" diff --git a/cffi/__init__.py b/cffi/__init__.py --- a/cffi/__init__.py +++ b/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "0.9.1" -__version_info__ = (0, 9, 1) +__version__ = "0.9.2" +__version_info__ = (0, 9, 2) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/doc/source/conf.py b/doc/source/conf.py --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -47,7 +47,7 @@ # The short X.Y version. version = '0.9' # The full version, including alpha/beta/rc tags. -release = '0.9.1' +release = '0.9.2' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/source/index.rst b/doc/source/index.rst --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -88,13 +88,13 @@ Download and Installation: -* http://pypi.python.org/packages/source/c/cffi/cffi-0.9.1.tar.gz +* http://pypi.python.org/packages/source/c/cffi/cffi-0.9.2.tar.gz - Or grab the most current version by following the instructions below. - - MD5: 8dbdf23c600845b75654024e434601ce + - MD5: ... - - SHA: 6a1b297ee7108803f54bb6333cf1e5630a193756 + - SHA: ... * Or get it from the `Bitbucket page`_: ``hg clone https://bitbucket.org/cffi/cffi`` diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -139,7 +139,7 @@ `Mailing list `_ """, - version='0.9.1', + version='0.9.2', packages=['cffi'], zip_safe=False, From noreply at buildbot.pypy.org Fri Mar 13 10:11:16 2015 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 13 Mar 2015 10:11:16 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: blog post Message-ID: <20150313091116.731671C00EF@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: extradoc Changeset: r5512:6c157c34aeb8 Date: 2015-03-13 11:11 +0200 http://bitbucket.org/pypy/extradoc/changeset/6c157c34aeb8/ Log: blog post diff --git a/blog/draft/dmlockhart-pydgin/arm-bar-plot.png b/blog/draft/dmlockhart-pydgin/arm-bar-plot.png new file mode 100644 index 0000000000000000000000000000000000000000..bf46f09b2ae546971ebe7f4d0286adf646af2101 GIT binary patch [cut] diff --git a/blog/draft/dmlockhart-pydgin/blog.rst b/blog/draft/dmlockhart-pydgin/blog.rst new file mode 100644 --- /dev/null +++ b/blog/draft/dmlockhart-pydgin/blog.rst @@ -0,0 +1,494 @@ +=============================================================================== +Pydgin: Using RPython to Generate Fast Instruction-Set Simulators +=============================================================================== + +**Note:** This is a guest blog post by Derek Lockhart and Berkin Ilbeyi from +Computer Systems Laboratory of Cornell University. + +In this blog post I'd like to describe some recent work on using the RPython +translation toolchain to generate fast instruction set simulators. +Our open-source framework, Pydgin [a]_, provides a domain-specific +language (DSL) embedded in Python for concisely describing instruction set +architectures [b]_ and then uses these descriptions to generate fast, +JIT-enabled simulators. +Pydgin will be presented at the *IEEE International Symposium on Performance +Analysis of Systems and Software (ISPASS)* and in this post we provide a +preview of that work. +In addition, we discuss some additional progress updates that occurred after +the publishing deadline and will not appear in the final paper [1]_. + +Our area of research expertise is computer architecture, which is perhaps an +unfamiliar topic for some readers of the PyPy blog. +Below we provide some brief background on hardware simulation in the field of +computer architecture, as well as some context as to why instruction set +simulators in particular are such an important tool. + +------------------------------------------------------------------------------- +Simulators: Designing Hardware with Software +------------------------------------------------------------------------------- + +For computer architects in both academia and industry, a key step in designing +new computational hardware (e.g., CPUs, GPUs, and mobile system-on-chips) is +simulation [c]_ of the target system. +While numerous models for simulation exist, three classes are particularly +important in hardware design. + +**Functional Level** models simulate the *behavior* of the target system. +These models are useful for creating a "golden" reference which can serve as an +executable specification or alternatively as an emulation platform for software +development. + +**Cycle Level** models aim to simulate both the *behavior* and the approximate +*timing* of a hardware component. +These models help computer architects explore design tradeoffs and quickly +determine things like how big caches should be, how many functional units are +needed to meet throughput targets, and how the addition of a custom accelerator +block may impact total system performance. + +**Register-Transfer Level** (RTL) models specify the *behavior*, *timing*, and +*resources* (e.g., registers, wires, logic gates) of a hardware component. +RTL models are bit-accurate hardware specifications typically written in a +hardware description language (HDL) such as Verilog or VHDL. +Once verified through extensive simulation, HDL specifications can be passed +into synthesis and place-and-route tools to estimate area/energy/timing or to +create FPGA or ASIC prototypes. + +An *instruction set simulator* (ISS) is a special kind of +*functional-level* model that simulates the behavior of a processor or +system-on-chip (SOC). ISSs serve an important role in hardware design +because they model the instruction set architecture (ISA) interface: the +contractual boundary between hardware designers and software developers. +ISSs allow hardware designers to quickly experiment with adding new processor +instructions while also allowing software developers to build new compilers, +libraries, and applications long before physical silicon is available. + +------------------------------------------------------------------------------- +Instruction-Set Simulators Must be Fast and Productive +------------------------------------------------------------------------------- + +Instruction-set simulators are more important than ever because the ISA +boundary has become increasingly fluid. +While `Moore's law`_ has continued to deliver larger numbers of transistors +which computer architects can use to build increasingly complex chips, limits +in `Dennard scaling`_ have restricted how these transistors can be used [d]_. +In more simple terms, thermal constraints (and energy constraints in mobile +devices) have resulted in a growing interest in pervasive *specialization*: +using custom accelerators to more efficiently perform compute intensive tasks. +This is already a reality for designers of mobile SOCs who continually add new +accelerator blocks and custom processor instructions in order to achieve higher +performance with less energy consumption. +ISSs are indispensable tools in this SOC design process for both hardware +architects building the silicon and software engineers developing the software +stack on top of it. + +An instruction set simulator has two primary responsibilities: 1) accurately +emulating the external execution behavior of the target, and 2) providing +observability by accurately reproducing the target's internal state (e.g., +register values, program counter, status flags) at each time step. +However, other qualities critical to an effective ISS are **simulation +performance** and **designer productivity**. +Simulation performance is important because shorter simulation times allow +developers to more quickly execute and verify large software applications. +Designer productivity is important because it allows hardware architects to +easily experiment with adding new instructions and estimate their impact on +application performance. + +To improve simulation performance, high-performance ISSs use dynamic binary +translation (DBT) as a mechanism to translate frequently visited blocks of +target instructions into optimized sequences of host instructions. +To improve designer productivity, many design toolchains automatically generate +ISSs from an architectural description language (ADL): a special +domain-specific language for succinctly specifying instruction encodings and +instruction semantics of an ISA. +Very few existing systems have managed to encapsulate the design complexity of +DBT engines such that high-performance, DBT-accelerated ISSs could be +automatically generated from ADLs [e]_. +Unfortunately, tools which have done so are either proprietary software or +leave much to be desired in terms of performance or productivity. + +------------------------------------------------------------------------------- +Why RPython? +------------------------------------------------------------------------------- + +Our research group learned of the RPython translation toolchain through our +experiences with PyPy, which we had used in conjunction with our Python +hardware modeling framework to achieve significant improvements in simulation +performance [2]_. +We realized that the RPython translation toolchain could potentially be adapted +to create fast instruction set simulators since the process of interpreting +executables comprised of binary instructions shared many similarities with the +process of interpreting bytecodes in a dynamic-language VM. +In addition, we were inspired by PyPy's meta-tracing approach to JIT-optimizing +VM design which effectively separates the process of specifying a language +interpreter from the optimization machinery needed to achieve good performance. + +Existing ADL-driven ISS generators have tended to use domain-specific +languages that require custom parsers or verbose C-based syntax that +distracts from the instruction specification. +Creating an embedded-ADL within Python provides several benefits over these +existing approaches including a gentler learning curve for new users, access to +better debugging tools, and easier maintenance and extension by avoiding a +custom parser. +Additionally, we have found that the ability to directly execute Pydgin +ISA descriptions in a standard Python interpreter such as CPython or PyPy +significantly helps debugging and testing during initial ISA exploration. +Python's concise, pseudocode-like syntax also manages to map quite closely to +the pseudocode specifications provided by many ISA manuals [f]_. + +------------------------------------------------------------------------------- +The Pydgin embedded-ADL +------------------------------------------------------------------------------- + +Defining a new ISA in the Pydgin embedded-ADL requires four primary pieces of +information: the architectural state (e.g. register file, program counter, +control registers), the bit encodings of each instruction, the instruction +fields, and the semantic definitions for each instruction. Pydgin aims to make +this process as painless as possible by providing helper classes and functions +where possible. + +For example, below we provide a truncated example of the ARMv5 instruction +encoding table. Pydgin maintains encodings of all instructions in a centralized +``encodings`` data structure for easy maintenance and quick lookup. The +user-provided instruction names and bit encodings are used to automatically +generate decoders for the simulator. Unlike many ADLs, Pydgin does not require +that the user explicitly specify instruction types or mask bits for field +matching because the Pydgin decoder generator can automatically infer decoder +fields from the encoding table. + +.. code:: python + + encodings = [ + ['adc', 'xxxx00x0101xxxxxxxxxxxxxxxxxxxxx'], + ['add', 'xxxx00x0100xxxxxxxxxxxxxxxxxxxxx'], + ['and', 'xxxx00x0000xxxxxxxxxxxxxxxxxxxxx'], + ['b', 'xxxx1010xxxxxxxxxxxxxxxxxxxxxxxx'], + ['bl', 'xxxx1011xxxxxxxxxxxxxxxxxxxxxxxx'], + ['bic', 'xxxx00x1110xxxxxxxxxxxxxxxxxxxxx'], + ['bkpt', '111000010010xxxxxxxxxxxx0111xxxx'], + ['blx1', '1111101xxxxxxxxxxxxxxxxxxxxxxxxx'], + ['blx2', 'xxxx00010010xxxxxxxxxxxx0011xxxx'], + # ... + ['teq', 'xxxx00x10011xxxxxxxxxxxxxxxxxxxx'], + ['tst', 'xxxx00x10001xxxxxxxxxxxxxxxxxxxx'], + ] + + +A major goal of Pydgin was ensuring instruction semantic definitions map to ISA +manual specifications as much as possible. The code below shows one such +definition for the ARMv5 ``add`` instruction. +A user-defined ``Instruction`` class (not shown) specifies field names that can +be used to conveniently access bit positions within an instruction (e.g. +``rd``, ``rn``, ``S``). +Additionally, users can choose to define their own helper functions, such as +the ``condition_passed`` function, to create more concise syntax that better +matches the ISA manual. + +.. code:: python + + def execute_add( s, inst ): + if condition_passed( s, inst.cond() ): + a, = s.rf[ inst.rn() ] + b, _ = shifter_operand( s, inst ) + result = a + b + s.rf[ inst.rd() ] = trim_32( result ) + + if inst.S(): + if inst.rd() == 15: + raise FatalError('Writing SPSR not implemented!') + s.N = (result >> 31)&1 + s.Z = trim_32( result ) == 0 + s.C = carry_from( result ) + s.V = overflow_from_add( a, b, result ) + + if inst.rd() == 15: + return + + s.rf[PC] = s.fetch_pc() + 4 + +Compared to the ARM ISA Reference manual shown below, the Pydgin instruction +definition is a fairly close match. Pydgin's definitions could certainly be +made more concise by using a custom DSL, however, this would lose many of the +debugging benefits afforded to a well-supported language such as Python and +additionally require using a custom parser that would likely need modification +for each new ISA. + +.. code:: + + if ConditionPassed(cond) then + Rd = Rn + shifter_operand + if S == 1 and Rd == R15 then + if CurrentModeHasSPSR() then CPSR = SPSR + else UNPREDICTABLE else if S == 1 then + N Flag = Rd[31] + Z Flag = if Rd == 0 then 1 else 0 + C Flag = CarryFrom(Rn + shifter_operand) + V Flag = OverflowFrom(Rn + shifter_operand) + + +Creating an ISS that can run real applications is a rather complex task, even +for a bare metal simulator with no operating system such as Pydgin. +Each system call in the C library must be properly implemented, and +bootstrapping code must be provided to set up the program stack and +architectural state. +This is a very tedious and error prone process which Pydgin tries to +encapsulate so that it remains as transparent to the end user as possible. +In future versions of Pydgin we hope to make bootstrapping more painless and +support a wider variety of C libraries. + +.. Architectural state... leave out for now. +.. :: + + class State( object ): + _virtualizable_ = ['pc', 'ncycles'] + def __init__( self, memory, debug, reset_addr=0x400 ): + self.pc = reset_addr + self.rf = ArmRegisterFile( self, num_regs=16 ) + self.mem = memory + + self.rf[ 15 ] = reset_addr + + # current program status register (CPSR) + self.N = 0b0 # Negative condition + self.Z = 0b0 # Zero condition + self.C = 0b0 # Carry condition + self.V = 0b0 # Overflow condition + + # other registers + self.status = 0 + self.ncycles = 0 + + def fetch_pc( self ): + return self.pc + + +------------------------------------------------------------------------------- +Pydgin Performance +------------------------------------------------------------------------------- + +In order to achieve good simulation performance from Pydgin ISSs, significant +work went into adding appropriate JIT annotations to the Pydgin library +components. +These optimization hints, which allow the JIT generated by the RPython +translation toolchain to produce more efficient code, have been specifically +selected for the unique properties of ISSs. +For the sake of brevity, we do not talk about the exact optimizations here but +a detailed discussion can be found in the ISPASS paper [1]_. +In the paper we evaluate two ISSs, one for a simplified MIPS ISA and another +for the ARMv5 ISA, whereas below we only discuss results for the ARMv5 ISS. + +The performance of Pydgin-generated ARMv5 ISSs were compared against +several reference ISSs: the gem5_ ARM atomic simulator (*gem5*), +interpretive and JIT-enabled versions of SimIt-ARM_ (*simit-nojit* and +*simit-jit*), and QEMU_. +Atomic models from the gem5 simulator were chosen for comparison due their wide +usage amongst computer architects [g]_. +SimIt-ARM was selected because it is currently the highest performance +ADL-generated DBT-ISS publicly available. +QEMU has long been held as the gold-standard for DBT simulators due to its +extremely high performance, however, QEMU is generally intended for usage as an +emulator rather than a simulator [c]_ and therefore achieves its excellent +performance at the cost of observability. +Unlike QEMU, all other simulators in our study faithfully track architectural +state at an instruction level rather than block level. +Pydgin ISSs were generated with and without JITs using the RPython translation +toolchain in order to help quantify the performance benefit of the meta-tracing +JIT. + +The figure below shows the performance of each ISS executing applications from +the SPEC CINT2006 benchmark suite [h]_. +Benchmarks were run to completion on the high-performance DBT-ISSs +(*simit-jit*, *pydgin-jit*, and QEMU), but were terminated after only +10 billion simulated instructions for the non-JITed interpretive ISSs +(these would require many hours, in some cases days, to run to completion). +Simulation performance is measured in MIPS [i]_ and plotted on a **log +scale** due to the wide variance in performance. +The *WHMEAN* group summarizes each ISS's performance across all benchmarks +using the weighted harmonic mean. + +.. image:: arm-bar-plot.png + +A few points to take away from these results: + +- ISSs without JITs (*gem5*, *simit-nojit*, and *pydgin-nojit*) demonstrate + relatively consistent performance across applications, whereas ISSs with JITs + (*simit-jit*, *pydgin-jit*, and QEMU) demonstrate much greater + performance variability from application-to-application. +- The *gem5* atomic model demonstrates particularly miserable performance, only + 2-3 MIPS! +- QEMU lives up to its reputation as a gold-standard for simulator performance, + leading the pack on nearly every benchmark and reaching speeds of 240-1120 + MIPS. +- *pydgin-jit* is able to outperform *simit-jit* on four of the + applications, including considerable performance improvements of 1.44–1.52× + for the applications *456.hmmer*, *462.libquantum*, and *471.omnetpp* + (managing to even outperform QEMU on *471.omnetpp*). +- *simit-jit* is able to obtain much more consistent performance (230-459 + MIPS across all applications) than *pydgin-jit* (9.6-659 MIPS). This is + due to *simit-jit*'s page-based approach to JIT optimization compared to + *pydgin-jit*'s tracing-based approach. +- *464.h264ref* displays particularly bad pathological behavior in Pydgin’s + tracing JIT and is the only application to perform worse on *pydgin-jit* + than *pydgin-nojit* (9.6 MIPS vs. 21 MIPS). + +The pathological behavior demonstrated by *464.h264ref* was of particular +concern because it caused *pydgin-jit* to perform even worse than having no +JIT at all. RPython JIT logs indicated that the reason for this performance +degradation was a large number of tracing aborts due to JIT traces growing too +long. However, time limitations before the publication deadline prevented us +from investigating this issue thoroughly. + +Since the deadline we've applied some minor bug fixes and made some small +improvements in the memory representation. +More importantly, we've addressed the performance degradation in *464.h264ref* +by increasing trace lengths for the JIT. +Below we show how the performance of *464.h264ref* changes as the +**trace_limit** parameter exposed by the RPython JIT is varied from the default +size of 6000 operations. + +.. image:: trace-length-plot.png + +By quadrupling the trace limit we achieve an 11x performance improvement in +*464.h264ref*. +The larger trace limit allows the JIT to optimize long code paths that were +previously triggering trace aborts, greatly helping amortize the costs of +tracing. +Note that arbitrarily increasing this limit can potentially hurt performance if +longer traces are not able to detect optimizable code sequences. + +After performing similar experiments across the applications in the SPEC +CINT2006 benchmark suite, we settled on a trace limit of 400,000 operations. +In the figure below we show how the updated Pydgin ISS (*pydgin-400K*) improves +performance across all benchmarks and fixes the performance degradation +previously seen in *464.h264ref*. Note that the non-JITted simulators have been +removed for clarity, and simulation performance is now plotted on a +**linear scale** to more clearly distinguish the performance gap between +each ISS. + +.. image:: new-bar-plot.png + +With these improvements, we are now able to beat *simit-jit* on all but two +benchmarks. In future work we hope to further close the gap with QEMU as well. + +------------------------------------------------------------------------------- +Conclusions and Future Work +------------------------------------------------------------------------------- + +Pydgin demonstrates that the impressive work put into the RPython translation +toolchain, designed to simplify the process of building fast dynamic-language +VMs, can also be leveraged to build fast instruction set simulators. +Our prototype ARMv5 ISS shows that Pydgin can generate ISSs with performance +competitive to SimIt-ARM while also providing a more productive development +experience: RPython allowed us to develop Pydgin with only four person-months +of work. +Another significant benefit of the Pydgin approach is that any performance +improvements applied to the RPython translation toolchain immediately benefit +Pydgin ISSs after a simple software download and retranslation. +This allows Pydgin to track the continual advances in JIT technology introduced +by the PyPy development team. + +Pydgin is very much a work in progress. There are many features we would like +to add, including: + +- more concise syntax for accessing arbitrary instruction bits +- support for other C libraries such as glibc, uClibc, and musl + (we currently only support binaries compiled with newlib) +- support for self-modifying code +- features for more productive debugging of target applications +- ISS descriptions for other ISAs such as RISC-V, ARMv8, and x86 +- automatic generation of compilers and toolchains from Pydgin descriptions + +In addition, we think there are opportunities for even greater performance +improvements with more advanced techniques such as: + +- automatic generation of optimized instruction decoders +- optimizations for floating-point intensive applications +- multiple tracing-JITs for parallel simulation of multicore SOCs +- a parallel JIT compilation engine as proposed by Böhm et al. [3]_ + +We hope that Pydgin can be of use to others, so if you try it out please let us +know what you think. Feel free to contact us if you find any of the above +development projects interesting, or simply fork the project on GitHub and hack +away! + +-- Derek Lockhart and Berkin Ilbeyi + +------------------------------------------------------------------------------- +Acknowledgements +------------------------------------------------------------------------------- + +We would like to sincerely thank Carl Friedrich Bolz and Maciej +Fijalkowski for their feedback on the Pydgin publication and their +guidance on improving the JIT performance of our simulators. We would also +like to thank for the whole PyPy team for their incredible work on the +PyPy and the RPython translation toolchain. + +------------------------------------------------------------------------------- +Footnotes +------------------------------------------------------------------------------- + +.. [a] Pydgin loosely stands for [Py]thon [D]SL for [G]enerating + [In]struction set simulators and is pronounced the same as “pigeon”. The + name is inspired by the word “pidgin” which is a grammatically simplified + form of language and captures the intent of the Pydgin embedded-ADL. + https://github.com/cornell-brg/pydgin +.. [b] Popular instruction set architectures (ISAs) include MIPs, ARM, + x86, and more recently RISC-V +.. [c] For a good discussion of simulators vs. emulators, please see the + following post on StackOverflow: + http://stackoverflow.com/questions/1584617/simulator-or-emulator-what-is-the-difference +.. [d] http://en.wikipedia.org/wiki/Dark_silicon +.. [e] Please see the Pydgin paper for a more detailed discussion of prior work. +.. [f] For more examples of Pydgin ISA specifications, please see the ISPASS + paper [1]_ or the Pydgin source code on GitHub. + + Pydgin instruction definitions for a simple MIPS-inspired ISA can be + found here: + + - https://github.com/cornell-brg/pydgin/blob/master/parc/isa.py + + Pydgin instruction definitions for a simplified ARMv5 ISA can be found + here: + + - https://github.com/cornell-brg/pydgin/blob/master/arm/isa.py + +.. [g] gem5 is a cycle-level simulation framework that contains both + functional-level (atomic) and cycle-level processor models. Although + primarily used for detailed, cycle-approximate processor simulation, + gem5's atomic model is a popular tool for many ISS tasks. + + - http://www.m5sim.org/SimpleCPU +.. [h] All performance measurements were taken on an unloaded server-class + machine. +.. [i] Millions of instructions per second. + +.. _gem5: http://www.gem5.org/ +.. _SimIt-ARM: http://simit-arm.sourceforge.net/ +.. _QEMU: http://wiki.qemu.org/ +.. _`Moore's Law`: http://en.wikipedia.org/wiki/Moore%27s_law +.. _`Dennard scaling`: http://en.wikipedia.org/wiki/Dennard_scaling#Recent_breakdown_of_Dennard_scaling + +------------------------------------------------------------------------------- +References +------------------------------------------------------------------------------- + +.. [1] Derek Lockhart, Berkin Ilbeyi, and Christopher Batten. "Pydgin: + Generating Fast Instruction Set Simulators from Simple Architecture + Descriptions with Meta-Tracing JIT Compilers." IEEE Int'l Symp. on + Performance Analysis of Systems and Software (ISPASS), Mar. 2015. + + - http://csl.cornell.edu/~cbatten/pdfs/lockhart-pydgin-ispass2015.pdf + - https://github.com/cornell-brg/pydgin + +.. [2] Derek Lockhart, Gary Zibrat, and Christopher Batten. "PyMTL: A Unified + Framework for Vertically Integrated Computer Architecture Research." 47th + ACM/IEEE Int'l Symp. on Microarchitecture (MICRO-47), Dec. 2014. + + - http://csl.cornell.edu/~cbatten/pdfs/lockhart-pymtl-micro2014.pdf + - https://github.com/cornell-brg/pymtl + +.. [3] I. Böhm, B. Franke, and N. Topham. Generalized Just-In-Time Trace + Compilation Using a Parallel Task Farm in a Dynamic Binary Translator. + ACM SIGPLAN Conference on Programming Language Design and Implementation + (PLDI), Jun 2011. + + diff --git a/blog/draft/dmlockhart-pydgin/new-bar-plot.png b/blog/draft/dmlockhart-pydgin/new-bar-plot.png new file mode 100644 index 0000000000000000000000000000000000000000..51ccd40b5fbe1ce82cb3fc2182eb44d906522cb6 GIT binary patch [cut] diff --git a/blog/draft/dmlockhart-pydgin/trace-length-plot.png b/blog/draft/dmlockhart-pydgin/trace-length-plot.png new file mode 100644 index 0000000000000000000000000000000000000000..b7df22c0613da8825e51dd5d76577d64a8929f16 GIT binary patch [cut] From noreply at buildbot.pypy.org Fri Mar 13 10:11:17 2015 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 13 Mar 2015 10:11:17 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: merge Message-ID: <20150313091117.9C8E81C00EF@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: extradoc Changeset: r5513:c39e9226809e Date: 2015-03-13 11:11 +0200 http://bitbucket.org/pypy/extradoc/changeset/c39e9226809e/ Log: merge diff --git a/blog/draft/stm-mar2015.txt b/blog/draft/stm-mar2015.txt new file mode 100644 --- /dev/null +++ b/blog/draft/stm-mar2015.txt @@ -0,0 +1,10 @@ +============= +Status of STM +============= + +- stmgc-c7 reasonably good results, now porting to stmgc-c8 + +- stmgc-c7 results (with download link), explain the new + features + +- stmgc-c8 in two words diff --git a/sprintinfo/leysin-winter-2015/people.txt b/sprintinfo/leysin-winter-2015/people.txt --- a/sprintinfo/leysin-winter-2015/people.txt +++ b/sprintinfo/leysin-winter-2015/people.txt @@ -17,7 +17,6 @@ Manuel Jacob 21-28 Ermina Joan Massich 20-? Ermina Quim Sanchez 20-? Ermina -Francisco Fernandez 20-28 Ermina Alexander Schremmer 21-23 Ermina ==================== ============== ======================= @@ -27,6 +26,7 @@ ==================== ============== ===================== Name Arrive/Depart Accomodation ==================== ============== ===================== +Francisco Fernandez no-show in Leysin Winter 2015 Romain Guillebert ? ? Christian Clauss ? ? Johan Råde ? ? From noreply at buildbot.pypy.org Fri Mar 13 14:59:24 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 13 Mar 2015 14:59:24 +0100 (CET) Subject: [pypy-commit] cffi release-0.9: Add the MD5/SHA Message-ID: <20150313135924.1FBEE1C050C@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: release-0.9 Changeset: r1671:166d09f9f626 Date: 2015-03-13 10:04 +0100 http://bitbucket.org/cffi/cffi/changeset/166d09f9f626/ Log: Add the MD5/SHA diff --git a/doc/source/index.rst b/doc/source/index.rst --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -92,9 +92,9 @@ - Or grab the most current version by following the instructions below. - - MD5: ... + - MD5: b1bf4625ae07a8a932f2f1a2eb200c54 - - SHA: ... + - SHA: 7cfc992699ef8b65d6300c04f3efad00bd2a6cba * Or get it from the `Bitbucket page`_: ``hg clone https://bitbucket.org/cffi/cffi`` From noreply at buildbot.pypy.org Fri Mar 13 14:59:25 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 13 Mar 2015 14:59:25 +0100 (CET) Subject: [pypy-commit] cffi default: hg merge release-0.9 (this is 0.9.2) Message-ID: <20150313135925.411541C050C@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1672:457ae0dfbfa9 Date: 2015-03-13 10:04 +0100 http://bitbucket.org/cffi/cffi/changeset/457ae0dfbfa9/ Log: hg merge release-0.9 (this is 0.9.2) diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -5787,7 +5787,7 @@ if (v == NULL || PyModule_AddObject(m, "_C_API", v) < 0) INITERROR; - v = PyText_FromString("0.9.1"); + v = PyText_FromString("0.9.2"); if (v == NULL || PyModule_AddObject(m, "__version__", v) < 0) INITERROR; diff --git a/c/test_c.py b/c/test_c.py --- a/c/test_c.py +++ b/c/test_c.py @@ -3342,4 +3342,4 @@ def test_version(): # this test is here mostly for PyPy - assert __version__ == "0.9.1" + assert __version__ == "0.9.2" diff --git a/cffi/__init__.py b/cffi/__init__.py --- a/cffi/__init__.py +++ b/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "0.9.1" -__version_info__ = (0, 9, 1) +__version__ = "0.9.2" +__version_info__ = (0, 9, 2) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/doc/source/conf.py b/doc/source/conf.py --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -47,7 +47,7 @@ # The short X.Y version. version = '0.9' # The full version, including alpha/beta/rc tags. -release = '0.9.1' +release = '0.9.2' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/source/index.rst b/doc/source/index.rst --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -88,13 +88,13 @@ Download and Installation: -* http://pypi.python.org/packages/source/c/cffi/cffi-0.9.1.tar.gz +* http://pypi.python.org/packages/source/c/cffi/cffi-0.9.2.tar.gz - Or grab the most current version by following the instructions below. - - MD5: 8dbdf23c600845b75654024e434601ce + - MD5: b1bf4625ae07a8a932f2f1a2eb200c54 - - SHA: 6a1b297ee7108803f54bb6333cf1e5630a193756 + - SHA: 7cfc992699ef8b65d6300c04f3efad00bd2a6cba * Or get it from the `Bitbucket page`_: ``hg clone https://bitbucket.org/cffi/cffi`` diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -139,7 +139,7 @@ `Mailing list `_ """, - version='0.9.1', + version='0.9.2', packages=['cffi'], zip_safe=False, From noreply at buildbot.pypy.org Fri Mar 13 15:04:27 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 13 Mar 2015 15:04:27 +0100 (CET) Subject: [pypy-commit] pypy default: Update to cffi 0.9.2. Message-ID: <20150313140427.613001C0541@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76361:a240afa2a1fb Date: 2015-03-13 15:04 +0100 http://bitbucket.org/pypy/pypy/changeset/a240afa2a1fb/ Log: Update to cffi 0.9.2. diff --git a/lib_pypy/cffi.egg-info b/lib_pypy/cffi.egg-info --- a/lib_pypy/cffi.egg-info +++ b/lib_pypy/cffi.egg-info @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: cffi -Version: 0.9.1 +Version: 0.9.2 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "0.9.1" -__version_info__ = (0, 9, 1) +__version__ = "0.9.2" +__version_info__ = (0, 9, 2) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -2,7 +2,7 @@ from pypy.interpreter.mixedmodule import MixedModule from rpython.rlib import rdynload -VERSION = "0.9.1" +VERSION = "0.9.2" class Module(MixedModule): diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -3247,6 +3247,88 @@ cast(p, c)[1] += 500 assert list(a) == [10000, 20500, 30000] +def test_from_buffer_not_str_unicode_bytearray(): + BChar = new_primitive_type("char") + BCharP = new_pointer_type(BChar) + BCharA = new_array_type(BCharP, None) + py.test.raises(TypeError, from_buffer, BCharA, b"foo") + py.test.raises(TypeError, from_buffer, BCharA, u"foo") + py.test.raises(TypeError, from_buffer, BCharA, bytearray(b"foo")) + try: + from __builtin__ import buffer + except ImportError: + pass + else: + py.test.raises(TypeError, from_buffer, BCharA, buffer(b"foo")) + py.test.raises(TypeError, from_buffer, BCharA, buffer(u"foo")) + py.test.raises(TypeError, from_buffer, BCharA, + buffer(bytearray(b"foo"))) + try: + from __builtin__ import memoryview + except ImportError: + pass + else: + py.test.raises(TypeError, from_buffer, BCharA, memoryview(b"foo")) + py.test.raises(TypeError, from_buffer, BCharA, + memoryview(bytearray(b"foo"))) + +def test_from_buffer_more_cases(): + try: + from _cffi_backend import _testbuff + except ImportError: + py.test.skip("not for pypy") + BChar = new_primitive_type("char") + BCharP = new_pointer_type(BChar) + BCharA = new_array_type(BCharP, None) + # + def check1(bufobj, expected): + c = from_buffer(BCharA, bufobj) + assert typeof(c) is BCharA + if sys.version_info >= (3,): + expected = [bytes(c, "ascii") for c in expected] + assert list(c) == list(expected) + # + def check(methods, expected, expected_for_memoryview=None): + if sys.version_info >= (3,): + if methods <= 7: + return + if expected_for_memoryview is not None: + expected = expected_for_memoryview + class X(object): + pass + _testbuff(X, methods) + bufobj = X() + check1(bufobj, expected) + try: + from __builtin__ import buffer + bufobjb = buffer(bufobj) + except (TypeError, ImportError): + pass + else: + check1(bufobjb, expected) + try: + bufobjm = memoryview(bufobj) + except (TypeError, NameError): + pass + else: + check1(bufobjm, expected_for_memoryview or expected) + # + check(1, "RDB") + check(2, "WRB") + check(4, "CHB") + check(8, "GTB") + check(16, "ROB") + # + check(1 | 2, "RDB") + check(1 | 4, "RDB") + check(2 | 4, "CHB") + check(1 | 8, "RDB", "GTB") + check(1 | 16, "RDB", "ROB") + check(2 | 8, "WRB", "GTB") + check(2 | 16, "WRB", "ROB") + check(4 | 8, "CHB", "GTB") + check(4 | 16, "CHB", "ROB") + def test_version(): # this test is here mostly for PyPy - assert __version__ == "0.9.1" + assert __version__ == "0.9.2" diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py b/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py --- a/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py @@ -27,7 +27,10 @@ if e.errno in (errno.ENOTDIR, errno.EINVAL): shutil.copy(src, dst) else: - print 'got errno',e.errno,'not',errno.ENOTDIR + print('got errno') + print(e.errno) + print('not') + print(errno.ENOTDIR) raise site_packages = None From noreply at buildbot.pypy.org Fri Mar 13 15:12:21 2015 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 13 Mar 2015 15:12:21 +0100 (CET) Subject: [pypy-commit] pypy object-dtype2: add a placeholder for gc customtrace function Message-ID: <20150313141221.512AC1C03D9@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: object-dtype2 Changeset: r76362:ec15bfb10429 Date: 2015-03-13 16:10 +0200 http://bitbucket.org/pypy/pypy/changeset/ec15bfb10429/ Log: add a placeholder for gc customtrace function diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py --- a/pypy/module/micronumpy/concrete.py +++ b/pypy/module/micronumpy/concrete.py @@ -381,6 +381,9 @@ if storage == lltype.nullptr(RAW_STORAGE): storage = dtype.itemtype.malloc(support.product(shape) * dtype.elsize, zero=zero) + if dtype.num == NPY.OBJECT: + # Register a customtrace function for this storage + pass ConcreteArrayNotOwning.__init__(self, shape, dtype, order, strides, backstrides, storage) From noreply at buildbot.pypy.org Fri Mar 13 15:12:23 2015 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 13 Mar 2015 15:12:23 +0100 (CET) Subject: [pypy-commit] pypy release-2.5.x: merge default into branch Message-ID: <20150313141223.043471C03D9@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: release-2.5.x Changeset: r76363:0b5d7c30ef74 Date: 2015-03-13 16:12 +0200 http://bitbucket.org/pypy/pypy/changeset/0b5d7c30ef74/ Log: merge default into branch diff --git a/lib_pypy/cffi.egg-info b/lib_pypy/cffi.egg-info --- a/lib_pypy/cffi.egg-info +++ b/lib_pypy/cffi.egg-info @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: cffi -Version: 0.9.1 +Version: 0.9.2 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "0.9.1" -__version_info__ = (0, 9, 1) +__version__ = "0.9.2" +__version_info__ = (0, 9, 2) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -2,7 +2,7 @@ from pypy.interpreter.mixedmodule import MixedModule from rpython.rlib import rdynload -VERSION = "0.9.1" +VERSION = "0.9.2" class Module(MixedModule): diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -3247,6 +3247,88 @@ cast(p, c)[1] += 500 assert list(a) == [10000, 20500, 30000] +def test_from_buffer_not_str_unicode_bytearray(): + BChar = new_primitive_type("char") + BCharP = new_pointer_type(BChar) + BCharA = new_array_type(BCharP, None) + py.test.raises(TypeError, from_buffer, BCharA, b"foo") + py.test.raises(TypeError, from_buffer, BCharA, u"foo") + py.test.raises(TypeError, from_buffer, BCharA, bytearray(b"foo")) + try: + from __builtin__ import buffer + except ImportError: + pass + else: + py.test.raises(TypeError, from_buffer, BCharA, buffer(b"foo")) + py.test.raises(TypeError, from_buffer, BCharA, buffer(u"foo")) + py.test.raises(TypeError, from_buffer, BCharA, + buffer(bytearray(b"foo"))) + try: + from __builtin__ import memoryview + except ImportError: + pass + else: + py.test.raises(TypeError, from_buffer, BCharA, memoryview(b"foo")) + py.test.raises(TypeError, from_buffer, BCharA, + memoryview(bytearray(b"foo"))) + +def test_from_buffer_more_cases(): + try: + from _cffi_backend import _testbuff + except ImportError: + py.test.skip("not for pypy") + BChar = new_primitive_type("char") + BCharP = new_pointer_type(BChar) + BCharA = new_array_type(BCharP, None) + # + def check1(bufobj, expected): + c = from_buffer(BCharA, bufobj) + assert typeof(c) is BCharA + if sys.version_info >= (3,): + expected = [bytes(c, "ascii") for c in expected] + assert list(c) == list(expected) + # + def check(methods, expected, expected_for_memoryview=None): + if sys.version_info >= (3,): + if methods <= 7: + return + if expected_for_memoryview is not None: + expected = expected_for_memoryview + class X(object): + pass + _testbuff(X, methods) + bufobj = X() + check1(bufobj, expected) + try: + from __builtin__ import buffer + bufobjb = buffer(bufobj) + except (TypeError, ImportError): + pass + else: + check1(bufobjb, expected) + try: + bufobjm = memoryview(bufobj) + except (TypeError, NameError): + pass + else: + check1(bufobjm, expected_for_memoryview or expected) + # + check(1, "RDB") + check(2, "WRB") + check(4, "CHB") + check(8, "GTB") + check(16, "ROB") + # + check(1 | 2, "RDB") + check(1 | 4, "RDB") + check(2 | 4, "CHB") + check(1 | 8, "RDB", "GTB") + check(1 | 16, "RDB", "ROB") + check(2 | 8, "WRB", "GTB") + check(2 | 16, "WRB", "ROB") + check(4 | 8, "CHB", "GTB") + check(4 | 16, "CHB", "ROB") + def test_version(): # this test is here mostly for PyPy - assert __version__ == "0.9.1" + assert __version__ == "0.9.2" diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -700,7 +700,8 @@ @rgc.must_be_light_finalizer def __del__(self): - lltype.free(self.buffer, flavor='raw') + if self.buffer: + lltype.free(self.buffer, flavor='raw') def setlen(self, size, zero=False, overallocate=True): if size > 0: diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py b/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py --- a/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py @@ -165,7 +165,8 @@ assert lib.sin(12.3) == math.sin(12.3) v = ffi.verifier ext = v.get_extension() - assert 'distutils.extension.Extension' in str(ext.__class__) + assert 'distutils.extension.Extension' in str(ext.__class__) or \ + 'setuptools.extension.Extension' in str(ext.__class__) assert ext.sources == [maybe_relative_path(v.sourcefilename)] assert ext.name == v.get_module_name() assert ext.define_macros == [('TEST_EXTENSION_OBJECT', '1')] @@ -194,7 +195,8 @@ assert lib.test1eoes(7.0) == 42.0 v = ffi.verifier ext = v.get_extension() - assert 'distutils.extension.Extension' in str(ext.__class__) + assert 'distutils.extension.Extension' in str(ext.__class__) or \ + 'setuptools.extension.Extension' in str(ext.__class__) assert ext.sources == [maybe_relative_path(v.sourcefilename), extra_source] assert ext.name == v.get_module_name() diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py b/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py --- a/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py @@ -4,6 +4,9 @@ import subprocess from pypy.module.test_lib_pypy.cffi_tests.udir import udir +if sys.platform == 'win32': + py.test.skip('snippets do not run on win32') + def create_venv(name): tmpdir = udir.join(name) try: @@ -13,6 +16,23 @@ except OSError as e: py.test.skip("Cannot execute virtualenv: %s" % (e,)) + try: + deepcopy = os.symlink + except: + import shutil, errno + def deepcopy(src, dst): + try: + shutil.copytree(src, dst) + except OSError as e: + if e.errno in (errno.ENOTDIR, errno.EINVAL): + shutil.copy(src, dst) + else: + print('got errno') + print(e.errno) + print('not') + print(errno.ENOTDIR) + raise + site_packages = None for dirpath, dirnames, filenames in os.walk(str(tmpdir)): if os.path.basename(dirpath) == 'site-packages': @@ -32,7 +52,7 @@ modules += ('ply',) # needed for older versions of pycparser for module in modules: target = imp.find_module(module)[1] - os.symlink(target, os.path.join(site_packages, + deepcopy(target, os.path.join(site_packages, os.path.basename(target))) return tmpdir @@ -51,7 +71,11 @@ python_f.write(py.code.Source(python_snippet)) try: os.chdir(str(SNIPPET_DIR.join(dirname))) - vp = str(venv_dir.join('bin/python')) + if os.name == 'nt': + bindir = 'Scripts' + else: + bindir = 'bin' + vp = str(venv_dir.join(bindir).join('python')) subprocess.check_call((vp, 'setup.py', 'clean')) subprocess.check_call((vp, 'setup.py', 'install')) subprocess.check_call((vp, str(python_f))) diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py --- a/rpython/annotator/bookkeeper.py +++ b/rpython/annotator/bookkeeper.py @@ -12,7 +12,7 @@ SomeBuiltin, SomePBC, SomeInteger, TLS, SomeUnicodeCodePoint, s_None, s_ImpossibleValue, SomeBool, SomeTuple, SomeImpossibleValue, SomeUnicodeString, SomeList, HarmlesslyBlocked, - SomeWeakRef, SomeByteArray, SomeConstantType) + SomeWeakRef, SomeByteArray, SomeConstantType, SomeProperty) from rpython.annotator.classdef import InstanceSource, ClassDef from rpython.annotator.listdef import ListDef, ListItem from rpython.annotator.dictdef import DictDef @@ -301,6 +301,8 @@ s1 = self.immutablevalue(x1) assert isinstance(s1, SomeInstance) result = SomeWeakRef(s1.classdef) + elif tp is property: + return SomeProperty(x) elif ishashable(x) and x in BUILTIN_ANALYZERS: _module = getattr(x,"__module__","unknown") result = SomeBuiltin(BUILTIN_ANALYZERS[x], methodname="%s.%s" % (_module, x.__name__)) diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py --- a/rpython/annotator/description.py +++ b/rpython/annotator/description.py @@ -482,6 +482,19 @@ self.all_enforced_attrs = [] # no attribute allowed def add_source_attribute(self, name, value, mixin=False): + if isinstance(value, property): + # special case for property object + if value.fget is not None: + newname = name + '__getter__' + func = func_with_new_name(value.fget, newname) + self.add_source_attribute(newname, func, mixin) + if value.fset is not None: + newname = name + '__setter__' + func = func_with_new_name(value.fset, newname) + self.add_source_attribute(newname, func, mixin) + self.classdict[name] = Constant(value) + return + if isinstance(value, types.FunctionType): # for debugging if not hasattr(value, 'class_'): diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py --- a/rpython/annotator/model.py +++ b/rpython/annotator/model.py @@ -587,6 +587,19 @@ return False +class SomeProperty(SomeObject): + # used for union error only + immutable = True + knowntype = type(property) + + def __init__(self, prop): + self.fget = prop.fget + self.fset = prop.fset + + def can_be_none(self): + return False + + s_None = SomeNone() s_Bool = SomeBool() s_Int = SomeInteger() diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py --- a/rpython/annotator/test/test_annrpython.py +++ b/rpython/annotator/test/test_annrpython.py @@ -4326,6 +4326,82 @@ assert isinstance(s, annmodel.SomeString) assert not s.can_be_none() + def test_property_getter(self): + class O1(object): + def __init__(self, x): + self._x = x + @property + def x(self): + return self._x + def f(n): + o = O1(n) + return o.x + getattr(o, 'x') + a = self.RPythonAnnotator() + s = a.build_types(f, [int]) + assert isinstance(s, annmodel.SomeInteger) + op = list(graphof(a, f).iterblocks())[0].operations + i = 0 + c = 0 + while i < len(op): + if op[i].opname == 'getattr': + c += 1 + assert op[i].args[1].value == 'x__getter__' + i += 1 + assert i < len(op) and op[i].opname == 'simple_call' and \ + op[i].args[0] == op[i - 1].result + i += 1 + assert c == 2 + + def test_property_setter(self): + class O2(object): + def __init__(self): + self._x = 0 + def set_x(self, v): + self._x = v + x = property(fset=set_x) + def f(n): + o = O2() + o.x = n + setattr(o, 'x', n) + a = self.RPythonAnnotator() + s = a.build_types(f, [int]) + op = list(graphof(a, f).iterblocks())[0].operations + i = 0 + c = 0 + while i < len(op): + if op[i].opname == 'getattr': + c += 1 + assert op[i].args[1].value == 'x__setter__' + i += 1 + assert i < len(op) and op[i].opname == 'simple_call' and \ + op[i].args[0] == op[i - 1].result and len(op[i].args) == 2 + i += 1 + assert c == 2 + + def test_property_unionerr(self): + class O1(object): + def __init__(self, x): + self._x = x + @property + def x(self): + return self._x + class O2(O1): + def set_x(self, v): + self._x = v + x = property(fset=set_x) + def f1(n): + o = O2(n) + return o.x + def f2(n): + o = O2(n) + o.x = 20 + a = self.RPythonAnnotator() + with py.test.raises(annmodel.UnionError) as exc: + a.build_types(f1, [int]) + a = self.RPythonAnnotator() + with py.test.raises(annmodel.UnionError) as exc: + a.build_types(f2, [int]) + def g(n): return [0, 1, 2, n] diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py --- a/rpython/annotator/unaryop.py +++ b/rpython/annotator/unaryop.py @@ -5,7 +5,7 @@ from __future__ import absolute_import from rpython.flowspace.operation import op -from rpython.flowspace.model import const +from rpython.flowspace.model import const, Constant from rpython.annotator.model import (SomeObject, SomeInteger, SomeBool, SomeString, SomeChar, SomeList, SomeDict, SomeTuple, SomeImpossibleValue, SomeUnicodeCodePoint, SomeInstance, SomeBuiltin, SomeBuiltinMethod, @@ -715,6 +715,50 @@ op.simple_call(get_setslice.result, v_start, v_stop, v_iterable)] +def _find_property_meth(s_obj, attr, meth): + result = [] + for clsdef in s_obj.classdef.getmro(): + dct = clsdef.classdesc.classdict + if attr not in dct: + continue + obj = dct[attr] + if (not isinstance(obj, Constant) or + not isinstance(obj.value, property)): + return + result.append(getattr(obj.value, meth)) + return result + + + at op.getattr.register_transform(SomeInstance) +def getattr_SomeInstance(annotator, v_obj, v_attr): + s_attr = annotator.annotation(v_attr) + if not s_attr.is_constant() or not isinstance(s_attr.const, str): + return + attr = s_attr.const + getters = _find_property_meth(annotator.annotation(v_obj), attr, 'fget') + if getters: + if all(getters): + get_getter = op.getattr(v_obj, const(attr + '__getter__')) + return [get_getter, op.simple_call(get_getter.result)] + elif not any(getters): + raise AnnotatorError("Attribute %r is unreadable" % attr) + + + at op.setattr.register_transform(SomeInstance) +def setattr_SomeInstance(annotator, v_obj, v_attr, v_value): + s_attr = annotator.annotation(v_attr) + if not s_attr.is_constant() or not isinstance(s_attr.const, str): + return + attr = s_attr.const + setters = _find_property_meth(annotator.annotation(v_obj), attr, 'fset') + if setters: + if all(setters): + get_setter = op.getattr(v_obj, const(attr + '__setter__')) + return [get_setter, op.simple_call(get_setter.result, v_value)] + elif not any(setters): + raise AnnotatorError("Attribute %r is unwritable" % attr) + + class __extend__(SomeBuiltin): def call(self, args, implicit_init=False): args_s, kwds = args.unpack() diff --git a/rpython/doc/arm.rst b/rpython/doc/arm.rst --- a/rpython/doc/arm.rst +++ b/rpython/doc/arm.rst @@ -160,5 +160,5 @@ The gcrootfinder option is needed to work around `issue 1377`_ and the jit-backend works around `issue 1376`_ -.. _issue 1377: https://bugs.pypy.org/issue1377 -.. _issue 1376: https://bugs.pypy.org/issue1376 +.. _issue 1377: https://bitbucket.org/pypy/pypy/issue/1377 +.. _issue 1376: https://bitbucket.org/pypy/pypy/issue/1376 diff --git a/rpython/doc/conf.py b/rpython/doc/conf.py --- a/rpython/doc/conf.py +++ b/rpython/doc/conf.py @@ -59,16 +59,16 @@ # General information about the project. project = u'RPython' -copyright = u'2013, The PyPy Project' +copyright = u'2015, The PyPy Project' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '2.0' +version = '2.5' # The full version, including alpha/beta/rc tags. -release = '2.0-beta1' +release = '2.5.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py --- a/rpython/flowspace/flowcontext.py +++ b/rpython/flowspace/flowcontext.py @@ -772,15 +772,10 @@ def FOR_ITER(self, target): w_iterator = self.peekvalue() - try: - w_nextitem = op.next(w_iterator).eval(self) - self.pushvalue(w_nextitem) - except Raise as e: - if self.exception_match(e.w_exc.w_type, const(StopIteration)): - self.popvalue() - return target - else: - raise + self.blockstack.append(IterBlock(self, target)) + w_nextitem = op.next(w_iterator).eval(self) + self.blockstack.pop() + self.pushvalue(w_nextitem) def SETUP_LOOP(self, target): block = LoopBlock(self, target) @@ -1333,6 +1328,16 @@ ctx.last_exception = w_exc return self.handlerposition # jump to the handler +class IterBlock(ExceptBlock): + """A pseudo-block to catch the StopIteration inside FOR_ITER""" + def handle(self, ctx, unroller): + w_exc = unroller.w_exc + if ctx.exception_match(w_exc.w_type, const(StopIteration)): + ctx.popvalue() + return self.handlerposition + else: + return ctx.unroll(unroller) + class FinallyBlock(FrameBlock): """A try:finally: block. Stores the position of the exception handler.""" diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py --- a/rpython/flowspace/operation.py +++ b/rpython/flowspace/operation.py @@ -555,7 +555,7 @@ opname = 'next' arity = 1 can_overflow = False - canraise = [] + canraise = [StopIteration, RuntimeError] pyfunc = staticmethod(next) def eval(self, ctx): @@ -571,9 +571,7 @@ else: ctx.replace_in_stack(it, next_unroller) return const(v) - w_item = ctx.do_op(self) - ctx.recorder.guessexception(ctx, StopIteration, RuntimeError) - return w_item + return HLOperation.eval(self, ctx) class GetAttr(SingleDispatchMixin, HLOperation): opname = 'getattr' diff --git a/rpython/translator/c/src/mem.c b/rpython/translator/c/src/mem.c --- a/rpython/translator/c/src/mem.c +++ b/rpython/translator/c/src/mem.c @@ -29,6 +29,8 @@ RPY_EXTERN void pypy_debug_alloc_stop(void *addr) { + if (!addr) + return; struct pypy_debug_alloc_s **p; for (p = &pypy_debug_alloc_list; *p; p = &((*p)->next)) if ((*p)->addr == addr) From noreply at buildbot.pypy.org Fri Mar 13 16:28:31 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Fri, 13 Mar 2015 16:28:31 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c8: probably missing memclear after allocation of nonmovable obj Message-ID: <20150313152831.582781C0471@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: stmgc-c8 Changeset: r76364:f30c3fd32ba8 Date: 2015-03-13 14:41 +0100 http://bitbucket.org/pypy/pypy/changeset/f30c3fd32ba8/ Log: probably missing memclear after allocation of nonmovable obj diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py --- a/rpython/translator/stm/funcgen.py +++ b/rpython/translator/stm/funcgen.py @@ -123,7 +123,9 @@ result = funcgen.expr(op.result) # XXX NULL returns? return ('%s = (rpygcchar_t *)_stm_allocate_external(%s >= 16 ? %s : 16); ' % - (result, arg_size, arg_size) + + (result, arg_size, arg_size) + + 'pypy_stm_memclearinit((object_t*)%s, 0, %s >= 16 ? %s : 16);' % + (result, arg_size, arg_size) + '((rpyobj_t *)%s)->tid = %s;' % (result, arg_type_id)) def stm_set_into_obj(funcgen, op): From noreply at buildbot.pypy.org Fri Mar 13 16:28:48 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Fri, 13 Mar 2015 16:28:48 +0100 (CET) Subject: [pypy-commit] stmgc default: may help for debugging Message-ID: <20150313152848.2B7251C0471@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1734:14427b503629 Date: 2015-03-13 13:14 +0100 http://bitbucket.org/pypy/stmgc/changeset/14427b503629/ Log: may help for debugging diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c --- a/c8/stm/nursery.c +++ b/c8/stm/nursery.c @@ -461,16 +461,20 @@ } OPT_ASSERT((nursery_used & 7) == 0); - -#if _STM_NURSERY_ZEROED +#ifndef NDEBUG /* reset the nursery by zeroing it */ char *realnursery; realnursery = REAL_ADDRESS(pseg->pub.segment_base, _stm_nursery_start); +#if _STM_NURSERY_ZEROED memset(realnursery, 0, nursery_used); /* assert that the rest of the nursery still contains only zeroes */ assert_memset_zero(realnursery + nursery_used, (NURSERY_END - _stm_nursery_start) - nursery_used); + +#else + memset(realnursery, 0xa0, nursery_used); +#endif #endif pseg->pub.nursery_current = (stm_char *)_stm_nursery_start; From noreply at buildbot.pypy.org Fri Mar 13 16:28:49 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Fri, 13 Mar 2015 16:28:49 +0100 (CET) Subject: [pypy-commit] stmgc default: another debugging helper Message-ID: <20150313152849.3CE9F1C0471@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1735:0328ee7e02d1 Date: 2015-03-13 14:47 +0100 http://bitbucket.org/pypy/stmgc/changeset/0328ee7e02d1/ Log: another debugging helper diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c --- a/c8/stm/nursery.c +++ b/c8/stm/nursery.c @@ -640,6 +640,11 @@ #if _STM_NURSERY_ZEROED memset(REAL_ADDRESS(STM_SEGMENT->segment_base, o), 0, size_rounded_up); #else + +#ifndef NDEBUG + memset(REAL_ADDRESS(STM_SEGMENT->segment_base, o), 0xb0, size_rounded_up); +#endif + o->stm_flags = 0; /* make all pages of 'o' accessible as synchronize_obj_flush() in minor collections assumes all young objs are fully accessible. */ From noreply at buildbot.pypy.org Fri Mar 13 17:36:26 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 13 Mar 2015 17:36:26 +0100 (CET) Subject: [pypy-commit] pypy default: in-progress: adapt stm.rst Message-ID: <20150313163626.4DA091C03D9@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76365:1cf084edb3ab Date: 2015-03-13 17:36 +0100 http://bitbucket.org/pypy/pypy/changeset/1cf084edb3ab/ Log: in-progress: adapt stm.rst diff --git a/pypy/doc/stm.rst b/pypy/doc/stm.rst --- a/pypy/doc/stm.rst +++ b/pypy/doc/stm.rst @@ -45,11 +45,10 @@ it as a drop-in replacement and multithreaded programs will run on multiple cores. -* ``pypy-stm`` does not impose any special API to the user, but it - provides a new pure Python module called `transactional_memory`_ with - features to inspect the state or debug conflicts_ that prevent - parallelization. This module can also be imported on top of a non-STM - PyPy or CPython. +* ``pypy-stm`` provides (but does not impose) a special API to the + user in the pure Python module `transaction`_. This module is based + on the lower-level module `pypystm`_, but also provides some + compatibily with non-STM PyPy's or CPython's. * Building on top of the way the GIL is removed, we will talk about `Atomic sections, Transactions, etc.: a better way to write @@ -63,9 +62,10 @@ Development is done in the branch `stmgc-c7`_. If you are only interested in trying it out, you can download a Ubuntu binary here__ -(``pypy-stm-2.3*.tar.bz2``, Ubuntu 12.04-14.04). The current version +(``pypy-stm-2.*.tar.bz2``, for Ubuntu 12.04-14.04). The current version supports four "segments", which means that it will run up to four -threads in parallel. +threads in parallel. (Development recently switched to `stmgc-c8`_, +but that is not ready for trying out yet.) To build a version from sources, you first need to compile a custom version of clang(!); we recommend downloading `llvm and clang like @@ -78,6 +78,7 @@ rpython/bin/rpython -Ojit --stm pypy/goal/targetpypystandalone.py .. _`stmgc-c7`: https://bitbucket.org/pypy/pypy/src/stmgc-c7/ +.. _`stmgc-c8`: https://bitbucket.org/pypy/pypy/src/stmgc-c8/ .. __: https://bitbucket.org/pypy/pypy/downloads/ .. __: http://clang.llvm.org/get_started.html .. __: https://bitbucket.org/pypy/stmgc/src/default/c7/llvmfix/ @@ -85,11 +86,11 @@ .. _caveats: -Current status --------------- +Current status (stmgc-c7) +------------------------- -* So far, small examples work fine, but there are still a few bugs. - We're busy fixing them as we find them; feel free to `report bugs`_. +* It seems to work fine, without crashing any more. Please `report + any crash`_ you find (or other bugs). * It runs with an overhead as low as 20% on examples like "richards". There are also other examples with higher overheads --currently up to @@ -97,8 +98,9 @@ One suspect is our partial GC implementation, see below. * Currently limited to 1.5 GB of RAM (this is just a parameter in - `core.h`__). Memory overflows are not correctly handled; they cause - segfaults. + `core.h`__ -- theoretically. In practice, increase it too much and + clang crashes again). Memory overflows are not correctly handled; + they cause segfaults. * The JIT warm-up time improved recently but is still bad. In order to produce machine code, the JIT needs to enter a special single-threaded @@ -114,11 +116,9 @@ numbers of small objects that don't immediately die (surely a common situation) suffer from these missing optimizations. -* The GC has no support for destructors: the ``__del__`` method is never - called (including on file objects, which won't be closed for you). - This is of course temporary. Also, weakrefs might appear to work a - bit strangely for now (staying alive even though ``gc.collect()``, or - even dying but then un-dying for a short time before dying again). +* Weakrefs might appear to work a bit strangely for now, sometimes + staying alive throught ``gc.collect()``, or even dying but then + un-dying for a short time before dying again. * The STM system is based on very efficient read/write barriers, which are mostly done (their placement could be improved a bit in @@ -130,7 +130,7 @@ * Very long-running processes (on the order of days) will eventually crash on an assertion error because of a non-implemented overflow of - an internal 29-bit number. + an internal 28-bit counter. .. _`report bugs`: https://bugs.pypy.org/ .. __: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/rpython/translator/stm/src_stm/stm/core.h @@ -175,29 +175,89 @@ This works by internally considering the points where a standard PyPy or CPython would release the GIL, and replacing them with the boundaries of -"transaction". Like their database equivalent, multiple transactions +"transactions". Like their database equivalent, multiple transactions can execute in parallel, but will commit in some serial order. They appear to behave as if they were completely run in this serialization order. +A better way to write parallel programs +--------------------------------------- + +In CPU-hungry programs, we can often easily identify outermost loops +over some data structure, or other repetitive algorithm, where each +"block" consists of processing a non-trivial amount of data, and where +the blocks "have a good chance" to be independent from each other. We +don't need to prove that they are actually independent: it is enough +if they are *often independent* --- or, more precisely, if we *think +they should be* often independent. + +One typical example would look like this, where the function ``func()`` +typically invokes a large amount of code:: + + for key, value in bigdict.items(): + func(key, value) + +Then you simply replace the loop with:: + + from transaction import TransactionQueue + + tr = TransactionQueue() + for key, value in bigdict.items(): + tr.add(func, key, value) + tr.run() + +This code's behavior is equivalent. Internally, the +``TransactionQueue`` object will start N threads and try to run the +``func(key, value)`` calls on all threads in parallel. But note the +difference with a regular thread-pooling library, as found in many +lower-level languages than Python: the function calls are not randomly +interleaved with each other just because they run in parallel. The +behavior did not change because we are using ``TransactionQueue``. +All the calls still *appear* to execute in some serial order. + +Now the performance should ideally be improved: if the function calls +turn out to be actually independent (most of the time), then it will +be. But if the function calls are not, then the total performance +will crawl back to the previous case, with additionally some small +penalty for the overhead. + +This case occurs typically when you see the total CPU usage remaining +low (closer to 1 than N cores). Note first that it is expected that +the CPU usage should not go much higher than 1 in the JIT warm-up +phase. You must run a program for several seconds, or for larger +programs at least one minute, to give the JIT a chance to warm up +correctly. But if CPU usage remains low even though all code is +executing in a ``TransactionQueue.run()``, then the ``PYPYSTM`` +environment variable can be used to track what is going on. + +Run your program with ``PYPYSTM=stmlog`` to produce a log file called +``stmlog``. Afterwards, use the ``pypy/stm/print_stm_log.py`` utility +to inspect the content of this log file. It produces output like +this:: + + documentation in progress! + + + Atomic sections --------------- -PyPy supports *atomic sections,* which are blocks of code which you want -to execute without "releasing the GIL". *This is experimental and may -be removed in the future.* In STM terms, this means blocks of code that -are executed while guaranteeing that the transaction is not interrupted -in the middle. +PyPy supports *atomic sections,* which are blocks of code which you +want to execute without "releasing the GIL". In STM terms, this means +blocks of code that are executed while guaranteeing that the +transaction is not interrupted in the middle. *This is experimental +and may be removed in the future* if `lock elision`_ is ever +implemented. Here is a usage example:: - with __pypy__.thread.atomic: + with transaction.atomic: assert len(lst1) == 10 x = lst1.pop(0) lst1.append(x) -In this (bad) example, we are sure that the item popped off one end of +In this example, we are sure that the item popped off one end of the list is appened again at the other end atomically. It means that another thread can run ``len(lst1)`` or ``x in lst1`` without any particular synchronization, and always see the same results, @@ -225,21 +285,22 @@ manually a transaction break just before the atomic block. This is because the boundaries of the block are not guaranteed to be the boundaries of the transaction: the latter is at least as big as the -block, but maybe bigger. Therefore, if you run a big atomic block, it +block, but may be bigger. Therefore, if you run a big atomic block, it is a good idea to break the transaction just before. This can be done -e.g. by the hack of calling ``time.sleep(0)``. (This may be fixed at +by calling ``transaction.hint_commit_soon()``. (This may be fixed at some point.) -There are also issues with the interaction of locks and atomic blocks. -This can be seen if you write to files (which have locks), including -with a ``print`` to standard output. If one thread tries to acquire a -lock while running in an atomic block, and another thread has got the -same lock, then the former may fail with a ``thread.error``. The reason -is that "waiting" for some condition to become true --while running in -an atomic block-- does not really make sense. For now you can work -around it by making sure that, say, all your prints are either in an -``atomic`` block or none of them are. (This kind of issue is -theoretically hard to solve.) +There are also issues with the interaction of regular locks and atomic +blocks. This can be seen if you write to files (which have locks), +including with a ``print`` to standard output. If one thread tries to +acquire a lock while running in an atomic block, and another thread +has got the same lock at that point, then the former may fail with a +``thread.error``. The reason is that "waiting" for some condition to +become true --while running in an atomic block-- does not really make +sense. For now you can work around it by making sure that, say, all +your prints are either in an ``atomic`` block or none of them are. +(This kind of issue is theoretically hard to solve and may be the +reason for atomic block support to eventually be removed.) Locks From noreply at buildbot.pypy.org Sat Mar 14 09:21:22 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 14 Mar 2015 09:21:22 +0100 (CET) Subject: [pypy-commit] pypy.org extradoc: update the values Message-ID: <20150314082122.0224A1C00EF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r581:d35a09b1e01c Date: 2015-03-14 09:21 +0100 http://bitbucket.org/pypy/pypy.org/changeset/d35a09b1e01c/ Log: update the values diff --git a/don1.html b/don1.html --- a/don1.html +++ b/don1.html @@ -15,7 +15,7 @@ - $58921 of $105000 (56.1%) + $58925 of $105000 (56.1%)
From noreply at buildbot.pypy.org Sat Mar 14 14:44:08 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 14 Mar 2015 14:44:08 +0100 (CET) Subject: [pypy-commit] pypy default: In pystate.py, it's too late to call rffi.aroundstate: the immediate Message-ID: <20150314134408.5AC9E1C0196@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76366:e8775e429f3d Date: 2015-03-14 14:44 +0100 http://bitbucket.org/pypy/pypy/changeset/e8775e429f3d/ Log: In pystate.py, it's too late to call rffi.aroundstate: the immediate caller, which is wrapper() function, needs to do it around everything. It fails an assert in debug builds and probably gives nonsense in non-debug builds. diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -192,7 +192,7 @@ class ApiFunction: def __init__(self, argtypes, restype, callable, error=_NOT_SPECIFIED, - c_name=None): + c_name=None, gil=None): self.argtypes = argtypes self.restype = restype self.functype = lltype.Ptr(lltype.FuncType(argtypes, restype)) @@ -208,6 +208,7 @@ assert argnames[0] == 'space' self.argnames = argnames[1:] assert len(self.argnames) == len(self.argtypes) + self.gil = gil def _freeze_(self): return True @@ -223,14 +224,15 @@ def get_wrapper(self, space): wrapper = getattr(self, '_wrapper', None) if wrapper is None: - wrapper = make_wrapper(space, self.callable) + wrapper = make_wrapper(space, self.callable, self.gil) self._wrapper = wrapper wrapper.relax_sig_check = True if self.c_name is not None: wrapper.c_name = cpyext_namespace.uniquename(self.c_name) return wrapper -def cpython_api(argtypes, restype, error=_NOT_SPECIFIED, external=True): +def cpython_api(argtypes, restype, error=_NOT_SPECIFIED, external=True, + gil=None): """ Declares a function to be exported. - `argtypes`, `restype` are lltypes and describe the function signature. @@ -240,6 +242,8 @@ SytemError. - set `external` to False to get a C function pointer, but not exported by the API headers. + - set `gil` to "acquire", "release" or "around" to acquire the GIL, + release the GIL, or both """ if isinstance(restype, lltype.Typedef): real_restype = restype.OF @@ -262,7 +266,8 @@ c_name = None else: c_name = func_name - api_function = ApiFunction(argtypes, restype, func, error, c_name=c_name) + api_function = ApiFunction(argtypes, restype, func, error, + c_name=c_name, gil=gil) func.api_func = api_function if external: @@ -594,12 +599,15 @@ pypy_debug_catch_fatal_exception = rffi.llexternal('pypy_debug_catch_fatal_exception', [], lltype.Void) # Make the wrapper for the cases (1) and (2) -def make_wrapper(space, callable): +def make_wrapper(space, callable, gil=None): "NOT_RPYTHON" names = callable.api_func.argnames argtypes_enum_ui = unrolling_iterable(enumerate(zip(callable.api_func.argtypes, [name.startswith("w_") for name in names]))) fatal_value = callable.api_func.restype._defl() + gil_acquire = (gil == "acquire" or gil == "around") + gil_release = (gil == "release" or gil == "around") + assert gil is None or gil_acquire or gil_release @specialize.ll() def wrapper(*args): @@ -607,6 +615,10 @@ from pypy.module.cpyext.pyobject import Reference # we hope that malloc removal removes the newtuple() that is # inserted exactly here by the varargs specializer + if gil_acquire: + after = rffi.aroundstate.after + if after: + after() rffi.stackcounter.stacks_counter += 1 llop.gc_stack_bottom(lltype.Void) # marker for trackgcroot.py retval = fatal_value @@ -678,6 +690,10 @@ print str(e) pypy_debug_catch_fatal_exception() rffi.stackcounter.stacks_counter -= 1 + if gil_release: + before = rffi.aroundstate.before + if before: + before() return retval callable._always_inline_ = 'try' wrapper.__name__ = "wrapper for %r" % (callable, ) diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py --- a/pypy/module/cpyext/pystate.py +++ b/pypy/module/cpyext/pystate.py @@ -19,7 +19,7 @@ class NoThreads(Exception): pass - at cpython_api([], PyThreadState, error=CANNOT_FAIL) + at cpython_api([], PyThreadState, error=CANNOT_FAIL, gil="release") def PyEval_SaveThread(space): """Release the global interpreter lock (if it has been created and thread support is enabled) and reset the thread state to NULL, returning the @@ -29,19 +29,15 @@ state = space.fromcache(InterpreterState) tstate = state.swap_thread_state( space, lltype.nullptr(PyThreadState.TO)) - if rffi.aroundstate.before: - rffi.aroundstate.before() return tstate - at cpython_api([PyThreadState], lltype.Void) + at cpython_api([PyThreadState], lltype.Void, gil="acquire") def PyEval_RestoreThread(space, tstate): """Acquire the global interpreter lock (if it has been created and thread support is enabled) and set the thread state to tstate, which must not be NULL. If the lock has been created, the current thread must not have acquired it, otherwise deadlock ensues. (This function is available even when thread support is disabled at compile time.)""" - if rffi.aroundstate.after: - rffi.aroundstate.after() state = space.fromcache(InterpreterState) state.swap_thread_state(space, tstate) @@ -182,17 +178,14 @@ state = space.fromcache(InterpreterState) return state.swap_thread_state(space, tstate) - at cpython_api([PyThreadState], lltype.Void) + at cpython_api([PyThreadState], lltype.Void, gil="acquire") def PyEval_AcquireThread(space, tstate): """Acquire the global interpreter lock and set the current thread state to tstate, which should not be NULL. The lock must have been created earlier. If this thread already has the lock, deadlock ensues. This function is not available when thread support is disabled at compile time.""" - if rffi.aroundstate.after: - # After external call is before entering Python - rffi.aroundstate.after() - at cpython_api([PyThreadState], lltype.Void) + at cpython_api([PyThreadState], lltype.Void, gil="release") def PyEval_ReleaseThread(space, tstate): """Reset the current thread state to NULL and release the global interpreter lock. The lock must have been created earlier and must be held by the current @@ -200,28 +193,20 @@ that it represents the current thread state --- if it isn't, a fatal error is reported. This function is not available when thread support is disabled at compile time.""" - if rffi.aroundstate.before: - # Before external call is after running Python - rffi.aroundstate.before() PyGILState_STATE = rffi.INT - at cpython_api([], PyGILState_STATE, error=CANNOT_FAIL) + at cpython_api([], PyGILState_STATE, error=CANNOT_FAIL, gil="acquire") def PyGILState_Ensure(space): # XXX XXX XXX THIS IS A VERY MINIMAL IMPLEMENTATION THAT WILL HAPPILY # DEADLOCK IF CALLED TWICE ON THE SAME THREAD, OR CRASH IF CALLED IN A # NEW THREAD. We should very carefully follow what CPython does instead. - if rffi.aroundstate.after: - # After external call is before entering Python - rffi.aroundstate.after() return rffi.cast(PyGILState_STATE, 0) - at cpython_api([PyGILState_STATE], lltype.Void) + at cpython_api([PyGILState_STATE], lltype.Void, gil="release") def PyGILState_Release(space, state): # XXX XXX XXX We should very carefully follow what CPython does instead. - if rffi.aroundstate.before: - # Before external call is after running Python - rffi.aroundstate.before() + pass @cpython_api([], PyInterpreterState, error=CANNOT_FAIL) def PyInterpreterState_Head(space): @@ -236,7 +221,8 @@ """ return lltype.nullptr(PyInterpreterState.TO) - at cpython_api([PyInterpreterState], PyThreadState, error=CANNOT_FAIL) + at cpython_api([PyInterpreterState], PyThreadState, error=CANNOT_FAIL, + gil="around") def PyThreadState_New(space, interp): """Create a new thread state object belonging to the given interpreter object. The global interpreter lock need not be held, but may be held if @@ -245,12 +231,8 @@ raise NoThreads # PyThreadState_Get will allocate a new execution context, # we need to protect gc and other globals with the GIL. - rffi.aroundstate.after() - try: - rthread.gc_thread_start() - return PyThreadState_Get(space) - finally: - rffi.aroundstate.before() + rthread.gc_thread_start() + return PyThreadState_Get(space) @cpython_api([PyThreadState], lltype.Void) def PyThreadState_Clear(space, tstate): diff --git a/pypy/module/cpyext/test/test_translate.py b/pypy/module/cpyext/test/test_translate.py --- a/pypy/module/cpyext/test/test_translate.py +++ b/pypy/module/cpyext/test/test_translate.py @@ -11,7 +11,7 @@ FT = lltype.FuncType([], lltype.Signed) FTPTR = lltype.Ptr(FT) - def make_wrapper(space, func): + def make_wrapper(space, func, gil=None): def wrapper(): return func(space) return wrapper From noreply at buildbot.pypy.org Sat Mar 14 14:50:52 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 14 Mar 2015 14:50:52 +0100 (CET) Subject: [pypy-commit] pypy release-2.5.x: merge default into branch Message-ID: <20150314135052.2DE411C00EF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: release-2.5.x Changeset: r76367:976b02d5d893 Date: 2015-03-14 14:50 +0100 http://bitbucket.org/pypy/pypy/changeset/976b02d5d893/ Log: merge default into branch diff --git a/pypy/doc/stm.rst b/pypy/doc/stm.rst --- a/pypy/doc/stm.rst +++ b/pypy/doc/stm.rst @@ -45,11 +45,10 @@ it as a drop-in replacement and multithreaded programs will run on multiple cores. -* ``pypy-stm`` does not impose any special API to the user, but it - provides a new pure Python module called `transactional_memory`_ with - features to inspect the state or debug conflicts_ that prevent - parallelization. This module can also be imported on top of a non-STM - PyPy or CPython. +* ``pypy-stm`` provides (but does not impose) a special API to the + user in the pure Python module `transaction`_. This module is based + on the lower-level module `pypystm`_, but also provides some + compatibily with non-STM PyPy's or CPython's. * Building on top of the way the GIL is removed, we will talk about `Atomic sections, Transactions, etc.: a better way to write @@ -63,9 +62,10 @@ Development is done in the branch `stmgc-c7`_. If you are only interested in trying it out, you can download a Ubuntu binary here__ -(``pypy-stm-2.3*.tar.bz2``, Ubuntu 12.04-14.04). The current version +(``pypy-stm-2.*.tar.bz2``, for Ubuntu 12.04-14.04). The current version supports four "segments", which means that it will run up to four -threads in parallel. +threads in parallel. (Development recently switched to `stmgc-c8`_, +but that is not ready for trying out yet.) To build a version from sources, you first need to compile a custom version of clang(!); we recommend downloading `llvm and clang like @@ -78,6 +78,7 @@ rpython/bin/rpython -Ojit --stm pypy/goal/targetpypystandalone.py .. _`stmgc-c7`: https://bitbucket.org/pypy/pypy/src/stmgc-c7/ +.. _`stmgc-c8`: https://bitbucket.org/pypy/pypy/src/stmgc-c8/ .. __: https://bitbucket.org/pypy/pypy/downloads/ .. __: http://clang.llvm.org/get_started.html .. __: https://bitbucket.org/pypy/stmgc/src/default/c7/llvmfix/ @@ -85,11 +86,11 @@ .. _caveats: -Current status --------------- +Current status (stmgc-c7) +------------------------- -* So far, small examples work fine, but there are still a few bugs. - We're busy fixing them as we find them; feel free to `report bugs`_. +* It seems to work fine, without crashing any more. Please `report + any crash`_ you find (or other bugs). * It runs with an overhead as low as 20% on examples like "richards". There are also other examples with higher overheads --currently up to @@ -97,8 +98,9 @@ One suspect is our partial GC implementation, see below. * Currently limited to 1.5 GB of RAM (this is just a parameter in - `core.h`__). Memory overflows are not correctly handled; they cause - segfaults. + `core.h`__ -- theoretically. In practice, increase it too much and + clang crashes again). Memory overflows are not correctly handled; + they cause segfaults. * The JIT warm-up time improved recently but is still bad. In order to produce machine code, the JIT needs to enter a special single-threaded @@ -114,11 +116,9 @@ numbers of small objects that don't immediately die (surely a common situation) suffer from these missing optimizations. -* The GC has no support for destructors: the ``__del__`` method is never - called (including on file objects, which won't be closed for you). - This is of course temporary. Also, weakrefs might appear to work a - bit strangely for now (staying alive even though ``gc.collect()``, or - even dying but then un-dying for a short time before dying again). +* Weakrefs might appear to work a bit strangely for now, sometimes + staying alive throught ``gc.collect()``, or even dying but then + un-dying for a short time before dying again. * The STM system is based on very efficient read/write barriers, which are mostly done (their placement could be improved a bit in @@ -130,7 +130,7 @@ * Very long-running processes (on the order of days) will eventually crash on an assertion error because of a non-implemented overflow of - an internal 29-bit number. + an internal 28-bit counter. .. _`report bugs`: https://bugs.pypy.org/ .. __: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/rpython/translator/stm/src_stm/stm/core.h @@ -175,29 +175,89 @@ This works by internally considering the points where a standard PyPy or CPython would release the GIL, and replacing them with the boundaries of -"transaction". Like their database equivalent, multiple transactions +"transactions". Like their database equivalent, multiple transactions can execute in parallel, but will commit in some serial order. They appear to behave as if they were completely run in this serialization order. +A better way to write parallel programs +--------------------------------------- + +In CPU-hungry programs, we can often easily identify outermost loops +over some data structure, or other repetitive algorithm, where each +"block" consists of processing a non-trivial amount of data, and where +the blocks "have a good chance" to be independent from each other. We +don't need to prove that they are actually independent: it is enough +if they are *often independent* --- or, more precisely, if we *think +they should be* often independent. + +One typical example would look like this, where the function ``func()`` +typically invokes a large amount of code:: + + for key, value in bigdict.items(): + func(key, value) + +Then you simply replace the loop with:: + + from transaction import TransactionQueue + + tr = TransactionQueue() + for key, value in bigdict.items(): + tr.add(func, key, value) + tr.run() + +This code's behavior is equivalent. Internally, the +``TransactionQueue`` object will start N threads and try to run the +``func(key, value)`` calls on all threads in parallel. But note the +difference with a regular thread-pooling library, as found in many +lower-level languages than Python: the function calls are not randomly +interleaved with each other just because they run in parallel. The +behavior did not change because we are using ``TransactionQueue``. +All the calls still *appear* to execute in some serial order. + +Now the performance should ideally be improved: if the function calls +turn out to be actually independent (most of the time), then it will +be. But if the function calls are not, then the total performance +will crawl back to the previous case, with additionally some small +penalty for the overhead. + +This case occurs typically when you see the total CPU usage remaining +low (closer to 1 than N cores). Note first that it is expected that +the CPU usage should not go much higher than 1 in the JIT warm-up +phase. You must run a program for several seconds, or for larger +programs at least one minute, to give the JIT a chance to warm up +correctly. But if CPU usage remains low even though all code is +executing in a ``TransactionQueue.run()``, then the ``PYPYSTM`` +environment variable can be used to track what is going on. + +Run your program with ``PYPYSTM=stmlog`` to produce a log file called +``stmlog``. Afterwards, use the ``pypy/stm/print_stm_log.py`` utility +to inspect the content of this log file. It produces output like +this:: + + documentation in progress! + + + Atomic sections --------------- -PyPy supports *atomic sections,* which are blocks of code which you want -to execute without "releasing the GIL". *This is experimental and may -be removed in the future.* In STM terms, this means blocks of code that -are executed while guaranteeing that the transaction is not interrupted -in the middle. +PyPy supports *atomic sections,* which are blocks of code which you +want to execute without "releasing the GIL". In STM terms, this means +blocks of code that are executed while guaranteeing that the +transaction is not interrupted in the middle. *This is experimental +and may be removed in the future* if `lock elision`_ is ever +implemented. Here is a usage example:: - with __pypy__.thread.atomic: + with transaction.atomic: assert len(lst1) == 10 x = lst1.pop(0) lst1.append(x) -In this (bad) example, we are sure that the item popped off one end of +In this example, we are sure that the item popped off one end of the list is appened again at the other end atomically. It means that another thread can run ``len(lst1)`` or ``x in lst1`` without any particular synchronization, and always see the same results, @@ -225,21 +285,22 @@ manually a transaction break just before the atomic block. This is because the boundaries of the block are not guaranteed to be the boundaries of the transaction: the latter is at least as big as the -block, but maybe bigger. Therefore, if you run a big atomic block, it +block, but may be bigger. Therefore, if you run a big atomic block, it is a good idea to break the transaction just before. This can be done -e.g. by the hack of calling ``time.sleep(0)``. (This may be fixed at +by calling ``transaction.hint_commit_soon()``. (This may be fixed at some point.) -There are also issues with the interaction of locks and atomic blocks. -This can be seen if you write to files (which have locks), including -with a ``print`` to standard output. If one thread tries to acquire a -lock while running in an atomic block, and another thread has got the -same lock, then the former may fail with a ``thread.error``. The reason -is that "waiting" for some condition to become true --while running in -an atomic block-- does not really make sense. For now you can work -around it by making sure that, say, all your prints are either in an -``atomic`` block or none of them are. (This kind of issue is -theoretically hard to solve.) +There are also issues with the interaction of regular locks and atomic +blocks. This can be seen if you write to files (which have locks), +including with a ``print`` to standard output. If one thread tries to +acquire a lock while running in an atomic block, and another thread +has got the same lock at that point, then the former may fail with a +``thread.error``. The reason is that "waiting" for some condition to +become true --while running in an atomic block-- does not really make +sense. For now you can work around it by making sure that, say, all +your prints are either in an ``atomic`` block or none of them are. +(This kind of issue is theoretically hard to solve and may be the +reason for atomic block support to eventually be removed.) Locks diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -192,7 +192,7 @@ class ApiFunction: def __init__(self, argtypes, restype, callable, error=_NOT_SPECIFIED, - c_name=None): + c_name=None, gil=None): self.argtypes = argtypes self.restype = restype self.functype = lltype.Ptr(lltype.FuncType(argtypes, restype)) @@ -208,6 +208,7 @@ assert argnames[0] == 'space' self.argnames = argnames[1:] assert len(self.argnames) == len(self.argtypes) + self.gil = gil def _freeze_(self): return True @@ -223,14 +224,15 @@ def get_wrapper(self, space): wrapper = getattr(self, '_wrapper', None) if wrapper is None: - wrapper = make_wrapper(space, self.callable) + wrapper = make_wrapper(space, self.callable, self.gil) self._wrapper = wrapper wrapper.relax_sig_check = True if self.c_name is not None: wrapper.c_name = cpyext_namespace.uniquename(self.c_name) return wrapper -def cpython_api(argtypes, restype, error=_NOT_SPECIFIED, external=True): +def cpython_api(argtypes, restype, error=_NOT_SPECIFIED, external=True, + gil=None): """ Declares a function to be exported. - `argtypes`, `restype` are lltypes and describe the function signature. @@ -240,6 +242,8 @@ SytemError. - set `external` to False to get a C function pointer, but not exported by the API headers. + - set `gil` to "acquire", "release" or "around" to acquire the GIL, + release the GIL, or both """ if isinstance(restype, lltype.Typedef): real_restype = restype.OF @@ -262,7 +266,8 @@ c_name = None else: c_name = func_name - api_function = ApiFunction(argtypes, restype, func, error, c_name=c_name) + api_function = ApiFunction(argtypes, restype, func, error, + c_name=c_name, gil=gil) func.api_func = api_function if external: @@ -594,12 +599,15 @@ pypy_debug_catch_fatal_exception = rffi.llexternal('pypy_debug_catch_fatal_exception', [], lltype.Void) # Make the wrapper for the cases (1) and (2) -def make_wrapper(space, callable): +def make_wrapper(space, callable, gil=None): "NOT_RPYTHON" names = callable.api_func.argnames argtypes_enum_ui = unrolling_iterable(enumerate(zip(callable.api_func.argtypes, [name.startswith("w_") for name in names]))) fatal_value = callable.api_func.restype._defl() + gil_acquire = (gil == "acquire" or gil == "around") + gil_release = (gil == "release" or gil == "around") + assert gil is None or gil_acquire or gil_release @specialize.ll() def wrapper(*args): @@ -607,6 +615,10 @@ from pypy.module.cpyext.pyobject import Reference # we hope that malloc removal removes the newtuple() that is # inserted exactly here by the varargs specializer + if gil_acquire: + after = rffi.aroundstate.after + if after: + after() rffi.stackcounter.stacks_counter += 1 llop.gc_stack_bottom(lltype.Void) # marker for trackgcroot.py retval = fatal_value @@ -678,6 +690,10 @@ print str(e) pypy_debug_catch_fatal_exception() rffi.stackcounter.stacks_counter -= 1 + if gil_release: + before = rffi.aroundstate.before + if before: + before() return retval callable._always_inline_ = 'try' wrapper.__name__ = "wrapper for %r" % (callable, ) diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py --- a/pypy/module/cpyext/pystate.py +++ b/pypy/module/cpyext/pystate.py @@ -19,7 +19,7 @@ class NoThreads(Exception): pass - at cpython_api([], PyThreadState, error=CANNOT_FAIL) + at cpython_api([], PyThreadState, error=CANNOT_FAIL, gil="release") def PyEval_SaveThread(space): """Release the global interpreter lock (if it has been created and thread support is enabled) and reset the thread state to NULL, returning the @@ -29,19 +29,15 @@ state = space.fromcache(InterpreterState) tstate = state.swap_thread_state( space, lltype.nullptr(PyThreadState.TO)) - if rffi.aroundstate.before: - rffi.aroundstate.before() return tstate - at cpython_api([PyThreadState], lltype.Void) + at cpython_api([PyThreadState], lltype.Void, gil="acquire") def PyEval_RestoreThread(space, tstate): """Acquire the global interpreter lock (if it has been created and thread support is enabled) and set the thread state to tstate, which must not be NULL. If the lock has been created, the current thread must not have acquired it, otherwise deadlock ensues. (This function is available even when thread support is disabled at compile time.)""" - if rffi.aroundstate.after: - rffi.aroundstate.after() state = space.fromcache(InterpreterState) state.swap_thread_state(space, tstate) @@ -182,17 +178,14 @@ state = space.fromcache(InterpreterState) return state.swap_thread_state(space, tstate) - at cpython_api([PyThreadState], lltype.Void) + at cpython_api([PyThreadState], lltype.Void, gil="acquire") def PyEval_AcquireThread(space, tstate): """Acquire the global interpreter lock and set the current thread state to tstate, which should not be NULL. The lock must have been created earlier. If this thread already has the lock, deadlock ensues. This function is not available when thread support is disabled at compile time.""" - if rffi.aroundstate.after: - # After external call is before entering Python - rffi.aroundstate.after() - at cpython_api([PyThreadState], lltype.Void) + at cpython_api([PyThreadState], lltype.Void, gil="release") def PyEval_ReleaseThread(space, tstate): """Reset the current thread state to NULL and release the global interpreter lock. The lock must have been created earlier and must be held by the current @@ -200,28 +193,20 @@ that it represents the current thread state --- if it isn't, a fatal error is reported. This function is not available when thread support is disabled at compile time.""" - if rffi.aroundstate.before: - # Before external call is after running Python - rffi.aroundstate.before() PyGILState_STATE = rffi.INT - at cpython_api([], PyGILState_STATE, error=CANNOT_FAIL) + at cpython_api([], PyGILState_STATE, error=CANNOT_FAIL, gil="acquire") def PyGILState_Ensure(space): # XXX XXX XXX THIS IS A VERY MINIMAL IMPLEMENTATION THAT WILL HAPPILY # DEADLOCK IF CALLED TWICE ON THE SAME THREAD, OR CRASH IF CALLED IN A # NEW THREAD. We should very carefully follow what CPython does instead. - if rffi.aroundstate.after: - # After external call is before entering Python - rffi.aroundstate.after() return rffi.cast(PyGILState_STATE, 0) - at cpython_api([PyGILState_STATE], lltype.Void) + at cpython_api([PyGILState_STATE], lltype.Void, gil="release") def PyGILState_Release(space, state): # XXX XXX XXX We should very carefully follow what CPython does instead. - if rffi.aroundstate.before: - # Before external call is after running Python - rffi.aroundstate.before() + pass @cpython_api([], PyInterpreterState, error=CANNOT_FAIL) def PyInterpreterState_Head(space): @@ -236,7 +221,8 @@ """ return lltype.nullptr(PyInterpreterState.TO) - at cpython_api([PyInterpreterState], PyThreadState, error=CANNOT_FAIL) + at cpython_api([PyInterpreterState], PyThreadState, error=CANNOT_FAIL, + gil="around") def PyThreadState_New(space, interp): """Create a new thread state object belonging to the given interpreter object. The global interpreter lock need not be held, but may be held if @@ -245,12 +231,8 @@ raise NoThreads # PyThreadState_Get will allocate a new execution context, # we need to protect gc and other globals with the GIL. - rffi.aroundstate.after() - try: - rthread.gc_thread_start() - return PyThreadState_Get(space) - finally: - rffi.aroundstate.before() + rthread.gc_thread_start() + return PyThreadState_Get(space) @cpython_api([PyThreadState], lltype.Void) def PyThreadState_Clear(space, tstate): diff --git a/pypy/module/cpyext/test/test_translate.py b/pypy/module/cpyext/test/test_translate.py --- a/pypy/module/cpyext/test/test_translate.py +++ b/pypy/module/cpyext/test/test_translate.py @@ -11,7 +11,7 @@ FT = lltype.FuncType([], lltype.Signed) FTPTR = lltype.Ptr(FT) - def make_wrapper(space, func): + def make_wrapper(space, func, gil=None): def wrapper(): return func(space) return wrapper From noreply at buildbot.pypy.org Sat Mar 14 16:03:37 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 14 Mar 2015 16:03:37 +0100 (CET) Subject: [pypy-commit] pypy default: When translating PyPy, freeze the file name "/lastdirname/basename.py" instead of freezing the complete translation-time path. Message-ID: <20150314150337.C7E291C0980@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76368:f3d8768ebb1c Date: 2015-03-14 16:03 +0100 http://bitbucket.org/pypy/pypy/changeset/f3d8768ebb1c/ Log: When translating PyPy, freeze the file name "/lastdirname/basename.py" instead of freezing the complete translation-time path. diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -4,7 +4,7 @@ The bytecode interpreter itself is implemented by the PyFrame class. """ -import dis, imp, struct, types, new, sys +import dis, imp, struct, types, new, sys, os from pypy.interpreter import eval from pypy.interpreter.signature import Signature @@ -128,6 +128,15 @@ if (self.magic == cpython_magic and '__pypy__' not in sys.builtin_module_names): raise Exception("CPython host codes should not be rendered") + # When translating PyPy, freeze the file name + # /lastdirname/basename.py + # instead of freezing the complete translation-time path. + filename = self.co_filename.lstrip('<').rstrip('>') + if filename.lower().endswith('.pyc'): + filename = filename[:-1] + basename = os.path.basename(filename) + lastdirname = os.path.basename(os.path.dirname(filename)) + self.co_filename = '/%s/%s' % (lastdirname, basename) co_names = property(lambda self: [self.space.unwrap(w_name) for w_name in self.co_names_w]) # for trace From noreply at buildbot.pypy.org Sat Mar 14 17:51:34 2015 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 14 Mar 2015 17:51:34 +0100 (CET) Subject: [pypy-commit] pypy optresult: setfields Message-ID: <20150314165134.B8FB91C0196@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult Changeset: r76369:d7dcc6e2867f Date: 2015-03-13 10:06 +0200 http://bitbucket.org/pypy/pypy/changeset/d7dcc6e2867f/ Log: setfields diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -241,7 +241,7 @@ d.produce_potential_short_preamble_ops(self.optimizer, sb, descr) def invalidate_descr(self, descr, lst=None): - if lst is not None: + if lst is None: lst = self.infos_to_invalidate.get(descr, None) if lst is None: return @@ -523,15 +523,20 @@ optimize_GETFIELD_GC_PURE_F = optimize_GETFIELD_GC_PURE_I def optimize_SETFIELD_GC(self, op): + #opnum = OpHelpers.getfield_pure_for_descr(op.getdescr()) + #if self.has_pure_result(opnum, [op.getarg(0)], + # op.getdescr()): + # os.write(2, '[bogus _immutable_field_ declaration: %s]\n' % + # (op.getdescr().repr_of_descr())) + # raise BogusPureField + # + opinfo = self.ensure_ptr_info_arg0(op) + self.invalidate_descr(op.getdescr()) + opinfo.setfield(op.getdescr(), self.get_box_replacement(op.getarg(1)), + self) + # clear all the caches for this descr self.emit_operation(op) return - opnum = OpHelpers.getfield_pure_for_descr(op.getdescr()) - if self.has_pure_result(opnum, [op.getarg(0)], - op.getdescr()): - os.write(2, '[bogus _immutable_field_ declaration: %s]\n' % - (op.getdescr().repr_of_descr())) - raise BogusPureField - # cf = self.field_cache(op.getdescr()) cf.do_setfield(self, op) diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py --- a/rpython/jit/metainterp/optimizeopt/info.py +++ b/rpython/jit/metainterp/optimizeopt/info.py @@ -87,7 +87,7 @@ def setfield(self, descr, op, optheap=None): if not self.is_virtual(): - if self._fields[descr.index] is not None: + if self._fields[descr.index] is None: assert optheap is not None # we should only call it with virtuals without optheap optheap.register_dirty_field(descr, self) From noreply at buildbot.pypy.org Sat Mar 14 17:51:35 2015 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 14 Mar 2015 17:51:35 +0100 (CET) Subject: [pypy-commit] pypy optresult: get back to where we were with the correct CachedField and lazy_setfield Message-ID: <20150314165135.D64E61C0196@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult Changeset: r76370:f0ac69b41467 Date: 2015-03-14 18:51 +0200 http://bitbucket.org/pypy/pypy/changeset/f0ac69b41467/ Log: get back to where we were with the correct CachedField and lazy_setfield diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -13,49 +13,71 @@ from rpython.rlib.objectmodel import we_are_translated +class BogusPureField(JitException): + pass + + class CachedField(object): def __init__(self): # Cache information for a field descr, or for an (array descr, index) # pair. It can be in one of two states: # - # 1. 'cached_fields' is a dict mapping OptValues of structs - # to OptValues of fields. All fields on-heap are - # synchronized with the values stored in the cache. + # 1. 'cached_infos' is a list listing all the infos that are + # caching this descr # # 2. we just did one setfield, which is delayed (and thus # not synchronized). 'lazy_setfield' is the delayed - # ResOperation. In this state, 'cached_fields' contains + # ResOperation. In this state, 'cached_infos' contains # out-of-date information. More precisely, the field # value pending in the ResOperation is *not* visible in - # 'cached_fields'. + # 'cached_infos'. # - Xxxx - self._cached_fields = {} - self._cached_fields_getfield_op = {} + self.cached_infos = [] self._lazy_setfield = None self._lazy_setfield_registered = False + def register_dirty_field(self, info): + self.cached_infos.append(info) + + def invalidate(self, descr): + for info in self.cached_infos: + info._fields = [None] * len(info._fields) + self.cached_infos = [] + + + def possible_aliasing(self, optheap, opinfo): + # If lazy_setfield is set and contains a setfield on a different + # structvalue, then we are annoyed, because it may point to either + # the same or a different structure at runtime. + # XXX constants? + return (self._lazy_setfield is not None + and (optheap.getptrinfo(self._lazy_setfield.getarg(0)) + is not opinfo)) + def do_setfield(self, optheap, op): # Update the state with the SETFIELD_GC/SETARRAYITEM_GC operation 'op'. - structvalue = optheap.getvalue(op.getarg(0)) - fieldvalue = optheap.getvalue(op.getarglist()[-1]) - if self.possible_aliasing(optheap, structvalue): + structinfo = optheap.ensure_ptr_info_arg0(op) + arg1 = optheap.get_box_replacement(op.getarg(1)) + if self.possible_aliasing(optheap, structinfo): + xxx self.force_lazy_setfield(optheap) assert not self.possible_aliasing(optheap, structvalue) - cached_fieldvalue = self._cached_fields.get(structvalue, None) + cached_field = structinfo.getfield(op.getdescr()) + if cached_field is not None: + cached_field = optheap.get_box_replacement(cached_field) # Hack to ensure constants are imported from the preamble - if cached_fieldvalue and fieldvalue.is_constant(): - optheap.optimizer.ensure_imported(cached_fieldvalue) - cached_fieldvalue = self._cached_fields.get(structvalue, None) + # XXX no longer necessary? + #if cached_fieldvalue and fieldvalue.is_constant(): + # optheap.optimizer.ensure_imported(cached_fieldvalue) + # cached_fieldvalue = self._cached_fields.get(structvalue, None) - if not fieldvalue.same_value(cached_fieldvalue): + if not cached_field or not cached_field.same_box(arg1): # common case: store the 'op' as lazy_setfield, and register # myself in the optheap's _lazy_setfields_and_arrayitems list self._lazy_setfield = op - if not self._lazy_setfield_registered: - optheap._lazy_setfields_and_arrayitems.append(self) - self._lazy_setfield_registered = True + #if not self._lazy_setfield_registered: + # self._lazy_setfield_registered = True else: # this is the case where the pending setfield ends up @@ -66,44 +88,16 @@ # cancelling its previous effects with no side effect. self._lazy_setfield = None - def value_updated(self, oldvalue, newvalue, exporting_state): - try: - fieldvalue = self._cached_fields[oldvalue] - except KeyError: - pass - else: - self._cached_fields[newvalue] = fieldvalue - if exporting_state: - op = self._cached_fields_getfield_op[oldvalue].clone() - op.setarg(0, newvalue.box) - self._cached_fields_getfield_op[newvalue] = op - - def possible_aliasing(self, optheap, structvalue): - # If lazy_setfield is set and contains a setfield on a different - # structvalue, then we are annoyed, because it may point to either - # the same or a different structure at runtime. - return (self._lazy_setfield is not None - and (optheap.getvalue(self._lazy_setfield.getarg(0)) - is not structvalue)) - - def getfield_from_cache(self, optheap, structvalue): + def getfield_from_cache(self, optheap, opinfo, descr): # Returns the up-to-date field's value, or None if not cached. - if self.possible_aliasing(optheap, structvalue): + if self.possible_aliasing(optheap, opinfo): self.force_lazy_setfield(optheap) if self._lazy_setfield is not None: op = self._lazy_setfield - assert optheap.getvalue(op.getarg(0)) is structvalue - return optheap.getvalue(op.getarglist()[-1]) + assert optheap.getptrinfo(op.getarg(0)) is opinfo + return optheap.get_box_replacement(op.getarg(1)) else: - return self._cached_fields.get(structvalue, None) - - def remember_field_value(self, structvalue, fieldvalue, op=None, - optimizer=None): - assert self._lazy_setfield is None - self._cached_fields[structvalue] = fieldvalue - if optimizer.exporting_state: - op = optimizer.get_op_replacement(op) - self._cached_fields_getfield_op[structvalue] = op + return opinfo.getfield(descr, optheap) def force_lazy_setfield(self, optheap, can_cache=True): op = self._lazy_setfield @@ -112,11 +106,12 @@ # Now we clear _cached_fields, because actually doing the # setfield might impact any of the stored result (because of # possible aliasing). - self.clear() + self.invalidate(op.getdescr()) self._lazy_setfield = None if optheap.postponed_op: + xxx for a in op.getarglist(): - if a is optheap.postponed_op: + if a is optheap.postponed_op.result: optheap.emit_postponed_op() break optheap.next_optimization.propagate_forward(op) @@ -125,69 +120,19 @@ # Once it is done, we can put at least one piece of information # back in the cache: the value of this particular structure's # field. - structvalue = optheap.getvalue(op.getarg(0)) - fieldvalue = optheap.getvalue(op.getarglist()[-1]) - self.remember_field_value(structvalue, fieldvalue, op, - optheap.optimizer) + opinfo = optheap.ensure_ptr_info_arg0(op) + opinfo.setfield(op.getdescr(), + optheap.get_box_replacement(op.getarg(1))) elif not can_cache: - self.clear() - - def clear(self): - self._cached_fields.clear() - self._cached_fields_getfield_op.clear() - - def produce_potential_short_preamble_ops(self, optimizer, shortboxes, descr): - assert optimizer.exporting_state - if self._lazy_setfield is not None: - return - for structvalue in self._cached_fields_getfield_op.keys(): - op = self._cached_fields_getfield_op[structvalue] - if not op: - continue - value = optimizer.getvalue(op.getarg(0)) - if value in optimizer.opaque_pointers: - if value.getlevel() < LEVEL_KNOWNCLASS: - continue - if op.getopnum() != rop.SETFIELD_GC and op.getopnum() != rop.GETFIELD_GC: - continue - if structvalue in self._cached_fields: - if op.getopnum() == rop.SETFIELD_GC: - result = op.getarg(1) - opnum = OpHelpers.getfield_for_descr(op.getdescr()) - getop = ResOperation(opnum, [op.getarg(0)], - op.getdescr()) - xxx - if isinstance(result, Const): - optimizer.make_constant(getop, result) - getop.is_source_op = True - shortboxes.add_potential(getop, getop, synthetic=True) - else: - getop.source_op = result - shortboxes.add_potential(result, getop, synthetic=True) - if op.getopnum() == rop.SETARRAYITEM_GC: - result = op.getarg(2) - opnum = OpHelpers.getarrayitem_for_descr(op.getdescr()) - getop = ResOperation(opnum, [op.getarg(0), op.getarg(1)], - op.getdescr()) - getop.source_op = result - xxx - if isinstance(result, Const): - xxx - optimizer.make_constant(getop, result) - shortboxes.add_potential(result, getop, synthetic=True) - elif op.type != 'v': - shortboxes.add_potential(op, op) - -class BogusPureField(JitException): - pass - + self.invalidate() class OptHeap(Optimization): """Cache repeated heap accesses""" def __init__(self): - # mapping descr -> infos to invalidate - self.infos_to_invalidate = {} + # mapping descr -> CachedField + self.cached_fields = {} + # XXXX the rest is old # cached array items: {array descr: {index: CachedField}} self.cached_arrayitems = {} # cached dict items: {dict descr: {(optval, index): box-or-const}} @@ -240,22 +185,13 @@ for index, d in submap.items(): d.produce_potential_short_preamble_ops(self.optimizer, sb, descr) - def invalidate_descr(self, descr, lst=None): - if lst is None: - lst = self.infos_to_invalidate.get(descr, None) - if lst is None: - return - for info in lst: - info.clear_cache() - del lst[:] - def register_dirty_field(self, descr, info): - self.infos_to_invalidate.setdefault(descr, []).append(info) + self.field_cache(descr).register_dirty_field(info) def clean_caches(self): del self._lazy_setfields_and_arrayitems[:] - for descr, info in self.infos_to_invalidate.iteritems(): - self.invalidate_descr(descr, info) + for descr, cf in self.cached_fields.iteritems(): + cf.invalidate(descr) self.cached_arrayitems.clear() self.cached_dict_reads.clear() @@ -447,9 +383,11 @@ assert 0, "'cf' not in cached_fields/cached_arrayitems" def force_all_lazy_setfields_and_arrayitems(self): - for cf in self._lazy_setfields_and_arrayitems: - self._assert_valid_cf(cf) + for cf in self.cached_fields.values(): cf.force_lazy_setfield(self) + #for cf in self._lazy_setfields_and_arrayitems: + # self._assert_valid_cf(cf) + # cf.force_lazy_setfield(self) def force_lazy_setfields_and_arrayitems_for_guard(self): pendingfields = [] @@ -484,6 +422,19 @@ return pendingfields def optimize_GETFIELD_GC_I(self, op): + structinfo = self.ensure_ptr_info_arg0(op) + cf = self.field_cache(op.getdescr()) + field = cf.getfield_from_cache(self, structinfo, op.getdescr()) + if field is not None: + self.make_equal_to(op, field) + return + # default case: produce the operation + self.make_nonnull(op.getarg(0)) + self.emit_operation(op) + # then remember the result of reading the field + structinfo.setfield(op.getdescr(), op, self) + + def xxx_optimize_GETFIELD_GC_I(self, op): opinfo = self.ensure_ptr_info_arg0(op) fld = opinfo.getfield(op.getdescr(), self) if fld is not None: @@ -530,13 +481,6 @@ # (op.getdescr().repr_of_descr())) # raise BogusPureField # - opinfo = self.ensure_ptr_info_arg0(op) - self.invalidate_descr(op.getdescr()) - opinfo.setfield(op.getdescr(), self.get_box_replacement(op.getarg(1)), - self) - # clear all the caches for this descr - self.emit_operation(op) - return cf = self.field_cache(op.getdescr()) cf.do_setfield(self, op) diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py --- a/rpython/jit/metainterp/optimizeopt/info.py +++ b/rpython/jit/metainterp/optimizeopt/info.py @@ -62,7 +62,7 @@ def force_box(self, op, optforce): if self.is_virtual(): op.set_forwarded(None) - optforce.emit_operation(op) + optforce._emit_operation(op) newop = optforce.getlastop() op.set_forwarded(newop) newop.set_forwarded(self) @@ -86,11 +86,6 @@ self._fields = [None] * len(self._fields) def setfield(self, descr, op, optheap=None): - if not self.is_virtual(): - if self._fields[descr.index] is None: - assert optheap is not None - # we should only call it with virtuals without optheap - optheap.register_dirty_field(descr, self) self._fields[descr.index] = op def getfield(self, descr, optheap=None): @@ -107,7 +102,7 @@ subbox = optforce.force_box(fld) setfieldop = ResOperation(rop.SETFIELD_GC, [op, subbox], descr=flddescr) - optforce.emit_operation(setfieldop) + optforce._emit_operation(setfieldop) optforce.optheap.register_dirty_field(flddescr, self) count += 1 return count @@ -151,7 +146,7 @@ setop = ResOperation(rop.SETARRAYITEM_GC, [op, ConstInt(i), subbox], descr=arraydescr) - optforce.emit_operation(setop) + optforce._emit_operation(setop) # xxxx optforce.optheap count += 1 return count @@ -197,7 +192,7 @@ setfieldop = ResOperation(rop.SETINTERIORFIELD_GC, [op, ConstInt(index), subbox], descr=flddescr) - optforce.emit_operation(setfieldop) + optforce._emit_operation(setfieldop) # XXX optforce.optheap count += 1 i += 1 @@ -226,7 +221,7 @@ info = self._get_info(descr, optheap) return info.getfield(descr) - def setfield(self, descr, op, optheap): + def setfield(self, descr, op, optheap=None): info = self._get_info(descr, optheap) info.setfield(descr, op, optheap) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -604,10 +604,10 @@ p3sub = getfield_gc_r(p3, descr=nextdescr) i3 = getfield_gc_i(p3sub, descr=valuedescr) escape_n(i3) + p1 = new_with_vtable(descr=nodesize) p2sub = new_with_vtable(descr=nodesize2) setfield_gc(p2sub, i1, descr=valuedescr) setfield_gc(p2, p2sub, descr=nextdescr) - p1 = new_with_vtable(descr=nodesize) jump(i1, p1, p2) """ # The same as test_p123_simple, but in the end the "old" p2 contains From noreply at buildbot.pypy.org Sat Mar 14 19:04:09 2015 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 14 Mar 2015 19:04:09 +0100 (CET) Subject: [pypy-commit] pypy default: C89 for visual 2008 c compiler Message-ID: <20150314180409.5D5421C0997@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r76371:fb59c90861c3 Date: 2015-03-14 20:00 +0200 http://bitbucket.org/pypy/pypy/changeset/fb59c90861c3/ Log: C89 for visual 2008 c compiler diff --git a/rpython/translator/c/src/mem.c b/rpython/translator/c/src/mem.c --- a/rpython/translator/c/src/mem.c +++ b/rpython/translator/c/src/mem.c @@ -29,9 +29,9 @@ RPY_EXTERN void pypy_debug_alloc_stop(void *addr) { + struct pypy_debug_alloc_s **p; if (!addr) return; - struct pypy_debug_alloc_s **p; for (p = &pypy_debug_alloc_list; *p; p = &((*p)->next)) if ((*p)->addr == addr) { From noreply at buildbot.pypy.org Sat Mar 14 19:04:10 2015 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 14 Mar 2015 19:04:10 +0100 (CET) Subject: [pypy-commit] pypy release-2.5.x: merge default into branch Message-ID: <20150314180410.8EE6A1C0997@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: release-2.5.x Changeset: r76372:8e24dac0b8e2 Date: 2015-03-14 20:05 +0200 http://bitbucket.org/pypy/pypy/changeset/8e24dac0b8e2/ Log: merge default into branch diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -4,7 +4,7 @@ The bytecode interpreter itself is implemented by the PyFrame class. """ -import dis, imp, struct, types, new, sys +import dis, imp, struct, types, new, sys, os from pypy.interpreter import eval from pypy.interpreter.signature import Signature @@ -128,6 +128,15 @@ if (self.magic == cpython_magic and '__pypy__' not in sys.builtin_module_names): raise Exception("CPython host codes should not be rendered") + # When translating PyPy, freeze the file name + # /lastdirname/basename.py + # instead of freezing the complete translation-time path. + filename = self.co_filename.lstrip('<').rstrip('>') + if filename.lower().endswith('.pyc'): + filename = filename[:-1] + basename = os.path.basename(filename) + lastdirname = os.path.basename(os.path.dirname(filename)) + self.co_filename = '/%s/%s' % (lastdirname, basename) co_names = property(lambda self: [self.space.unwrap(w_name) for w_name in self.co_names_w]) # for trace diff --git a/rpython/translator/c/src/mem.c b/rpython/translator/c/src/mem.c --- a/rpython/translator/c/src/mem.c +++ b/rpython/translator/c/src/mem.c @@ -29,9 +29,9 @@ RPY_EXTERN void pypy_debug_alloc_stop(void *addr) { + struct pypy_debug_alloc_s **p; if (!addr) return; - struct pypy_debug_alloc_s **p; for (p = &pypy_debug_alloc_list; *p; p = &((*p)->next)) if ((*p)->addr == addr) { From noreply at buildbot.pypy.org Sat Mar 14 19:28:43 2015 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 14 Mar 2015 19:28:43 +0100 (CET) Subject: [pypy-commit] buildbot default: remove non-funcitoning build slaves Message-ID: <20150314182843.9C2631C00EF@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r938:6a6b8405633a Date: 2015-03-14 20:30 +0200 http://bitbucket.org/pypy/buildbot/changeset/6a6b8405633a/ Log: remove non-funcitoning build slaves diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -235,7 +235,7 @@ Nightly("nightly-1-00", [ JITBENCH, # on tannit32, uses 1 core (in part exclusively) JITBENCH64, # on tannit64, uses 1 core (in part exclusively) - JITBENCH64_NEW, # on speed64, uses 1 core (in part exclusively) + #JITBENCH64_NEW, # on speed64, uses 1 core (in part exclusively) ], branch=None, hour=1, minute=0), @@ -263,11 +263,11 @@ PYPYBUILDBOT, LINUX32, LINUX64, - INDIANA32, + #INDIANA32, MACOSX32, WIN32, - WIN64, + #WIN64, APPLVLLINUX32, APPLVLLINUX64, APPLVLWIN32, @@ -280,11 +280,11 @@ JITMACOSX64, #JITMACOSX64_2, JITWIN32, - JITWIN64, - JITFREEBSD764, - JITFREEBSD864, + #JITWIN64, + #JITFREEBSD764, + #JITFREEBSD864, JITFREEBSD964, - JITINDIANA32, + #JITINDIANA32, JITONLYLINUXPPC64, JITBENCH, @@ -376,7 +376,7 @@ # the locks are acquired with fine grain inside the build }, {"name": JITBENCH64_NEW, - "slavenames": ["speed-python-64"], + "slavenames": [], "builddir": JITBENCH64_NEW, "factory": pypyJITBenchmarkFactory64_speed, "category": "benchmark-run", @@ -407,7 +407,7 @@ "category": 'win32', }, {"name": WIN64, - "slavenames": ["snakepit64"], + "slavenames": [], "builddir": WIN64, "factory": pypyOwnTestFactoryWin, "category": 'win32' @@ -427,19 +427,19 @@ 'category' : 'win32', }, {"name" : JITWIN64, - "slavenames": ["snakepit64"], + "slavenames": [], 'builddir' : JITWIN64, 'factory' : pypyJITTranslatedTestFactoryWin, 'category' : 'win32', }, {"name" : JITFREEBSD764, - "slavenames": ['headless'], + "slavenames": [], 'builddir' : JITFREEBSD764, 'factory' : pypyJITTranslatedTestFactoryFreeBSD, "category": 'freebsd64' }, {"name": JITFREEBSD864, - "slavenames": ['ananke'], + "slavenames": [], 'builddir' : JITFREEBSD864, 'factory' : pypyJITTranslatedTestFactoryFreeBSD, "category": 'freebsd64' @@ -459,13 +459,13 @@ }, # openindiana {'name': JITINDIANA32, - 'slavenames': ['jcea-openindiana-32'], + 'slavenames': [], 'builddir': JITINDIANA32, 'factory': pypyJITTranslatedTestFactoryIndiana, 'category': 'openindiana32', }, {'name': INDIANA32, - 'slavenames': ['jcea-openindiana-32'], + 'slavenames': [], 'builddir': INDIANA32, 'factory': pypyOwnTestFactoryIndiana, 'category': 'openindiana32', From noreply at buildbot.pypy.org Sat Mar 14 19:44:20 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 14 Mar 2015 19:44:20 +0100 (CET) Subject: [pypy-commit] pypy default: Add str2chararray(), copying a string into some already-allocated Message-ID: <20150314184420.0F5381C014F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76373:7941975d4a8a Date: 2015-03-14 17:43 +0100 http://bitbucket.org/pypy/pypy/changeset/7941975d4a8a/ Log: Add str2chararray(), copying a string into some already-allocated 'char[]' buffer diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py --- a/rpython/rtyper/lltypesystem/rffi.py +++ b/rpython/rtyper/lltypesystem/rffi.py @@ -794,6 +794,12 @@ else: lltype.free(cp, flavor='raw', track_allocation=False) + # str -> already-existing char[maxsize] + def str2chararray(s, array, maxsize): + ll_s = llstrtype(s) + copy_string_to_raw(ll_s, array, 0, min(len(s), maxsize)) + str2chararray._annenforceargs_ = [strtype, None, int] + # char* -> str # doesn't free char* def charp2str(cp): @@ -944,19 +950,19 @@ return (str2charp, free_charp, charp2str, get_nonmovingbuffer, free_nonmovingbuffer, alloc_buffer, str_from_buffer, keep_buffer_alive_until_here, - charp2strn, charpsize2str, + charp2strn, charpsize2str, str2chararray, ) (str2charp, free_charp, charp2str, get_nonmovingbuffer, free_nonmovingbuffer, alloc_buffer, str_from_buffer, keep_buffer_alive_until_here, - charp2strn, charpsize2str, + charp2strn, charpsize2str, str2chararray, ) = make_string_mappings(str) (unicode2wcharp, free_wcharp, wcharp2unicode, get_nonmoving_unicodebuffer, free_nonmoving_unicodebuffer, alloc_unicodebuffer, unicode_from_buffer, keep_unicodebuffer_alive_until_here, - wcharp2unicoden, wcharpsize2unicode, + wcharp2unicoden, wcharpsize2unicode, unicode2wchararray, ) = make_string_mappings(unicode) # char** diff --git a/rpython/rtyper/lltypesystem/test/test_rffi.py b/rpython/rtyper/lltypesystem/test/test_rffi.py --- a/rpython/rtyper/lltypesystem/test/test_rffi.py +++ b/rpython/rtyper/lltypesystem/test/test_rffi.py @@ -676,6 +676,23 @@ assert interpret(f, [], backendopt=True) == 43 + def test_str2chararray(self): + eci = ExternalCompilationInfo(includes=['string.h']) + strlen = llexternal('strlen', [CCHARP], SIZE_T, + compilation_info=eci) + def f(): + raw = str2charp("XxxZy") + str2chararray("abcdef", raw, 4) + assert raw[0] == 'a' + assert raw[1] == 'b' + assert raw[2] == 'c' + assert raw[3] == 'd' + assert raw[4] == 'y' + lltype.free(raw, flavor='raw') + return 0 + + assert interpret(f, []) == 0 + def test_around_extcall(self): if sys.platform == "win32": py.test.skip('No pipes on windows') From noreply at buildbot.pypy.org Sat Mar 14 19:44:21 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 14 Mar 2015 19:44:21 +0100 (CET) Subject: [pypy-commit] pypy default: Finish the support for AF_PACKET. Message-ID: <20150314184421.4EDF81C014F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76374:936a4272826a Date: 2015-03-14 18:49 +0100 http://bitbucket.org/pypy/pypy/changeset/936a4272826a/ Log: Finish the support for AF_PACKET. diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -30,7 +30,7 @@ space.wrap(addr.get_protocol()), space.wrap(addr.get_pkttype()), space.wrap(addr.get_hatype()), - space.wrap(addr.get_addr())]) + space.wrap(addr.get_haddr())]) elif rsocket.HAS_AF_UNIX and isinstance(addr, rsocket.UNIXAddress): return space.wrap(addr.get_path()) elif rsocket.HAS_AF_NETLINK and isinstance(addr, rsocket.NETLINKAddress): @@ -79,7 +79,7 @@ raise NotImplementedError # XXX Hack to seperate rpython and pypy -def addr_from_object(family, space, w_address): +def addr_from_object(family, fd, space, w_address): if family == rsocket.AF_INET: w_host, w_port = space.unpackiterable(w_address, 2) host = space.str_w(w_host) @@ -89,8 +89,9 @@ if family == rsocket.AF_INET6: pieces_w = space.unpackiterable(w_address) if not (2 <= len(pieces_w) <= 4): - raise TypeError("AF_INET6 address must be a tuple of length 2 " - "to 4, not %d" % len(pieces_w)) + raise oefmt(space.w_TypeError, + "AF_INET6 address must be a tuple of length 2 " + "to 4, not %d", len(pieces_w)) host = space.str_w(pieces_w[0]) port = space.int_w(pieces_w[1]) port = make_ushort_port(space, port) @@ -105,6 +106,28 @@ if rsocket.HAS_AF_NETLINK and family == rsocket.AF_NETLINK: w_pid, w_groups = space.unpackiterable(w_address, 2) return rsocket.NETLINKAddress(space.uint_w(w_pid), space.uint_w(w_groups)) + if rsocket.HAS_AF_PACKET and family == rsocket.AF_PACKET: + pieces_w = space.unpackiterable(w_address) + if not (2 <= len(pieces_w) <= 5): + raise oefmt(space.w_TypeError, + "AF_PACKET address must be a tuple of length 2 " + "to 5, not %d", len(pieces_w)) + ifname = space.str_w(pieces_w[0]) + ifindex = rsocket.PacketAddress.get_ifindex_from_ifname(fd, ifname) + protocol = space.int_w(pieces_w[1]) + if len(pieces_w) > 2: pkttype = space.int_w(pieces_w[2]) + else: pkttype = 0 + if len(pieces_w) > 3: hatype = space.int_w(pieces_w[3]) + else: hatype = 0 + if len(pieces_w) > 4: haddr = space.str_w(pieces_w[4]) + else: haddr = "" + if len(haddr) > 8: + raise OperationError(space.w_ValueError, space.wrap( + "Hardware address must be 8 bytes or less")) + if protocol < 0 or protocol > 0xfffff: + raise OperationError(space.w_OverflowError, space.wrap( + "protoNumber must be 0-65535.")) + return rsocket.PacketAddress(ifindex, protocol, pkttype, hatype, haddr) raise RSocketError("unknown address family") # XXX Hack to seperate rpython and pypy @@ -172,7 +195,8 @@ # convert an app-level object into an Address # based on the current socket's family def addr_from_object(self, space, w_address): - return addr_from_object(self.sock.family, space, w_address) + fd = intmask(self.sock.fd) + return addr_from_object(self.sock.family, fd, space, w_address) def bind_w(self, space, w_addr): """bind(address) diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -1,4 +1,4 @@ -import sys +import sys, os import py from pypy.tool.pytest.objspace import gettestobjspace from rpython.tool.udir import udir @@ -615,6 +615,28 @@ os.chdir(oldcwd) +class AppTestPacket: + def setup_class(cls): + if not hasattr(os, 'getuid') or os.getuid() != 0: + py.test.skip("AF_PACKET needs to be root for testing") + w_ok = space.appexec([], "(): import _socket; " + + "return hasattr(_socket, 'AF_PACKET')") + if not space.is_true(w_ok): + py.test.skip("no AF_PACKET on this platform") + cls.space = space + + def test_convert_between_tuple_and_sockaddr_ll(self): + import _socket + s = _socket.socket(_socket.AF_PACKET, _socket.SOCK_RAW) + assert s.getsockname() == ('', 0, 0, 0, '') + s.bind(('lo', 123)) + a, b, c, d, e = s.getsockname() + assert (a, b, c) == ('lo', 123, 0) + assert isinstance(d, int) + assert isinstance(e, str) + assert 0 <= len(e) <= 8 + + class AppTestSocketTCP: HOST = 'localhost' diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py --- a/rpython/rlib/_rsocket_rffi.py +++ b/rpython/rlib/_rsocket_rffi.py @@ -199,7 +199,7 @@ WSA_INVALID_PARAMETER WSA_NOT_ENOUGH_MEMORY WSA_OPERATION_ABORTED SIO_RCVALL SIO_KEEPALIVE_VALS -SIOCGIFNAME +SIOCGIFNAME SIOCGIFINDEX '''.split() for name in constant_names: @@ -328,7 +328,8 @@ if _HAS_AF_PACKET: CConfig.sockaddr_ll = platform.Struct('struct sockaddr_ll', - [('sll_ifindex', rffi.INT), + [('sll_family', rffi.INT), + ('sll_ifindex', rffi.INT), ('sll_protocol', rffi.INT), ('sll_pkttype', rffi.INT), ('sll_hatype', rffi.INT), diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py --- a/rpython/rlib/rsocket.py +++ b/rpython/rlib/rsocket.py @@ -200,23 +200,49 @@ family = AF_PACKET struct = _c.sockaddr_ll maxlen = minlen = sizeof(struct) + ifr_name_size = _c.ifreq.c_ifr_name.length + sll_addr_size = _c.sockaddr_ll.c_sll_addr.length + + def __init__(self, ifindex, protocol, pkttype=0, hatype=0, haddr=""): + addr = lltype.malloc(_c.sockaddr_ll, flavor='raw', zero=True, + track_allocation=False) + self.setdata(addr, PacketAddress.maxlen) + rffi.setintfield(addr, 'c_sll_family', AF_PACKET) + rffi.setintfield(addr, 'c_sll_protocol', htons(protocol)) + rffi.setintfield(addr, 'c_sll_ifindex', ifindex) + rffi.setintfield(addr, 'c_sll_pkttype', pkttype) + rffi.setintfield(addr, 'c_sll_hatype', hatype) + halen = rffi.str2chararray(haddr, + rffi.cast(rffi.CCHARP, addr.c_sll_addr), + PacketAddress.sll_addr_size) + rffi.setintfield(addr, 'c_sll_halen', halen) + + @staticmethod + def get_ifindex_from_ifname(fd, ifname): + p = lltype.malloc(_c.ifreq, flavor='raw') + iflen = rffi.str2chararray(ifname, + rffi.cast(rffi.CCHARP, p.c_ifr_name), + PacketAddress.ifr_name_size - 1) + p.c_ifr_name[iflen] = '\0' + err = _c.ioctl(fd, _c.SIOCGIFINDEX, p) + ifindex = p.c_ifr_ifindex + lltype.free(p, flavor='raw') + if err != 0: + raise RSocketError("invalid interface name") + return ifindex def get_ifname(self, fd): + ifname = "" a = self.lock(_c.sockaddr_ll) - p = lltype.malloc(_c.ifreq, flavor='raw') - rffi.setintfield(p, 'c_ifr_ifindex', - rffi.getintfield(a, 'c_sll_ifindex')) - if (_c.ioctl(fd, _c.SIOCGIFNAME, p) == 0): - # eh, the iface name is a constant length array - i = 0 - d = [] - while p.c_ifr_name[i] != '\x00' and i < len(p.c_ifr_name): - d.append(p.c_ifr_name[i]) - i += 1 - ifname = ''.join(d) - else: - ifname = "" - lltype.free(p, flavor='raw') + ifindex = rffi.getintfield(a, 'c_sll_ifindex') + if ifindex: + p = lltype.malloc(_c.ifreq, flavor='raw') + rffi.setintfield(p, 'c_ifr_ifindex', ifindex) + if (_c.ioctl(fd, _c.SIOCGIFNAME, p) == 0): + ifname = rffi.charp2strn( + rffi.cast(rffi.CCHARP, p.c_ifr_name), + PacketAddress.ifr_name_size) + lltype.free(p, flavor='raw') self.unlock() return ifname @@ -235,11 +261,11 @@ def get_hatype(self): a = self.lock(_c.sockaddr_ll) - res = bool(rffi.getintfield(a, 'c_sll_hatype')) + res = rffi.getintfield(a, 'c_sll_hatype') self.unlock() return res - def get_addr(self): + def get_haddr(self): a = self.lock(_c.sockaddr_ll) lgt = rffi.getintfield(a, 'c_sll_halen') d = [] diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py --- a/rpython/rtyper/lltypesystem/rffi.py +++ b/rpython/rtyper/lltypesystem/rffi.py @@ -796,8 +796,10 @@ # str -> already-existing char[maxsize] def str2chararray(s, array, maxsize): + length = min(len(s), maxsize) ll_s = llstrtype(s) - copy_string_to_raw(ll_s, array, 0, min(len(s), maxsize)) + copy_string_to_raw(ll_s, array, 0, length) + return length str2chararray._annenforceargs_ = [strtype, None, int] # char* -> str diff --git a/rpython/rtyper/lltypesystem/test/test_rffi.py b/rpython/rtyper/lltypesystem/test/test_rffi.py --- a/rpython/rtyper/lltypesystem/test/test_rffi.py +++ b/rpython/rtyper/lltypesystem/test/test_rffi.py @@ -682,16 +682,16 @@ compilation_info=eci) def f(): raw = str2charp("XxxZy") - str2chararray("abcdef", raw, 4) + n = str2chararray("abcdef", raw, 4) assert raw[0] == 'a' assert raw[1] == 'b' assert raw[2] == 'c' assert raw[3] == 'd' assert raw[4] == 'y' lltype.free(raw, flavor='raw') - return 0 + return n - assert interpret(f, []) == 0 + assert interpret(f, []) == 4 def test_around_extcall(self): if sys.platform == "win32": From noreply at buildbot.pypy.org Sat Mar 14 19:44:22 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 14 Mar 2015 19:44:22 +0100 (CET) Subject: [pypy-commit] pypy default: merge heads Message-ID: <20150314184422.6CC6D1C014F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76375:d4b0ff9d6c33 Date: 2015-03-14 19:44 +0100 http://bitbucket.org/pypy/pypy/changeset/d4b0ff9d6c33/ Log: merge heads diff --git a/rpython/translator/c/src/mem.c b/rpython/translator/c/src/mem.c --- a/rpython/translator/c/src/mem.c +++ b/rpython/translator/c/src/mem.c @@ -29,9 +29,9 @@ RPY_EXTERN void pypy_debug_alloc_stop(void *addr) { + struct pypy_debug_alloc_s **p; if (!addr) return; - struct pypy_debug_alloc_s **p; for (p = &pypy_debug_alloc_list; *p; p = &((*p)->next)) if ((*p)->addr == addr) { From noreply at buildbot.pypy.org Sat Mar 14 20:25:09 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Sat, 14 Mar 2015 20:25:09 +0100 (CET) Subject: [pypy-commit] pypy default: Kill unused op.getitem_key and op.getitem_idx_key Message-ID: <20150314192509.D9AED1C0196@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r76376:8e572d5558aa Date: 2015-03-14 19:01 +0000 http://bitbucket.org/pypy/pypy/changeset/8e572d5558aa/ Log: Kill unused op.getitem_key and op.getitem_idx_key diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py --- a/rpython/annotator/binaryop.py +++ b/rpython/annotator/binaryop.py @@ -132,13 +132,11 @@ impl = pair(s_c1, s_o2).getitem return read_can_only_throw(impl, s_c1, s_o2) - def getitem_idx_key((s_c1, s_o2)): + def getitem_idx((s_c1, s_o2)): impl = pair(s_c1, s_o2).getitem return impl() - getitem_idx_key.can_only_throw = _getitem_can_only_throw + getitem_idx.can_only_throw = _getitem_can_only_throw - getitem_idx = getitem_idx_key - getitem_key = getitem_idx_key class __extend__(pairtype(SomeType, SomeType), @@ -565,14 +563,10 @@ return lst1.listdef.read_item() getitem.can_only_throw = [] - getitem_key = getitem - def getitem_idx((lst1, int2)): return lst1.listdef.read_item() getitem_idx.can_only_throw = [IndexError] - getitem_idx_key = getitem_idx - def setitem((lst1, int2), s_value): lst1.listdef.mutate() lst1.listdef.generalize(s_value) @@ -588,14 +582,10 @@ return SomeChar(no_nul=str1.no_nul) getitem.can_only_throw = [] - getitem_key = getitem - def getitem_idx((str1, int2)): return SomeChar(no_nul=str1.no_nul) getitem_idx.can_only_throw = [IndexError] - getitem_idx_key = getitem_idx - def mul((str1, int2)): # xxx do we want to support this return SomeString(no_nul=str1.no_nul) @@ -604,14 +594,10 @@ return SomeUnicodeCodePoint() getitem.can_only_throw = [] - getitem_key = getitem - def getitem_idx((str1, int2)): return SomeUnicodeCodePoint() getitem_idx.can_only_throw = [IndexError] - getitem_idx_key = getitem_idx - def mul((str1, int2)): # xxx do we want to support this return SomeUnicodeString() diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py --- a/rpython/flowspace/operation.py +++ b/rpython/flowspace/operation.py @@ -422,8 +422,6 @@ add_operator('delattr', 2, dispatch=1, pyfunc=delattr) add_operator('getitem', 2, dispatch=2, pure=True) add_operator('getitem_idx', 2, dispatch=2, pure=True) -add_operator('getitem_key', 2, dispatch=2, pure=True) -add_operator('getitem_idx_key', 2, dispatch=2, pure=True) add_operator('setitem', 3, dispatch=2) add_operator('delitem', 2, dispatch=2) add_operator('getslice', 3, dispatch=1, pyfunc=do_getslice, pure=True) @@ -686,8 +684,6 @@ # the annotator tests op.getitem.canraise = [IndexError, KeyError, Exception] op.getitem_idx.canraise = [IndexError, KeyError, Exception] -op.getitem_key.canraise = [IndexError, KeyError, Exception] -op.getitem_idx_key.canraise = [IndexError, KeyError, Exception] op.setitem.canraise = [IndexError, KeyError, Exception] op.delitem.canraise = [IndexError, KeyError, Exception] op.contains.canraise = [Exception] # from an r_dict diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py --- a/rpython/flowspace/test/test_objspace.py +++ b/rpython/flowspace/test/test_objspace.py @@ -867,7 +867,7 @@ raise graph = self.codetest(f) simplify_graph(graph) - assert self.all_operations(graph) == {'getitem_idx_key': 1} + assert self.all_operations(graph) == {'getitem_idx': 1} g = lambda: None def f(c, x): @@ -877,7 +877,7 @@ g() graph = self.codetest(f) simplify_graph(graph) - assert self.all_operations(graph) == {'getitem_idx_key': 1, + assert self.all_operations(graph) == {'getitem_idx': 1, 'simple_call': 2} def f(c, x): @@ -896,7 +896,7 @@ raise graph = self.codetest(f) simplify_graph(graph) - assert self.all_operations(graph) == {'getitem_key': 1} + assert self.all_operations(graph) == {'getitem': 1} def f(c, x): try: @@ -915,7 +915,7 @@ graph = self.codetest(f) simplify_graph(graph) self.show(graph) - assert self.all_operations(graph) == {'getitem_idx_key': 1} + assert self.all_operations(graph) == {'getitem_idx': 1} def f(c, x): try: @@ -933,7 +933,7 @@ return -1 graph = self.codetest(f) simplify_graph(graph) - assert self.all_operations(graph) == {'getitem_key': 1} + assert self.all_operations(graph) == {'getitem': 1} def f(c, x): try: diff --git a/rpython/rtyper/rlist.py b/rpython/rtyper/rlist.py --- a/rpython/rtyper/rlist.py +++ b/rpython/rtyper/rlist.py @@ -268,13 +268,9 @@ v_res = hop.gendirectcall(llfn, c_func_marker, c_basegetitem, v_lst, v_index) return r_lst.recast(hop.llops, v_res) - rtype_getitem_key = rtype_getitem - def rtype_getitem_idx((r_lst, r_int), hop): return pair(r_lst, r_int).rtype_getitem(hop, checkidx=True) - rtype_getitem_idx_key = rtype_getitem_idx - def rtype_setitem((r_lst, r_int), hop): if hop.has_implicit_exception(IndexError): spec = dum_checkidx diff --git a/rpython/rtyper/rmodel.py b/rpython/rtyper/rmodel.py --- a/rpython/rtyper/rmodel.py +++ b/rpython/rtyper/rmodel.py @@ -285,11 +285,9 @@ # default implementation for checked getitems - def rtype_getitem_idx_key((r_c1, r_o1), hop): + def rtype_getitem_idx((r_c1, r_o1), hop): return pair(r_c1, r_o1).rtype_getitem(hop) - rtype_getitem_idx = rtype_getitem_idx_key - rtype_getitem_key = rtype_getitem_idx_key # ____________________________________________________________ diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py --- a/rpython/rtyper/rstr.py +++ b/rpython/rtyper/rstr.py @@ -580,13 +580,9 @@ hop.exception_cannot_occur() return hop.gendirectcall(llfn, v_str, v_index) - rtype_getitem_key = rtype_getitem - def rtype_getitem_idx((r_str, r_int), hop): return pair(r_str, r_int).rtype_getitem(hop, checkidx=True) - rtype_getitem_idx_key = rtype_getitem_idx - def rtype_mul((r_str, r_int), hop): str_repr = r_str.repr v_str, v_int = hop.inputargs(str_repr, Signed) diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py --- a/rpython/translator/simplify.py +++ b/rpython/translator/simplify.py @@ -203,8 +203,6 @@ for exit in block.exits: if exit.exitcase is IndexError: postfx.append('idx') - elif exit.exitcase is KeyError: - postfx.append('key') if postfx: Op = getattr(op, '_'.join(['getitem'] + postfx)) newop = Op(*last_op.args) From noreply at buildbot.pypy.org Sat Mar 14 21:08:56 2015 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 14 Mar 2015 21:08:56 +0100 (CET) Subject: [pypy-commit] buildbot default: refactor all invalid builds into an unused list Message-ID: <20150314200856.3CC491C03F7@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r939:98af64a63f90 Date: 2015-03-14 22:10 +0200 http://bitbucket.org/pypy/buildbot/changeset/98af64a63f90/ Log: refactor all invalid builds into an unused list diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -162,11 +162,9 @@ LINUX32 = "own-linux-x86-32" LINUX64 = "own-linux-x86-64" -INDIANA32 = "own-indiana-x86-32" MACOSX32 = "own-macosx-x86-32" WIN32 = "own-win-x86-32" -WIN64 = "own-win-x86-64" APPLVLLINUX32 = "pypy-c-app-level-linux-x86-32" APPLVLLINUX64 = "pypy-c-app-level-linux-x86-64" APPLVLWIN32 = "pypy-c-app-level-win-x86-32" @@ -179,22 +177,69 @@ JITMACOSX64 = "pypy-c-jit-macosx-x86-64" #JITMACOSX64_2 = "pypy-c-jit-macosx-x86-64-2" JITWIN32 = "pypy-c-jit-win-x86-32" -JITWIN64 = "pypy-c-jit-win-x86-64" -JITFREEBSD764 = 'pypy-c-jit-freebsd-7-x86-64' -JITFREEBSD864 = 'pypy-c-jit-freebsd-8-x86-64' -JITFREEBSD964 = 'pypy-c-jit-freebsd-9-x86-64' -JITINDIANA32 = "pypy-c-jit-indiana-x86-32" JITONLYLINUXPPC64 = "jitonly-own-linux-ppc-64" JITBENCH = "jit-benchmark-linux-x86-32" JITBENCH64 = "jit-benchmark-linux-x86-64" -JITBENCH64_NEW = 'jit-benchmark-linux-x86-64-single-run' CPYTHON_64 = "cpython-2-benchmark-x86-64" NUMPY_64 = "numpy-compatability-linux-x86-64" NUMPY_WIN = "numpy-compatability-win-x86-32" # buildbot builder PYPYBUILDBOT = 'pypy-buildbot' +JITFREEBSD964 = 'pypy-c-jit-freebsd-9-x86-64' +WIN64 = "own-win-x86-64" +INDIANA32 = "own-indiana-x86-32" +JITWIN64 = "pypy-c-jit-win-x86-64" +JITFREEBSD764 = 'pypy-c-jit-freebsd-7-x86-64' +JITFREEBSD864 = 'pypy-c-jit-freebsd-8-x86-64' +JITINDIANA32 = "pypy-c-jit-indiana-x86-32" +JITBENCH64_NEW = 'jit-benchmark-linux-x86-64-single-run' +inactive_slaves = [ + {"name": WIN64, + "slavenames": [], + "builddir": WIN64, + "factory": pypyOwnTestFactoryWin, + "category": 'win32' + }, + {'name': INDIANA32, + 'slavenames': [], + 'builddir': INDIANA32, + 'factory': pypyOwnTestFactoryIndiana, + 'category': 'openindiana32', + }, + {"name" : JITWIN64, + "slavenames": [], + 'builddir' : JITWIN64, + 'factory' : pypyJITTranslatedTestFactoryWin, + 'category' : 'win32', + }, + {"name" : JITFREEBSD764, + "slavenames": [], + 'builddir' : JITFREEBSD764, + 'factory' : pypyJITTranslatedTestFactoryFreeBSD, + "category": 'freebsd64' + }, + {"name": JITFREEBSD864, + "slavenames": [], + 'builddir' : JITFREEBSD864, + 'factory' : pypyJITTranslatedTestFactoryFreeBSD, + "category": 'freebsd64' + }, + {"name": JITBENCH64_NEW, + "slavenames": [], + "builddir": JITBENCH64_NEW, + "factory": pypyJITBenchmarkFactory64_speed, + "category": "benchmark-run", + }, + # openindiana + {'name': JITINDIANA32, + 'slavenames': [], + 'builddir': JITINDIANA32, + 'factory': pypyJITTranslatedTestFactoryIndiana, + 'category': 'openindiana32', + }, + ] extra_opts = {'xerxes': {'keepalive_interval': 15}, 'aurora': {'max_builds': 1}, 'salsa': {'max_builds': 1}, @@ -263,11 +308,9 @@ PYPYBUILDBOT, LINUX32, LINUX64, - #INDIANA32, MACOSX32, WIN32, - #WIN64, APPLVLLINUX32, APPLVLLINUX64, APPLVLWIN32, @@ -278,20 +321,22 @@ JITLINUX32, JITLINUX64, JITMACOSX64, - #JITMACOSX64_2, JITWIN32, - #JITWIN64, - #JITFREEBSD764, - #JITFREEBSD864, JITFREEBSD964, - #JITINDIANA32, JITONLYLINUXPPC64, JITBENCH, JITBENCH64, - JITBENCH64_NEW, + #JITBENCH64_NEW, NUMPY_64, NUMPY_WIN, + #INDIANA32, + #WIN64, + #JITMACOSX64_2, + #JITWIN64, + #JITFREEBSD764, + #JITFREEBSD864, + #JITINDIANA32, ] + ARM.builderNames, properties=[]), ] + ARM.schedulers, @@ -375,12 +420,6 @@ "category": "benchmark-run", # the locks are acquired with fine grain inside the build }, - {"name": JITBENCH64_NEW, - "slavenames": [], - "builddir": JITBENCH64_NEW, - "factory": pypyJITBenchmarkFactory64_speed, - "category": "benchmark-run", - }, {"name": MACOSX32, "slavenames": ["minime"], "builddir": MACOSX32, @@ -406,13 +445,7 @@ "locks": [WinSlaveLock.access('counting')], "category": 'win32', }, - {"name": WIN64, - "slavenames": [], - "builddir": WIN64, - "factory": pypyOwnTestFactoryWin, - "category": 'win32' - }, - {"name": APPLVLWIN32, + {"name": APPLVLWIN32, "slavenames": ["SalsaSalsa", "allegro_win32"], "builddir": APPLVLWIN32, "factory": pypyTranslatedAppLevelTestFactoryWin, @@ -426,24 +459,6 @@ "locks": [WinSlaveLock.access('counting')], 'category' : 'win32', }, - {"name" : JITWIN64, - "slavenames": [], - 'builddir' : JITWIN64, - 'factory' : pypyJITTranslatedTestFactoryWin, - 'category' : 'win32', - }, - {"name" : JITFREEBSD764, - "slavenames": [], - 'builddir' : JITFREEBSD764, - 'factory' : pypyJITTranslatedTestFactoryFreeBSD, - "category": 'freebsd64' - }, - {"name": JITFREEBSD864, - "slavenames": [], - 'builddir' : JITFREEBSD864, - 'factory' : pypyJITTranslatedTestFactoryFreeBSD, - "category": 'freebsd64' - }, {"name" : JITFREEBSD964, "slavenames": ['hybridlogic', 'tavendo-freebsd-9.2-amd64'], 'builddir' : JITFREEBSD964, @@ -457,19 +472,6 @@ "factory": pypyJitOnlyOwnTestFactory, "category": 'linux-ppc64', }, - # openindiana - {'name': JITINDIANA32, - 'slavenames': [], - 'builddir': JITINDIANA32, - 'factory': pypyJITTranslatedTestFactoryIndiana, - 'category': 'openindiana32', - }, - {'name': INDIANA32, - 'slavenames': [], - 'builddir': INDIANA32, - 'factory': pypyOwnTestFactoryIndiana, - 'category': 'openindiana32', - }, {'name': NUMPY_64, 'slavenames': ["tannit64"], 'builddir': NUMPY_64, From noreply at buildbot.pypy.org Sat Mar 14 21:08:57 2015 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 14 Mar 2015 21:08:57 +0100 (CET) Subject: [pypy-commit] buildbot default: define SucceedAlways class, use it to clean out tmp files, clean tmp files recursively Message-ID: <20150314200857.3D0EC1C03F7@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r940:93843c6cfb4c Date: 2015-03-14 22:10 +0200 http://bitbucket.org/pypy/buildbot/changeset/93843c6cfb4c/ Log: define SucceedAlways class, use it to clean out tmp files, clean tmp files recursively diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -170,6 +170,10 @@ d[key] = summary builder.saveYourself() +class SuccessAlways(ShellCmd): + def evaluateCommand(self, cmd): + return SUCCESS + # _______________________________________________________________ # XXX Currently the build properties got_revision and final_file_name contain # the revision number and the changeset-id, CheckGotRevision takes care to set @@ -401,8 +405,8 @@ '/D', '-' + nDays, '/c', "cmd /c rmdir /q /s @path"] else: command = ['find', Interpolate(tmp_or_crazy + pytest), '-mtime', - '+' + nDays, '-exec', 'rm', '{}', ';'] - factory.addStep(PytestCmd( + '+' + nDays, '-exec', 'rm -r', '{}', ';'] + factory.addStep(SuccessAlways( description="cleanout old test files", command = command, flunkOnFailure=False, @@ -500,8 +504,8 @@ '/D', '-' + nDays, '/c', "cmd /c rmdir /q /s @path"] else: command = ['find', Interpolate(tmp_or_crazy + pytest), '-mtime', - '+' + nDays, '-exec', 'rm', '{}', ';'] - self.addStep(PytestCmd( + '+' + nDays, '-exec', 'rm -r', '{}', ';'] + self.addStep(SuccessAlways( description="cleanout old test files", command = command, flunkOnFailure=False, From noreply at buildbot.pypy.org Sun Mar 15 06:07:04 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Sun, 15 Mar 2015 06:07:04 +0100 (CET) Subject: [pypy-commit] pypy default: Create .replace() method on Constant and Variable: make variable Message-ID: <20150315050704.484031C00EF@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r76377:f0188254b0f3 Date: 2015-03-15 05:02 +0000 http://bitbucket.org/pypy/pypy/changeset/f0188254b0f3/ Log: Create .replace() method on Constant and Variable: make variable substitution API a bit nicer. diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py --- a/rpython/flowspace/model.py +++ b/rpython/flowspace/model.py @@ -215,13 +215,12 @@ return uniqueitems([w for w in result if isinstance(w, Constant)]) def renamevariables(self, mapping): - self.inputargs = [mapping.get(a, a) for a in self.inputargs] - for op in self.operations: - op.args = [mapping.get(a, a) for a in op.args] - op.result = mapping.get(op.result, op.result) - self.exitswitch = mapping.get(self.exitswitch, self.exitswitch) + self.inputargs = [a.replace(mapping) for a in self.inputargs] + self.operations = [op.replace(mapping) for op in self.operations] + if self.exitswitch is not None: + self.exitswitch = self.exitswitch.replace(mapping) for link in self.exits: - link.args = [mapping.get(a, a) for a in link.args] + link.args = [a.replace(mapping) for a in link.args] def closeblock(self, *exits): assert self.exits == [], "block already closed" @@ -327,6 +326,8 @@ newvar.concretetype = self.concretetype return newvar + def replace(self, mapping): + return mapping.get(self, self) class Constant(Hashable): @@ -356,6 +357,9 @@ # cannot count on it not mutating at runtime! return False + def replace(self, mapping): + return self + class FSException(object): def __init__(self, w_type, w_value): @@ -431,8 +435,8 @@ ", ".join(map(repr, self.args))) def replace(self, mapping): - newargs = [mapping.get(arg, arg) for arg in self.args] - newresult = mapping.get(self.result, self.result) + newargs = [arg.replace(mapping) for arg in self.args] + newresult = self.result.replace(mapping) return type(self)(self.opname, newargs, newresult, self.offset) class Atom(object): diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py --- a/rpython/flowspace/operation.py +++ b/rpython/flowspace/operation.py @@ -76,8 +76,8 @@ self.offset = -1 def replace(self, mapping): - newargs = [mapping.get(arg, arg) for arg in self.args] - newresult = mapping.get(self.result, self.result) + newargs = [arg.replace(mapping) for arg in self.args] + newresult = self.result.replace(mapping) newop = type(self)(*newargs) newop.result = newresult newop.offset = self.offset diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py --- a/rpython/translator/simplify.py +++ b/rpython/translator/simplify.py @@ -616,7 +616,7 @@ assert len(links) == len(new_args) for link, args in zip(links, new_args): link.args = args - for block in graph.iterblocks(): + for block in entrymap: block.renamevariables(renaming) def remove_identical_vars(graph): From noreply at buildbot.pypy.org Sun Mar 15 06:07:05 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Sun, 15 Mar 2015 06:07:05 +0100 (CET) Subject: [pypy-commit] pypy default: create Link.replace() and use it Message-ID: <20150315050705.9727E1C00EF@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r76378:3400c234a4a2 Date: 2015-03-09 17:20 +0000 http://bitbucket.org/pypy/pypy/changeset/3400c234a4a2/ Log: create Link.replace() and use it diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py --- a/rpython/flowspace/model.py +++ b/rpython/flowspace/model.py @@ -140,6 +140,12 @@ newlink.llexitcase = self.llexitcase return newlink + def replace(self, mapping): + def rename(v): + if v is not None: + return v.replace(mapping) + return self.copy(rename) + def settarget(self, targetblock): assert len(self.args) == len(targetblock.inputargs), ( "output args mismatch") diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py --- a/rpython/translator/simplify.py +++ b/rpython/translator/simplify.py @@ -132,9 +132,6 @@ the block's single list of exits. """ renaming = {} - def rename(v): - return renaming.get(v, v) - for block in graph.iterblocks(): if not (block.canraise and block.exits[-1].exitcase is Exception): @@ -151,10 +148,10 @@ while len(query.exits) == 2: newrenaming = {} for lprev, ltarg in zip(exc.args, query.inputargs): - newrenaming[ltarg] = rename(lprev) + newrenaming[ltarg] = lprev.replace(renaming) op = query.operations[0] if not (op.opname in ("is_", "issubtype") and - newrenaming.get(op.args[0]) == last_exception): + op.args[0].replace(newrenaming) == last_exception): break renaming.update(newrenaming) case = query.operations[0].args[-1].value @@ -177,19 +174,16 @@ # construct the block's new exits exits = [] for case, oldlink in switches: - link = oldlink.copy(rename) + link = oldlink.replace(renaming) assert case is not None link.last_exception = last_exception link.last_exc_value = last_exc_value # make the above two variables unique renaming2 = {} - def rename2(v): - return renaming2.get(v, v) for v in link.getextravars(): renaming2[v] = Variable(v) - link = link.copy(rename2) + link = link.replace(renaming2) link.exitcase = case - link.prevblock = block exits.append(link) block.recloseblock(*(preserve + exits)) @@ -311,8 +305,6 @@ renaming = {} for vprev, vtarg in zip(link.args, link.target.inputargs): renaming[vtarg] = vprev - def rename(v): - return renaming.get(v, v) def rename_op(op): op = op.replace(renaming) # special case... @@ -326,9 +318,12 @@ link.prevblock.operations.append(rename_op(op)) exits = [] for exit in link.target.exits: - newexit = exit.copy(rename) + newexit = exit.replace(renaming) exits.append(newexit) - newexitswitch = rename(link.target.exitswitch) + if link.target.exitswitch: + newexitswitch = link.target.exitswitch.replace(renaming) + else: + newexitswitch = None link.prevblock.exitswitch = newexitswitch link.prevblock.recloseblock(*exits) if (isinstance(newexitswitch, Constant) and From noreply at buildbot.pypy.org Sun Mar 15 06:10:11 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Sun, 15 Mar 2015 06:10:11 +0100 (CET) Subject: [pypy-commit] pypy default: refactor join_blocks() Message-ID: <20150315051011.157AA1C00EF@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r76379:dc6eec95f4ed Date: 2014-11-10 18:51 +0000 http://bitbucket.org/pypy/pypy/changeset/dc6eec95f4ed/ Log: refactor join_blocks() diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py --- a/rpython/translator/simplify.py +++ b/rpython/translator/simplify.py @@ -63,28 +63,19 @@ When this happens, we need to replace the preceeding link with the following link. Arguments of the links should be updated.""" for link in list(graph.iterlinks()): - while not link.target.operations: - block1 = link.target - if block1.exitswitch is not None: - break - if not block1.exits: - break - exit = block1.exits[0] - assert block1 is not exit.target, ( - "the graph contains an empty infinite loop") - outputargs = [] - for v in exit.args: - if isinstance(v, Variable): - try: - i = block1.inputargs.index(v) - v = link.args[i] - except ValueError: - # the variable was passed implicitly to block1 - pass - outputargs.append(v) - link.args = outputargs - link.target = exit.target - # the while loop above will simplify recursively the new link + while not link.target.operations: + block1 = link.target + if block1.exitswitch is not None: + break + if not block1.exits: + break + exit = block1.exits[0] + assert block1 is not exit.target, ( + "the graph contains an empty infinite loop") + subst = dict(zip(block1.inputargs, link.args)) + link.args = [v.replace(subst) for v in exit.args] + link.target = exit.target + # the while loop above will simplify recursively the new link def transform_ovfcheck(graph): """The special function calls ovfcheck needs to From noreply at buildbot.pypy.org Sun Mar 15 10:27:01 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 15 Mar 2015 10:27:01 +0100 (CET) Subject: [pypy-commit] pypy default: Reduce comment Message-ID: <20150315092701.6EB671C0382@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76380:cbeb09d22625 Date: 2015-03-15 10:27 +0100 http://bitbucket.org/pypy/pypy/changeset/cbeb09d22625/ Log: Reduce comment diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py --- a/rpython/rlib/rsocket.py +++ b/rpython/rlib/rsocket.py @@ -5,15 +5,8 @@ a drop-in replacement for the 'socket' module. """ -# Known missing features: -# -# - address families other than AF_INET, AF_INET6, AF_UNIX, AF_PACKET -# - AF_PACKET is only supported on Linux -# - methods makefile(), -# - SSL -# -# It's unclear if makefile() and SSL support belong here or only as -# app-level code for PyPy. +# XXX this does not support yet the least common AF_xxx address families +# supported by CPython. from rpython.rlib import _rsocket_rffi as _c, jit, rgc from rpython.rlib.objectmodel import instantiate, keepalive_until_here From noreply at buildbot.pypy.org Sun Mar 15 10:34:46 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 15 Mar 2015 10:34:46 +0100 (CET) Subject: [pypy-commit] pypy default: Link to the bug report Message-ID: <20150315093446.89C531C0382@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76381:385a53dacc1c Date: 2015-03-15 10:34 +0100 http://bitbucket.org/pypy/pypy/changeset/385a53dacc1c/ Log: Link to the bug report diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py --- a/rpython/rlib/rsocket.py +++ b/rpython/rlib/rsocket.py @@ -6,7 +6,7 @@ """ # XXX this does not support yet the least common AF_xxx address families -# supported by CPython. +# supported by CPython. See http://bugs.pypy.org/issue1942 from rpython.rlib import _rsocket_rffi as _c, jit, rgc from rpython.rlib.objectmodel import instantiate, keepalive_until_here From noreply at buildbot.pypy.org Sun Mar 15 20:30:25 2015 From: noreply at buildbot.pypy.org (mattip) Date: Sun, 15 Mar 2015 20:30:25 +0100 (CET) Subject: [pypy-commit] pypy release-2.5.x: document critical fix for issue #1996 Message-ID: <20150315193025.EBE841C0382@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: release-2.5.x Changeset: r76382:2b4de661d01c Date: 2015-03-14 20:40 +0200 http://bitbucket.org/pypy/pypy/changeset/2b4de661d01c/ Log: document critical fix for issue #1996 diff --git a/pypy/doc/whatsnew-2.5.1.rst b/pypy/doc/whatsnew-2.5.1.rst --- a/pypy/doc/whatsnew-2.5.1.rst +++ b/pypy/doc/whatsnew-2.5.1.rst @@ -9,7 +9,7 @@ Non-blocking file reads sometimes raised EAGAIN even though they had buffered data waiting, fixed in b1c4fcb04a42 - +Fix a bug in cpyext in multithreded programs acquiring/releasing the GIL .. branch: vmprof From noreply at buildbot.pypy.org Sun Mar 15 20:30:27 2015 From: noreply at buildbot.pypy.org (mattip) Date: Sun, 15 Mar 2015 20:30:27 +0100 (CET) Subject: [pypy-commit] pypy default: Added tag release-2.5.1 for changeset 8e24dac0b8e2 Message-ID: <20150315193027.2FCC21C0382@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r76383:057b18ffac61 Date: 2015-03-15 21:31 +0200 http://bitbucket.org/pypy/pypy/changeset/057b18ffac61/ Log: Added tag release-2.5.1 for changeset 8e24dac0b8e2 diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -11,3 +11,4 @@ 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1 10f1b29a2bd21f837090286174a9ca030b8680b2 release-2.5.0 +8e24dac0b8e2db30d46d59f2c4daa3d4aaab7861 release-2.5.1 From noreply at buildbot.pypy.org Sun Mar 15 21:41:36 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 15 Mar 2015 21:41:36 +0100 (CET) Subject: [pypy-commit] pypy default: more doc Message-ID: <20150315204136.495471C0382@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76384:b2066f697580 Date: 2015-03-15 21:39 +0100 http://bitbucket.org/pypy/pypy/changeset/b2066f697580/ Log: more doc diff --git a/pypy/doc/stm.rst b/pypy/doc/stm.rst --- a/pypy/doc/stm.rst +++ b/pypy/doc/stm.rst @@ -216,27 +216,50 @@ behavior did not change because we are using ``TransactionQueue``. All the calls still *appear* to execute in some serial order. -Now the performance should ideally be improved: if the function calls -turn out to be actually independent (most of the time), then it will -be. But if the function calls are not, then the total performance -will crawl back to the previous case, with additionally some small -penalty for the overhead. +However, the performance typically does not increase out of the box. +In fact, it is likely to be worse at first. Typically, this is +indicated by the total CPU usage, which remains low (closer to 1 than +N cores). First note that it is expected that the CPU usage should +not go much higher than 1 in the JIT warm-up phase: you must run a +program for several seconds, or for larger programs at least one +minute, to give the JIT a chance to warm up enough. But if CPU usage +remains low even afterwards, then the ``PYPYSTM`` environment variable +can be used to track what is going on. -This case occurs typically when you see the total CPU usage remaining -low (closer to 1 than N cores). Note first that it is expected that -the CPU usage should not go much higher than 1 in the JIT warm-up -phase. You must run a program for several seconds, or for larger -programs at least one minute, to give the JIT a chance to warm up -correctly. But if CPU usage remains low even though all code is -executing in a ``TransactionQueue.run()``, then the ``PYPYSTM`` -environment variable can be used to track what is going on. +Run your program with ``PYPYSTM=logfile`` to produce a log file called +``logfile``. Afterwards, use the ``pypy/stm/print_stm_log.py`` +utility to inspect the content of this log file. It produces output +like this (sorted by amount of time lost, largest first):: -Run your program with ``PYPYSTM=stmlog`` to produce a log file called -``stmlog``. Afterwards, use the ``pypy/stm/print_stm_log.py`` utility -to inspect the content of this log file. It produces output like -this:: + 10.5s lost in aborts, 1.25s paused (12412x STM_CONTENTION_WRITE_WRITE) + File "foo.py", line 10, in f + someobj.stuff = 5 + File "bar.py", line 20, in g + someobj.other = 10 - documentation in progress! +This means that 10.5 seconds were lost running transactions that were +aborted (which caused another 1.25 seconds of lost time by pausing), +because of the reason shown in the two independent single-entry +tracebacks: one thread ran the line ``someobj.stuff = 5``, whereas +another thread concurrently ran the line ``someobj.other = 10`` on the +same object. Two writes to the same object cause a conflict, which +aborts one of the two transactions. In the example above this +occurred 12412 times. + +The two other conflict causes are ``STM_CONTENTION_INEVITABLE``, which +means that two transactions both tried to do an external operation, +like printing or reading from a socket or accessing an external array +of raw data; and ``STM_CONTENTION_WRITE_READ``, which means that one +transaction wrote to an object but the other one merely read it, not +wrote to it (in that case only the writing transaction is reported; +the location for the reads is not recorded because doing so is not +possible without a very large performance impact). + +Note that Python is a complicated language; there are a number of less +common cases that may cause conflict (of any type) where we might not +expect it at priori. In many of these cases it could be fixed; please +report any case that you don't understand. One known example so far +is creating weakrefs to objects: the new weakref is xxx From noreply at buildbot.pypy.org Sun Mar 15 21:41:37 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 15 Mar 2015 21:41:37 +0100 (CET) Subject: [pypy-commit] pypy release-2.5.x: Codename proposal (feel free to change :-) Message-ID: <20150315204137.7D7C31C0382@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: release-2.5.x Changeset: r76385:25201636ecff Date: 2015-03-15 21:40 +0100 http://bitbucket.org/pypy/pypy/changeset/25201636ecff/ Log: Codename proposal (feel free to change :-) diff --git a/pypy/doc/release-2.5.1.rst b/pypy/doc/release-2.5.1.rst --- a/pypy/doc/release-2.5.1.rst +++ b/pypy/doc/release-2.5.1.rst @@ -1,5 +1,5 @@ ============================== -PyPy 2.5.1 - XXXXXXXXXXXXXXXXX +PyPy 2.5.1 - Hakea Hookeriana ============================== We're pleased to announce PyPy 2.5.1, following on the heels of 2.5.0 From noreply at buildbot.pypy.org Sun Mar 15 21:41:38 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 15 Mar 2015 21:41:38 +0100 (CET) Subject: [pypy-commit] pypy default: merge default Message-ID: <20150315204138.9EEDE1C0382@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76386:58059c786617 Date: 2015-03-15 21:41 +0100 http://bitbucket.org/pypy/pypy/changeset/58059c786617/ Log: merge default diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -11,3 +11,4 @@ 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1 10f1b29a2bd21f837090286174a9ca030b8680b2 release-2.5.0 +8e24dac0b8e2db30d46d59f2c4daa3d4aaab7861 release-2.5.1 From noreply at buildbot.pypy.org Sun Mar 15 21:41:39 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 15 Mar 2015 21:41:39 +0100 (CET) Subject: [pypy-commit] pypy release-2.5.x: merge heads Message-ID: <20150315204139.C50891C0382@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: release-2.5.x Changeset: r76387:9cf558f55103 Date: 2015-03-15 21:41 +0100 http://bitbucket.org/pypy/pypy/changeset/9cf558f55103/ Log: merge heads diff --git a/pypy/doc/release-2.5.1.rst b/pypy/doc/release-2.5.1.rst --- a/pypy/doc/release-2.5.1.rst +++ b/pypy/doc/release-2.5.1.rst @@ -1,5 +1,5 @@ ============================== -PyPy 2.5.1 - XXXXXXXXXXXXXXXXX +PyPy 2.5.1 - Hakea Hookeriana ============================== We're pleased to announce PyPy 2.5.1, following on the heels of 2.5.0 From noreply at buildbot.pypy.org Mon Mar 16 01:16:13 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 16 Mar 2015 01:16:13 +0100 (CET) Subject: [pypy-commit] pypy py3.3: Fix test_numeric_tower. Message-ID: <20150316001613.2493C1C00EF@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3.3 Changeset: r76388:e7a9dfa4ec76 Date: 2015-03-11 16:12 +0100 http://bitbucket.org/pypy/pypy/changeset/e7a9dfa4ec76/ Log: Fix test_numeric_tower. Probably an overlook in CPython _decimal.c diff --git a/lib_pypy/_decimal.py b/lib_pypy/_decimal.py --- a/lib_pypy/_decimal.py +++ b/lib_pypy/_decimal.py @@ -750,7 +750,8 @@ with _CatchStatus(context) as (ctx, status_ptr): _mpdec.mpd_qmul(multiplied._mpd, vv, denom._mpd, ctx, status_ptr) - multiplied._mpd.exp = exp + multiplied._mpd.exp += exp # XXX probably a bug + # in _decimal.c finally: _mpdec.mpd_del(vv) From noreply at buildbot.pypy.org Mon Mar 16 01:16:14 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 16 Mar 2015 01:16:14 +0100 (CET) Subject: [pypy-commit] pypy py3.3: Refactor a bit generate_unicodedb. Message-ID: <20150316001614.5692F1C00EF@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3.3 Changeset: r76389:8144addb7b8d Date: 2015-03-13 14:48 +0100 http://bitbucket.org/pypy/pypy/changeset/8144addb7b8d/ Log: Refactor a bit generate_unicodedb. More objects. diff --git a/rpython/rlib/unicodedata/generate_unicodedb.py b/rpython/rlib/unicodedata/generate_unicodedb.py --- a/rpython/rlib/unicodedata/generate_unicodedb.py +++ b/rpython/rlib/unicodedata/generate_unicodedb.py @@ -1,6 +1,8 @@ #!/usr/bin/env python import sys, os +import itertools + sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))) MAXUNICODE = 0x10FFFF # the value of sys.maxunicode of wide Python builds @@ -18,7 +20,7 @@ def __str__(self): return repr(self) -class Unicodechar: +class UnicodeChar: def __init__(self, data=None): if data is None: return @@ -74,37 +76,74 @@ self.title = int(data[14], 16) def copy(self): - uc = Unicodechar() + uc = UnicodeChar() uc.__dict__.update(self.__dict__) return uc -def get_compat_decomposition(table, code): - if not table[code].decomposition: - return [code] - if not table[code].compat_decomp: - result = [] - for decomp in table[code].decomposition: - result.extend(get_compat_decomposition(table, decomp)) - table[code].compat_decomp = result - return table[code].compat_decomp +class UnicodeData(object): + def __init__(self): + self.table = [None] * (MAXUNICODE + 1) -def get_canonical_decomposition(table, code): - if not table[code].decomposition or table[code].isCompatibility: - return [code] - if not table[code].canonical_decomp: - result = [] - for decomp in table[code].decomposition: - result.extend(get_canonical_decomposition(table, decomp)) - table[code].canonical_decomp = result - return table[code].canonical_decomp + def add_char(self, code, char): + assert self.table[code] is None, ( + 'Multiply defined character %04X' % code) + if isinstance(char, list): + char = UnicodeChar(char) + self.table[code] = char + return char -def read_unicodedata(unicodedata_file, exclusions_file, east_asian_width_file, - unihan_file=None, linebreak_file=None, - derived_core_properties_file=None): + def all_codes(self): + return range(len(self.table)) + + def enum_chars(self): + for code in range(len(self.table)): + yield code, self.table[code] + + def get_char(self, code): + return self.table[code] + + def clone_char(self, code): + clone = self.table[code] = self.table[code].copy() + return clone + + def set_excluded(self, code): + self.table[code].excluded = True + + def set_linebreak(self, code): + self.table[code].linebreak = True + + def set_east_asian_width(self, code, width): + self.table[code].east_asian_width = width + + def add_property(self, code, p): + self.table[code].properties += (p,) + + def get_compat_decomposition(self, code): + if not self.table[code].decomposition: + return [code] + if not self.table[code].compat_decomp: + result = [] + for decomp in self.table[code].decomposition: + result.extend(self.get_compat_decomposition(decomp)) + self.table[code].compat_decomp = result + return self.table[code].compat_decomp + + def get_canonical_decomposition(self, code): + if (not self.table[code].decomposition or + self.table[code].isCompatibility): + return [code] + if not self.table[code].canonical_decomp: + result = [] + for decomp in self.table[code].decomposition: + result.extend(self.get_canonical_decomposition(decomp)) + self.table[code].canonical_decomp = result + return self.table[code].canonical_decomp + +def read_unicodedata(files): rangeFirst = {} rangeLast = {} - table = [None] * (MAXUNICODE + 1) - for line in unicodedata_file: + table = UnicodeData() + for line in files['data']: line = line.split('#', 1)[0].strip() if not line: continue @@ -119,26 +158,23 @@ rangeLast[name] = code continue code = int(data[0], 16) - u = Unicodechar(data) - assert table[code] is None, 'Multiply defined character %04X' % code - table[code] = u + table.add_char(code, data) # Collect ranges ranges = {} for name, (start, data) in rangeFirst.iteritems(): end = rangeLast[name] - unichar = Unicodechar(['0000', None] + data[2:]) - ranges[(start, end)] = unichar + ranges[(start, end)] = ['0000', None] + data[2:] # Read exclusions - for line in exclusions_file: + for line in files['exclusions']: line = line.split('#', 1)[0].strip() if not line: continue - table[int(line, 16)].excluded = True + table.set_excluded(int(line, 16)) # Read line breaks - for line in linebreak_file: + for line in files['linebreak']: line = line.split('#', 1)[0].strip() if not line: continue @@ -150,16 +186,15 @@ else: first, last = [int(c, 16) for c in data[0].split('..')] for char in range(first, last+1): - table[char].linebreak = True + table.set_linebreak(char) # Expand ranges - for (first, last), char in ranges.iteritems(): + for (first, last), data in ranges.iteritems(): for code in range(first, last + 1): - assert table[code] is None, 'Multiply defined character %04X' % code - table[code] = char + table.add_char(code, data) # Read east asian width - for line in east_asian_width_file: + for line in files['east_asian_width']: line = line.split('#', 1)[0].strip() if not line: continue @@ -167,16 +202,16 @@ if '..' in code: first, last = map(lambda x:int(x,16), code.split('..')) for code in range(first, last + 1): - uc = table[code] - if uc is None: - uc = table[code] = Unicodechar(['0000', None, - 'Cn'] + [''] * 12) + uc = table.get_char(code) + if not uc: + uc = table.add_char(code, ['0000', None, + 'Cn'] + [''] * 12) uc.east_asian_width = width else: - table[int(code, 16)].east_asian_width = width + table.set_east_asian_width(int(code, 16), width) # Read Derived Core Properties: - for line in derived_core_properties_file: + for line in files['derived_core_properties']: line = line.split('#', 1)[0].strip() if not line: continue @@ -190,27 +225,25 @@ else: chars = [int(r, 16)] for char in chars: - if not table[char]: + if not table.get_char(char): # Some properties (e.g. Default_Ignorable_Code_Point) # apply to unassigned code points; ignore them continue - table[char].properties += (p,) + table.add_property(char, p) - defaultChar = Unicodechar(['0000', None, 'Cn'] + [''] * 12) - for code in range(len(table)): - if table[code] is None: - table[code] = defaultChar + defaultChar = UnicodeChar(['0000', None, 'Cn'] + [''] * 12) + for code, char in table.enum_chars(): + if not char: + table.add_char(code, defaultChar) - extra_numeric = read_unihan(unihan_file) + extra_numeric = read_unihan(files['unihan']) for code, value in extra_numeric.iteritems(): - uc = table[code].copy() - uc.numeric = value - table[code] = uc + table.clone_char(code).numeric = value # Compute full decompositions. - for code in range(len(table)): - get_canonical_decomposition(table, code) - get_compat_decomposition(table, code) + for code, char in table.enum_chars(): + table.get_canonical_decomposition(code) + table.get_compat_decomposition(code) return table @@ -288,8 +321,8 @@ # Create the records db_records = {} - for code in range(len(table)): - char = table[code] + for code in table.all_codes(): + char = table.get_char(code) flags = 0 if char.category == "Zs" or char.bidirectional in ("WS", "B", "S"): flags |= IS_SPACE @@ -328,9 +361,11 @@ print >> outfile, '_db_pgtbl = (' pages = [] line = [] - for i in range(0, len(table), pgsize): + groups = [iter(table.enum_chars())] * pgsize + for group in itertools.izip_longest(*groups): result = [] - for char in table[i:i + pgsize]: + for code, char in group: + if not char: continue result.append(chr(db_records.index(char.db_record))) categorytbl = ''.join(result) try: @@ -380,7 +415,9 @@ import triegenerator - names = dict((table[code].name,code) for code in range(len(table)) if table[code].name) + names = dict((table.get_char(code).name, code) + for code in table.all_codes() + if table.get_char(code).name) sorted_names_codes = sorted(names.iteritems()) if base_mod is None: @@ -563,13 +600,13 @@ decimal = {} digit = {} numeric = {} - for code in range(len(table)): - if table[code].decimal is not None: - decimal[code] = table[code].decimal - if table[code].digit is not None: - digit[code] = table[code].digit - if table[code].numeric is not None: - numeric[code] = table[code].numeric + for code, char in table.enum_chars(): + if char.decimal is not None: + decimal[code] = char.decimal + if char.digit is not None: + digit[code] = char.digit + if char.numeric is not None: + numeric[code] = char.numeric writeDict(outfile, '_decimal', decimal, base_mod) writeDict(outfile, '_digit', digit, base_mod) @@ -606,13 +643,13 @@ toupper = {} tolower = {} totitle = {} - for code in range(len(table)): - if table[code].upper: - toupper[code] = table[code].upper - if table[code].lower: - tolower[code] = table[code].lower - if table[code].title: - totitle[code] = table[code].title + for code, char in table.enum_chars(): + if char.upper: + toupper[code] = char.upper + if char.lower: + tolower[code] = char.lower + if char.title: + totitle[code] = char.title writeDict(outfile, '_toupper', toupper, base_mod) writeDict(outfile, '_tolower', tolower, base_mod) writeDict(outfile, '_totitle', totitle, base_mod) @@ -646,9 +683,9 @@ ''' # Decomposition decomposition = {} - for code in range(len(table)): - if table[code].raw_decomposition: - decomposition[code] = table[code].raw_decomposition + for code, char in table.enum_chars(): + if char.raw_decomposition: + decomposition[code] = char.raw_decomposition writeDict(outfile, '_raw_decomposition', decomposition, base_mod) print >> outfile, ''' def decomposition(code): @@ -662,13 +699,12 @@ ''' # Collect the composition pairs. compositions = [] - for code in range(len(table)): - unichar = table[code] + for code, unichar in table.enum_chars(): if (not unichar.decomposition or unichar.isCompatibility or unichar.excluded or len(unichar.decomposition) != 2 or - table[unichar.decomposition[0]].combining): + table.get_char(unichar.decomposition[0]).combining): continue left, right = unichar.decomposition compositions.append((left, right, code)) @@ -680,15 +716,15 @@ print >> outfile decomposition = {} - for code in range(len(table)): - if table[code].canonical_decomp: - decomposition[code] = table[code].canonical_decomp + for code, char in table.enum_chars(): + if char.canonical_decomp: + decomposition[code] = char.canonical_decomp writeDict(outfile, '_canon_decomposition', decomposition, base_mod) decomposition = {} - for code in range(len(table)): - if table[code].compat_decomp: - decomposition[code] = table[code].compat_decomp + for code, char in table.enum_chars(): + if char.compat_decomp: + decomposition[code] = char.compat_decomp writeDict(outfile, '_compat_decomposition', decomposition, base_mod) print >> outfile, ''' def canon_decomposition(code): @@ -726,16 +762,23 @@ if options.output: outfile = open(options.output + '.py', "w") - infile = open('UnicodeData-%s.txt' % options.unidata_version) - exclusions = open('CompositionExclusions-%s.txt' % options.unidata_version) - east_asian_width = open('EastAsianWidth-%s.txt' % options.unidata_version) - unihan = open('UnihanNumeric-%s.txt' % options.unidata_version) - linebreak = open('LineBreak-%s.txt' % options.unidata_version) - derived_core_properties = open('DerivedCoreProperties-%s.txt' % - options.unidata_version) - table = read_unicodedata(infile, exclusions, east_asian_width, unihan, - linebreak, derived_core_properties) + filenames = dict( + data='UnicodeData-%(version)s.txt', + exclusions='CompositionExclusions-%(version)s.txt', + east_asian_width='EastAsianWidth-%(version)s.txt', + unihan='UnihanNumeric-%(version)s.txt', + linebreak='LineBreak-%(version)s.txt', + derived_core_properties='DerivedCoreProperties-%(version)s.txt', + name_aliases='NameAliases-%(version)s.txt', + named_sequences = 'NamedSequences-%(version)s.txt', + ) + filenames = dict((name, filename % dict(version=options.unidata_version)) + for (name, filename) in filenames.items()) + files = dict((name, open(filename)) + for (name, filename) in filenames.items()) + + table = read_unicodedata(files) print >> outfile, '# UNICODE CHARACTER DATABASE' print >> outfile, '# This file was generated with the command:' print >> outfile, '# ', ' '.join(sys.argv) From noreply at buildbot.pypy.org Mon Mar 16 01:16:21 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 16 Mar 2015 01:16:21 +0100 (CET) Subject: [pypy-commit] pypy py3.3: Add support for "named sequences" in unicode database Message-ID: <20150316001621.CEAF81C00EF@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3.3 Changeset: r76390:81b82168c663 Date: 2015-03-13 18:56 +0100 http://bitbucket.org/pypy/pypy/changeset/81b82168c663/ Log: Add support for "named sequences" in unicode database diff too long, truncating to 2000 out of 176971 lines diff --git a/pypy/module/unicodedata/interp_ucd.py b/pypy/module/unicodedata/interp_ucd.py --- a/pypy/module/unicodedata/interp_ucd.py +++ b/pypy/module/unicodedata/interp_ucd.py @@ -76,6 +76,7 @@ class UCD(W_Root): def __init__(self, unicodedb): self._lookup = unicodedb.lookup + self._lookup_named_sequence = unicodedb.lookup_named_sequence self._name = unicodedb.name self._decimal = unicodedb.decimal self._digit = unicodedb.digit @@ -108,6 +109,13 @@ except KeyError: msg = space.mod(space.wrap("undefined character name '%s'"), space.wrap(name)) raise OperationError(space.w_KeyError, msg) + + # The code may be a named sequence + sequence = self._lookup_named_sequence(code) + if sequence is not None: + # named sequences only contain UCS2 codes, no surrogates &co. + return space.wrap(sequence) + return space.wrap(code_to_unichr(code)) def name(self, space, w_unichr, w_default=None): diff --git a/pypy/module/unicodedata/test/test_unicodedata.py b/pypy/module/unicodedata/test/test_unicodedata.py --- a/pypy/module/unicodedata/test/test_unicodedata.py +++ b/pypy/module/unicodedata/test/test_unicodedata.py @@ -106,3 +106,16 @@ def test_bidirectional(self): import unicodedata raises(TypeError, unicodedata.bidirectional, 'xx') + + def test_named_sequences(self): + import unicodedata + sequences = [ + ('LATIN SMALL LETTER R WITH TILDE', '\u0072\u0303'), + ('TAMIL SYLLABLE SAI', '\u0BB8\u0BC8'), + ('TAMIL SYLLABLE MOO', '\u0BAE\u0BCB'), + ('TAMIL SYLLABLE NNOO', '\u0BA3\u0BCB'), + ('TAMIL CONSONANT KSS', '\u0B95\u0BCD\u0BB7\u0BCD'), + ] + for seqname, codepoints in sequences: + assert unicodedata.lookup(seqname) == codepoints + diff --git a/rpython/rlib/unicodedata/NameAliases-3.2.0.txt b/rpython/rlib/unicodedata/NameAliases-3.2.0.txt new file mode 100644 --- /dev/null +++ b/rpython/rlib/unicodedata/NameAliases-3.2.0.txt @@ -0,0 +1,1 @@ +# NameAliases-3.2.0.txt does not exist. diff --git a/rpython/rlib/unicodedata/NameAliases-5.2.0.txt b/rpython/rlib/unicodedata/NameAliases-5.2.0.txt new file mode 100644 --- /dev/null +++ b/rpython/rlib/unicodedata/NameAliases-5.2.0.txt @@ -0,0 +1,40 @@ +# NameAliases-5.2.0.txt +# Date: 2009-05-22, 13:05:00 PDT [KW] +# +# This file is a normative contributory data file in the +# Unicode Character Database. +# +# Copyright (c) 2005-2009 Unicode, Inc. +# For terms of use, see http://www.unicode.org/terms_of_use.html +# +# This file defines the formal name aliases for Unicode characters. +# +# For informative aliases see NamesList.txt +# +# For documentation, see NamesList.html and http://www.unicode.org/reports/tr44/ +# +# FORMAT +# +# Each line has two fields +# First field: Code point +# Second field: Alias +# +# In case multiple aliases are assigned, additional aliases +# would be provided on separate lines +# +#----------------------------------------------------------------- +01A2;LATIN CAPITAL LETTER GHA +01A3;LATIN SMALL LETTER GHA +0CDE;KANNADA LETTER LLLA +0E9D;LAO LETTER FO FON +0E9F;LAO LETTER FO FAY +0EA3;LAO LETTER RO +0EA5;LAO LETTER LO +0FD0;TIBETAN MARK BKA- SHOG GI MGO RGYAN +A015;YI SYLLABLE ITERATION MARK +FE18;PRESENTATION FORM FOR VERTICAL RIGHT WHITE LENTICULAR BRACKET +1D0C5;BYZANTINE MUSICAL SYMBOL FTHORA SKLIRON CHROMA VASIS + +# Total code points: 11 + +# EOF diff --git a/rpython/rlib/unicodedata/NameAliases-6.0.0.txt b/rpython/rlib/unicodedata/NameAliases-6.0.0.txt new file mode 100644 --- /dev/null +++ b/rpython/rlib/unicodedata/NameAliases-6.0.0.txt @@ -0,0 +1,40 @@ +# NameAliases-6.0.0.txt +# Date: 2010-05-10, 11:58:00 PDT [KW] +# +# This file is a normative contributory data file in the +# Unicode Character Database. +# +# Copyright (c) 2005-2010 Unicode, Inc. +# For terms of use, see http://www.unicode.org/terms_of_use.html +# +# This file defines the formal name aliases for Unicode characters. +# +# For informative aliases see NamesList.txt +# +# For documentation, see NamesList.html and http://www.unicode.org/reports/tr44/ +# +# FORMAT +# +# Each line has two fields +# First field: Code point +# Second field: Alias +# +# In case multiple aliases are assigned, additional aliases +# would be provided on separate lines +# +#----------------------------------------------------------------- +01A2;LATIN CAPITAL LETTER GHA +01A3;LATIN SMALL LETTER GHA +0CDE;KANNADA LETTER LLLA +0E9D;LAO LETTER FO FON +0E9F;LAO LETTER FO FAY +0EA3;LAO LETTER RO +0EA5;LAO LETTER LO +0FD0;TIBETAN MARK BKA- SHOG GI MGO RGYAN +A015;YI SYLLABLE ITERATION MARK +FE18;PRESENTATION FORM FOR VERTICAL RIGHT WHITE LENTICULAR BRACKET +1D0C5;BYZANTINE MUSICAL SYMBOL FTHORA SKLIRON CHROMA VASIS + +# Total code points: 11 + +# EOF diff --git a/rpython/rlib/unicodedata/NameAliases-6.2.0.txt b/rpython/rlib/unicodedata/NameAliases-6.2.0.txt new file mode 100644 --- /dev/null +++ b/rpython/rlib/unicodedata/NameAliases-6.2.0.txt @@ -0,0 +1,509 @@ +# NameAliases-6.2.0.txt +# Date: 2012-05-15, 18:44:00 GMT [KW] +# +# This file is a normative contributory data file in the +# Unicode Character Database. +# +# Copyright (c) 2005-2012 Unicode, Inc. +# For terms of use, see http://www.unicode.org/terms_of_use.html +# +# This file defines the formal name aliases for Unicode characters. +# +# For informative aliases see NamesList.txt +# +# The formal name aliases are divided into five types. +# +# 1. Corrections for serious problems in the character names +# 2. ISO 6429 names for C0 and C1 control functions, and other +# commonly occurring names for control codes +# 3. A few widely used alternate names for format characters +# 4. Several documented labels for C1 control code points which +# were never actually approved in any standard +# 5. Commonly occurring abbreviations (or acronyms) for control codes, +# format characters, spaces, and variation selectors +# +# The formal name aliases are part of the Unicode character namespace, which +# includes the character names and the names of named character sequences. +# The inclusion of ISO 6429 names and other commonly occurring names and +# abbreviations for control codes and format characters as formal name aliases +# is to help avoid name collisions between Unicode character names and the +# labels which commonly appear in text and/or in implementations such as regex, for +# control codes (which have no Unicode character name) or for format characters. +# +# For documentation, see NamesList.html and http://www.unicode.org/reports/tr44/ +# +# FORMAT +# +# Each line has three fields, as described here: +# +# First field: Code point +# Second field: Alias +# Third field: Type +# +# The Type labels used are: correction, control, alternate, figment, abbreviation +# +# Those Type labels can be mapped to other strings for display, if desired. +# +# In case multiple aliases are assigned, additional aliases +# are provided on separate lines. Parsers of this data file should +# take note that the same code point can (and does) occur more than once. +# +#----------------------------------------------------------------- + +0000;NULL;control +0000;NUL;abbreviation +0001;START OF HEADING;control +0001;SOH;abbreviation +0002;START OF TEXT;control +0002;STX;abbreviation +0003;END OF TEXT;control +0003;ETX;abbreviation +0004;END OF TRANSMISSION;control +0004;EOT;abbreviation +0005;ENQUIRY;control +0005;ENQ;abbreviation +0006;ACKNOWLEDGE;control +0006;ACK;abbreviation + +# Note that no formal name alias for the ISO 6429 "BELL" is +# provided for U+0007, because of the existing name collision +# with U+1F514 BELL. + +0007;ALERT;control +0007;BEL;abbreviation +0008;BACKSPACE;control +0008;BS;abbreviation +0009;CHARACTER TABULATION;control +0009;HORIZONTAL TABULATION;control +0009;HT;abbreviation +0009;TAB;abbreviation +000A;LINE FEED;control +000A;NEW LINE;control +000A;END OF LINE;control +000A;LF;abbreviation +000A;NL;abbreviation +000A;EOL;abbreviation +000B;LINE TABULATION;control +000B;VERTICAL TABULATION;control +000B;VT;abbreviation +000C;FORM FEED;control +000C;FF;abbreviation +000D;CARRIAGE RETURN;control +000D;CR;abbreviation +000E;SHIFT OUT;control +000E;LOCKING-SHIFT ONE;control +000E;SO;abbreviation +000F;SHIFT IN;control +000F;LOCKING-SHIFT ZERO;control +000F;SI;abbreviation +0010;DATA LINK ESCAPE;control +0010;DLE;abbreviation +0011;DEVICE CONTROL ONE;control +0011;DC1;abbreviation +0012;DEVICE CONTROL TWO;control +0012;DC2;abbreviation +0013;DEVICE CONTROL THREE;control +0013;DC3;abbreviation +0014;DEVICE CONTROL FOUR;control +0014;DC4;abbreviation +0015;NEGATIVE ACKNOWLEDGE;control +0015;NAK;abbreviation +0016;SYNCHRONOUS IDLE;control +0016;SYN;abbreviation +0017;END OF TRANSMISSION BLOCK;control +0017;ETB;abbreviation +0018;CANCEL;control +0018;CAN;abbreviation +0019;END OF MEDIUM;control +0019;EOM;abbreviation +001A;SUBSTITUTE;control +001A;SUB;abbreviation +001B;ESCAPE;control +001B;ESC;abbreviation +001C;INFORMATION SEPARATOR FOUR;control +001C;FILE SEPARATOR;control +001C;FS;abbreviation +001D;INFORMATION SEPARATOR THREE;control +001D;GROUP SEPARATOR;control +001D;GS;abbreviation +001E;INFORMATION SEPARATOR TWO;control +001E;RECORD SEPARATOR;control +001E;RS;abbreviation +001F;INFORMATION SEPARATOR ONE;control +001F;UNIT SEPARATOR;control +001F;US;abbreviation +0020;SP;abbreviation +007F;DELETE;control +007F;DEL;abbreviation +0080;PADDING CHARACTER;figment +0080;PAD;abbreviation +0081;HIGH OCTET PRESET;figment +0081;HOP;abbreviation +0082;BREAK PERMITTED HERE;control +0082;BPH;abbreviation +0083;NO BREAK HERE;control +0083;NBH;abbreviation +0084;INDEX;control +0084;IND;abbreviation +0085;NEXT LINE;control +0085;NEL;abbreviation +0086;START OF SELECTED AREA;control +0086;SSA;abbreviation +0087;END OF SELECTED AREA;control +0087;ESA;abbreviation +0088;CHARACTER TABULATION SET;control +0088;HORIZONTAL TABULATION SET;control +0088;HTS;abbreviation +0089;CHARACTER TABULATION WITH JUSTIFICATION;control +0089;HORIZONTAL TABULATION WITH JUSTIFICATION;control +0089;HTJ;abbreviation +008A;LINE TABULATION SET;control +008A;VERTICAL TABULATION SET;control +008A;VTS;abbreviation +008B;PARTIAL LINE FORWARD;control +008B;PARTIAL LINE DOWN;control +008B;PLD;abbreviation +008C;PARTIAL LINE BACKWARD;control +008C;PARTIAL LINE UP;control +008C;PLU;abbreviation +008D;REVERSE LINE FEED;control +008D;REVERSE INDEX;control +008D;RI;abbreviation +008E;SINGLE SHIFT TWO;control +008E;SINGLE-SHIFT-2;control +008E;SS2;abbreviation +008F;SINGLE SHIFT THREE;control +008F;SINGLE-SHIFT-3;control +008F;SS3;abbreviation +0090;DEVICE CONTROL STRING;control +0090;DCS;abbreviation +0091;PRIVATE USE ONE;control +0091;PRIVATE USE-1;control +0091;PU1;abbreviation +0092;PRIVATE USE TWO;control +0092;PRIVATE USE-2;control +0092;PU2;abbreviation +0093;SET TRANSMIT STATE;control +0093;STS;abbreviation +0094;CANCEL CHARACTER;control +0094;CCH;abbreviation +0095;MESSAGE WAITING;control +0095;MW;abbreviation +0096;START OF GUARDED AREA;control +0096;START OF PROTECTED AREA;control +0096;SPA;abbreviation +0097;END OF GUARDED AREA;control +0097;END OF PROTECTED AREA;control +0097;EPA;abbreviation +0098;START OF STRING;control +0098;SOS;abbreviation +0099;SINGLE GRAPHIC CHARACTER INTRODUCER;figment +0099;SGC;abbreviation +009A;SINGLE CHARACTER INTRODUCER;control +009A;SCI;abbreviation +009B;CONTROL SEQUENCE INTRODUCER;control +009B;CSI;abbreviation +009C;STRING TERMINATOR;control +009C;ST;abbreviation +009D;OPERATING SYSTEM COMMAND;control +009D;OSC;abbreviation +009E;PRIVACY MESSAGE;control +009E;PM;abbreviation +009F;APPLICATION PROGRAM COMMAND;control +009F;APC;abbreviation +00A0;NBSP;abbreviation +00AD;SHY;abbreviation +01A2;LATIN CAPITAL LETTER GHA;correction +01A3;LATIN SMALL LETTER GHA;correction +034F;CGJ;abbreviation +0709;SYRIAC SUBLINEAR COLON SKEWED LEFT;correction +0CDE;KANNADA LETTER LLLA;correction +0E9D;LAO LETTER FO FON;correction +0E9F;LAO LETTER FO FAY;correction +0EA3;LAO LETTER RO;correction +0EA5;LAO LETTER LO;correction +0FD0;TIBETAN MARK BKA- SHOG GI MGO RGYAN;correction +180B;FVS1;abbreviation +180C;FVS2;abbreviation +180D;FVS3;abbreviation +180E;MVS;abbreviation +200B;ZWSP;abbreviation +200C;ZWNJ;abbreviation +200D;ZWJ;abbreviation +200E;LRM;abbreviation +200F;RLM;abbreviation +202A;LRE;abbreviation +202B;RLE;abbreviation +202C;PDF;abbreviation +202D;LRO;abbreviation +202E;RLO;abbreviation +202F;NNBSP;abbreviation +205F;MMSP;abbreviation +2060;WJ;abbreviation +2118;WEIERSTRASS ELLIPTIC FUNCTION;correction +2448;MICR ON US SYMBOL;correction +2449;MICR DASH SYMBOL;correction +A015;YI SYLLABLE ITERATION MARK;correction +FE18;PRESENTATION FORM FOR VERTICAL RIGHT WHITE LENTICULAR BRACKET;correction +FE00;VS1;abbreviation +FE01;VS2;abbreviation +FE02;VS3;abbreviation +FE03;VS4;abbreviation +FE04;VS5;abbreviation +FE05;VS6;abbreviation +FE06;VS7;abbreviation +FE07;VS8;abbreviation +FE08;VS9;abbreviation +FE09;VS10;abbreviation +FE0A;VS11;abbreviation +FE0B;VS12;abbreviation +FE0C;VS13;abbreviation +FE0D;VS14;abbreviation +FE0E;VS15;abbreviation +FE0F;VS16;abbreviation +FEFF;BYTE ORDER MARK;alternate +FEFF;BOM;abbreviation +FEFF;ZWNBSP;abbreviation +1D0C5;BYZANTINE MUSICAL SYMBOL FTHORA SKLIRON CHROMA VASIS;correction +E0100;VS17;abbreviation +E0101;VS18;abbreviation +E0102;VS19;abbreviation +E0103;VS20;abbreviation +E0104;VS21;abbreviation +E0105;VS22;abbreviation +E0106;VS23;abbreviation +E0107;VS24;abbreviation +E0108;VS25;abbreviation +E0109;VS26;abbreviation +E010A;VS27;abbreviation +E010B;VS28;abbreviation +E010C;VS29;abbreviation +E010D;VS30;abbreviation +E010E;VS31;abbreviation +E010F;VS32;abbreviation +E0110;VS33;abbreviation +E0111;VS34;abbreviation +E0112;VS35;abbreviation +E0113;VS36;abbreviation +E0114;VS37;abbreviation +E0115;VS38;abbreviation +E0116;VS39;abbreviation +E0117;VS40;abbreviation +E0118;VS41;abbreviation +E0119;VS42;abbreviation +E011A;VS43;abbreviation +E011B;VS44;abbreviation +E011C;VS45;abbreviation +E011D;VS46;abbreviation +E011E;VS47;abbreviation +E011F;VS48;abbreviation +E0120;VS49;abbreviation +E0121;VS50;abbreviation +E0122;VS51;abbreviation +E0123;VS52;abbreviation +E0124;VS53;abbreviation +E0125;VS54;abbreviation +E0126;VS55;abbreviation +E0127;VS56;abbreviation +E0128;VS57;abbreviation +E0129;VS58;abbreviation +E012A;VS59;abbreviation +E012B;VS60;abbreviation +E012C;VS61;abbreviation +E012D;VS62;abbreviation +E012E;VS63;abbreviation +E012F;VS64;abbreviation +E0130;VS65;abbreviation +E0131;VS66;abbreviation +E0132;VS67;abbreviation +E0133;VS68;abbreviation +E0134;VS69;abbreviation +E0135;VS70;abbreviation +E0136;VS71;abbreviation +E0137;VS72;abbreviation +E0138;VS73;abbreviation +E0139;VS74;abbreviation +E013A;VS75;abbreviation +E013B;VS76;abbreviation +E013C;VS77;abbreviation +E013D;VS78;abbreviation +E013E;VS79;abbreviation +E013F;VS80;abbreviation +E0140;VS81;abbreviation +E0141;VS82;abbreviation +E0142;VS83;abbreviation +E0143;VS84;abbreviation +E0144;VS85;abbreviation +E0145;VS86;abbreviation +E0146;VS87;abbreviation +E0147;VS88;abbreviation +E0148;VS89;abbreviation +E0149;VS90;abbreviation +E014A;VS91;abbreviation +E014B;VS92;abbreviation +E014C;VS93;abbreviation +E014D;VS94;abbreviation +E014E;VS95;abbreviation +E014F;VS96;abbreviation +E0150;VS97;abbreviation +E0151;VS98;abbreviation +E0152;VS99;abbreviation +E0153;VS100;abbreviation +E0154;VS101;abbreviation +E0155;VS102;abbreviation +E0156;VS103;abbreviation +E0157;VS104;abbreviation +E0158;VS105;abbreviation +E0159;VS106;abbreviation +E015A;VS107;abbreviation +E015B;VS108;abbreviation +E015C;VS109;abbreviation +E015D;VS110;abbreviation +E015E;VS111;abbreviation +E015F;VS112;abbreviation +E0160;VS113;abbreviation +E0161;VS114;abbreviation +E0162;VS115;abbreviation +E0163;VS116;abbreviation +E0164;VS117;abbreviation +E0165;VS118;abbreviation +E0166;VS119;abbreviation +E0167;VS120;abbreviation +E0168;VS121;abbreviation +E0169;VS122;abbreviation +E016A;VS123;abbreviation +E016B;VS124;abbreviation +E016C;VS125;abbreviation +E016D;VS126;abbreviation +E016E;VS127;abbreviation +E016F;VS128;abbreviation +E0170;VS129;abbreviation +E0171;VS130;abbreviation +E0172;VS131;abbreviation +E0173;VS132;abbreviation +E0174;VS133;abbreviation +E0175;VS134;abbreviation +E0176;VS135;abbreviation +E0177;VS136;abbreviation +E0178;VS137;abbreviation +E0179;VS138;abbreviation +E017A;VS139;abbreviation +E017B;VS140;abbreviation +E017C;VS141;abbreviation +E017D;VS142;abbreviation +E017E;VS143;abbreviation +E017F;VS144;abbreviation +E0180;VS145;abbreviation +E0181;VS146;abbreviation +E0182;VS147;abbreviation +E0183;VS148;abbreviation +E0184;VS149;abbreviation +E0185;VS150;abbreviation +E0186;VS151;abbreviation +E0187;VS152;abbreviation +E0188;VS153;abbreviation +E0189;VS154;abbreviation +E018A;VS155;abbreviation +E018B;VS156;abbreviation +E018C;VS157;abbreviation +E018D;VS158;abbreviation +E018E;VS159;abbreviation +E018F;VS160;abbreviation +E0190;VS161;abbreviation +E0191;VS162;abbreviation +E0192;VS163;abbreviation +E0193;VS164;abbreviation +E0194;VS165;abbreviation +E0195;VS166;abbreviation +E0196;VS167;abbreviation +E0197;VS168;abbreviation +E0198;VS169;abbreviation +E0199;VS170;abbreviation +E019A;VS171;abbreviation +E019B;VS172;abbreviation +E019C;VS173;abbreviation +E019D;VS174;abbreviation +E019E;VS175;abbreviation +E019F;VS176;abbreviation +E01A0;VS177;abbreviation +E01A1;VS178;abbreviation +E01A2;VS179;abbreviation +E01A3;VS180;abbreviation +E01A4;VS181;abbreviation +E01A5;VS182;abbreviation +E01A6;VS183;abbreviation +E01A7;VS184;abbreviation +E01A8;VS185;abbreviation +E01A9;VS186;abbreviation +E01AA;VS187;abbreviation +E01AB;VS188;abbreviation +E01AC;VS189;abbreviation +E01AD;VS190;abbreviation +E01AE;VS191;abbreviation +E01AF;VS192;abbreviation +E01B0;VS193;abbreviation +E01B1;VS194;abbreviation +E01B2;VS195;abbreviation +E01B3;VS196;abbreviation +E01B4;VS197;abbreviation +E01B5;VS198;abbreviation +E01B6;VS199;abbreviation +E01B7;VS200;abbreviation +E01B8;VS201;abbreviation +E01B9;VS202;abbreviation +E01BA;VS203;abbreviation +E01BB;VS204;abbreviation +E01BC;VS205;abbreviation +E01BD;VS206;abbreviation +E01BE;VS207;abbreviation +E01BF;VS208;abbreviation +E01C0;VS209;abbreviation +E01C1;VS210;abbreviation +E01C2;VS211;abbreviation +E01C3;VS212;abbreviation +E01C4;VS213;abbreviation +E01C5;VS214;abbreviation +E01C6;VS215;abbreviation +E01C7;VS216;abbreviation +E01C8;VS217;abbreviation +E01C9;VS218;abbreviation +E01CA;VS219;abbreviation +E01CB;VS220;abbreviation +E01CC;VS221;abbreviation +E01CD;VS222;abbreviation +E01CE;VS223;abbreviation +E01CF;VS224;abbreviation +E01D0;VS225;abbreviation +E01D1;VS226;abbreviation +E01D2;VS227;abbreviation +E01D3;VS228;abbreviation +E01D4;VS229;abbreviation +E01D5;VS230;abbreviation +E01D6;VS231;abbreviation +E01D7;VS232;abbreviation +E01D8;VS233;abbreviation +E01D9;VS234;abbreviation +E01DA;VS235;abbreviation +E01DB;VS236;abbreviation +E01DC;VS237;abbreviation +E01DD;VS238;abbreviation +E01DE;VS239;abbreviation +E01DF;VS240;abbreviation +E01E0;VS241;abbreviation +E01E1;VS242;abbreviation +E01E2;VS243;abbreviation +E01E3;VS244;abbreviation +E01E4;VS245;abbreviation +E01E5;VS246;abbreviation +E01E6;VS247;abbreviation +E01E7;VS248;abbreviation +E01E8;VS249;abbreviation +E01E9;VS250;abbreviation +E01EA;VS251;abbreviation +E01EB;VS252;abbreviation +E01EC;VS253;abbreviation +E01ED;VS254;abbreviation +E01EE;VS255;abbreviation +E01EF;VS256;abbreviation + +# EOF diff --git a/rpython/rlib/unicodedata/NamedSequences-3.2.0.txt b/rpython/rlib/unicodedata/NamedSequences-3.2.0.txt new file mode 100644 --- /dev/null +++ b/rpython/rlib/unicodedata/NamedSequences-3.2.0.txt @@ -0,0 +1,1 @@ +# NamedSequences-3.2.0.txt does not exist. diff --git a/rpython/rlib/unicodedata/NamedSequences-5.2.0.txt b/rpython/rlib/unicodedata/NamedSequences-5.2.0.txt new file mode 100644 --- /dev/null +++ b/rpython/rlib/unicodedata/NamedSequences-5.2.0.txt @@ -0,0 +1,448 @@ +# NamedSequences-5.2.0.txt +# Date: 2009-09-14, 12:44:00 PDT [KW] +# +# Unicode Character Database +# Copyright (c) 1991-2009 Unicode, Inc. +# For terms of use, see http://www.unicode.org/terms_of_use.html +# For documentation, see http://www.unicode.org/reports/tr44/ +# +# Format: +# Name of Sequence; Code Point Sequence for USI +# +# Note: The order of entries in this file is not significant. +# However, entries are generally in script order corresponding +# to block order in the Unicode Standard, to make it easier +# to find entries in the list. + +# ================================================ + +LATIN CAPITAL LETTER A WITH MACRON AND GRAVE;0100 0300 +LATIN SMALL LETTER A WITH MACRON AND GRAVE;0101 0300 +LATIN CAPITAL LETTER E WITH VERTICAL LINE BELOW;0045 0329 +LATIN SMALL LETTER E WITH VERTICAL LINE BELOW;0065 0329 +LATIN CAPITAL LETTER E WITH VERTICAL LINE BELOW AND GRAVE;00C8 0329 +LATIN SMALL LETTER E WITH VERTICAL LINE BELOW AND GRAVE;00E8 0329 +LATIN CAPITAL LETTER E WITH VERTICAL LINE BELOW AND ACUTE;00C9 0329 +LATIN SMALL LETTER E WITH VERTICAL LINE BELOW AND ACUTE;00E9 0329 +LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND MACRON;00CA 0304 +LATIN SMALL LETTER E WITH CIRCUMFLEX AND MACRON;00EA 0304 +LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND CARON;00CA 030C +LATIN SMALL LETTER E WITH CIRCUMFLEX AND CARON;00EA 030C +LATIN CAPITAL LETTER I WITH MACRON AND GRAVE;012A 0300 +LATIN SMALL LETTER I WITH MACRON AND GRAVE;012B 0300 +LATIN SMALL LETTER I WITH DOT ABOVE AND ACUTE;0069 0307 0301 +LATIN SMALL LETTER NG WITH TILDE ABOVE;006E 0360 0067 +LATIN CAPITAL LETTER O WITH VERTICAL LINE BELOW;004F 0329 +LATIN SMALL LETTER O WITH VERTICAL LINE BELOW;006F 0329 +LATIN CAPITAL LETTER O WITH VERTICAL LINE BELOW AND GRAVE;00D2 0329 +LATIN SMALL LETTER O WITH VERTICAL LINE BELOW AND GRAVE;00F2 0329 +LATIN CAPITAL LETTER O WITH VERTICAL LINE BELOW AND ACUTE;00D3 0329 +LATIN SMALL LETTER O WITH VERTICAL LINE BELOW AND ACUTE;00F3 0329 +LATIN CAPITAL LETTER S WITH VERTICAL LINE BELOW;0053 0329 +LATIN SMALL LETTER S WITH VERTICAL LINE BELOW;0073 0329 +LATIN CAPITAL LETTER U WITH MACRON AND GRAVE;016A 0300 +LATIN SMALL LETTER U WITH MACRON AND GRAVE;016B 0300 + +# Additions for Lithuanian. Provisional 2006-05-18, Approved 2007-10-19 + +LATIN CAPITAL LETTER A WITH OGONEK AND ACUTE;0104 0301 +LATIN SMALL LETTER A WITH OGONEK AND ACUTE;0105 0301 +LATIN CAPITAL LETTER A WITH OGONEK AND TILDE;0104 0303 +LATIN SMALL LETTER A WITH OGONEK AND TILDE;0105 0303 +LATIN CAPITAL LETTER E WITH OGONEK AND ACUTE;0118 0301 +LATIN SMALL LETTER E WITH OGONEK AND ACUTE;0119 0301 +LATIN CAPITAL LETTER E WITH OGONEK AND TILDE;0118 0303 +LATIN SMALL LETTER E WITH OGONEK AND TILDE;0119 0303 +LATIN CAPITAL LETTER E WITH DOT ABOVE AND ACUTE;0116 0301 +LATIN SMALL LETTER E WITH DOT ABOVE AND ACUTE;0117 0301 +LATIN CAPITAL LETTER E WITH DOT ABOVE AND TILDE;0116 0303 +LATIN SMALL LETTER E WITH DOT ABOVE AND TILDE;0117 0303 +LATIN SMALL LETTER I WITH DOT ABOVE AND GRAVE;0069 0307 0300 +LATIN SMALL LETTER I WITH DOT ABOVE AND TILDE;0069 0307 0303 +LATIN CAPITAL LETTER I WITH OGONEK AND ACUTE;012E 0301 +LATIN SMALL LETTER I WITH OGONEK AND DOT ABOVE AND ACUTE;012F 0307 0301 +LATIN CAPITAL LETTER I WITH OGONEK AND TILDE;012E 0303 +LATIN SMALL LETTER I WITH OGONEK AND DOT ABOVE AND TILDE;012F 0307 0303 +LATIN CAPITAL LETTER J WITH TILDE;004A 0303 +LATIN SMALL LETTER J WITH DOT ABOVE AND TILDE;006A 0307 0303 +LATIN CAPITAL LETTER L WITH TILDE;004C 0303 +LATIN SMALL LETTER L WITH TILDE;006C 0303 +LATIN CAPITAL LETTER M WITH TILDE;004D 0303 +LATIN SMALL LETTER M WITH TILDE;006D 0303 +LATIN CAPITAL LETTER R WITH TILDE;0052 0303 +LATIN SMALL LETTER R WITH TILDE;0072 0303 +LATIN CAPITAL LETTER U WITH OGONEK AND ACUTE;0172 0301 +LATIN SMALL LETTER U WITH OGONEK AND ACUTE;0173 0301 +LATIN CAPITAL LETTER U WITH OGONEK AND TILDE;0172 0303 +LATIN SMALL LETTER U WITH OGONEK AND TILDE;0173 0303 +LATIN CAPITAL LETTER U WITH MACRON AND ACUTE;016A 0301 +LATIN SMALL LETTER U WITH MACRON AND ACUTE;016B 0301 +LATIN CAPITAL LETTER U WITH MACRON AND TILDE;016A 0303 +LATIN SMALL LETTER U WITH MACRON AND TILDE;016B 0303 + +# Additions for Tamil. Provisional 2008-02-08, Approved 2009-08-14 +# +# A visual display of the Tamil named sequences is available +# in the documentation for Unicode 5.2. See: +# http://www.unicode.org/versions/Unicode5.2.0/ + +TAMIL CONSONANT K; 0B95 0BCD +TAMIL CONSONANT NG; 0B99 0BCD +TAMIL CONSONANT C; 0B9A 0BCD +TAMIL CONSONANT NY; 0B9E 0BCD +TAMIL CONSONANT TT; 0B9F 0BCD +TAMIL CONSONANT NN; 0BA3 0BCD +TAMIL CONSONANT T; 0BA4 0BCD +TAMIL CONSONANT N; 0BA8 0BCD +TAMIL CONSONANT P; 0BAA 0BCD +TAMIL CONSONANT M; 0BAE 0BCD +TAMIL CONSONANT Y; 0BAF 0BCD +TAMIL CONSONANT R; 0BB0 0BCD +TAMIL CONSONANT L; 0BB2 0BCD +TAMIL CONSONANT V; 0BB5 0BCD +TAMIL CONSONANT LLL;0BB4 0BCD +TAMIL CONSONANT LL; 0BB3 0BCD +TAMIL CONSONANT RR; 0BB1 0BCD +TAMIL CONSONANT NNN;0BA9 0BCD +TAMIL CONSONANT J; 0B9C 0BCD +TAMIL CONSONANT SH; 0BB6 0BCD +TAMIL CONSONANT SS; 0BB7 0BCD +TAMIL CONSONANT S; 0BB8 0BCD +TAMIL CONSONANT H; 0BB9 0BCD +TAMIL CONSONANT KSS;0B95 0BCD 0BB7 0BCD + +TAMIL SYLLABLE KAA; 0B95 0BBE +TAMIL SYLLABLE KI; 0B95 0BBF +TAMIL SYLLABLE KII; 0B95 0BC0 +TAMIL SYLLABLE KU; 0B95 0BC1 +TAMIL SYLLABLE KUU; 0B95 0BC2 +TAMIL SYLLABLE KE; 0B95 0BC6 +TAMIL SYLLABLE KEE; 0B95 0BC7 +TAMIL SYLLABLE KAI; 0B95 0BC8 +TAMIL SYLLABLE KO; 0B95 0BCA +TAMIL SYLLABLE KOO; 0B95 0BCB +TAMIL SYLLABLE KAU; 0B95 0BCC + +TAMIL SYLLABLE NGAA; 0B99 0BBE +TAMIL SYLLABLE NGI; 0B99 0BBF +TAMIL SYLLABLE NGII; 0B99 0BC0 +TAMIL SYLLABLE NGU; 0B99 0BC1 +TAMIL SYLLABLE NGUU; 0B99 0BC2 +TAMIL SYLLABLE NGE; 0B99 0BC6 +TAMIL SYLLABLE NGEE; 0B99 0BC7 +TAMIL SYLLABLE NGAI; 0B99 0BC8 +TAMIL SYLLABLE NGO; 0B99 0BCA +TAMIL SYLLABLE NGOO; 0B99 0BCB +TAMIL SYLLABLE NGAU; 0B99 0BCC + +TAMIL SYLLABLE CAA; 0B9A 0BBE +TAMIL SYLLABLE CI; 0B9A 0BBF +TAMIL SYLLABLE CII; 0B9A 0BC0 +TAMIL SYLLABLE CU; 0B9A 0BC1 +TAMIL SYLLABLE CUU; 0B9A 0BC2 +TAMIL SYLLABLE CE; 0B9A 0BC6 +TAMIL SYLLABLE CEE; 0B9A 0BC7 +TAMIL SYLLABLE CAI; 0B9A 0BC8 +TAMIL SYLLABLE CO; 0B9A 0BCA +TAMIL SYLLABLE COO; 0B9A 0BCB +TAMIL SYLLABLE CAU; 0B9A 0BCC + +TAMIL SYLLABLE NYAA; 0B9E 0BBE +TAMIL SYLLABLE NYI; 0B9E 0BBF +TAMIL SYLLABLE NYII; 0B9E 0BC0 +TAMIL SYLLABLE NYU; 0B9E 0BC1 +TAMIL SYLLABLE NYUU; 0B9E 0BC2 +TAMIL SYLLABLE NYE; 0B9E 0BC6 +TAMIL SYLLABLE NYEE; 0B9E 0BC7 +TAMIL SYLLABLE NYAI; 0B9E 0BC8 +TAMIL SYLLABLE NYO; 0B9E 0BCA +TAMIL SYLLABLE NYOO; 0B9E 0BCB +TAMIL SYLLABLE NYAU; 0B9E 0BCC + +TAMIL SYLLABLE TTAA; 0B9F 0BBE +TAMIL SYLLABLE TTI; 0B9F 0BBF +TAMIL SYLLABLE TTII; 0B9F 0BC0 +TAMIL SYLLABLE TTU; 0B9F 0BC1 +TAMIL SYLLABLE TTUU; 0B9F 0BC2 +TAMIL SYLLABLE TTE; 0B9F 0BC6 +TAMIL SYLLABLE TTEE; 0B9F 0BC7 +TAMIL SYLLABLE TTAI; 0B9F 0BC8 +TAMIL SYLLABLE TTO; 0B9F 0BCA +TAMIL SYLLABLE TTOO; 0B9F 0BCB +TAMIL SYLLABLE TTAU; 0B9F 0BCC + +TAMIL SYLLABLE NNAA; 0BA3 0BBE +TAMIL SYLLABLE NNI; 0BA3 0BBF +TAMIL SYLLABLE NNII; 0BA3 0BC0 +TAMIL SYLLABLE NNU; 0BA3 0BC1 +TAMIL SYLLABLE NNUU; 0BA3 0BC2 +TAMIL SYLLABLE NNE; 0BA3 0BC6 +TAMIL SYLLABLE NNEE; 0BA3 0BC7 +TAMIL SYLLABLE NNAI; 0BA3 0BC8 +TAMIL SYLLABLE NNO; 0BA3 0BCA +TAMIL SYLLABLE NNOO; 0BA3 0BCB +TAMIL SYLLABLE NNAU; 0BA3 0BCC + +TAMIL SYLLABLE TAA; 0BA4 0BBE +TAMIL SYLLABLE TI; 0BA4 0BBF +TAMIL SYLLABLE TII; 0BA4 0BC0 +TAMIL SYLLABLE TU; 0BA4 0BC1 +TAMIL SYLLABLE TUU; 0BA4 0BC2 +TAMIL SYLLABLE TE; 0BA4 0BC6 +TAMIL SYLLABLE TEE; 0BA4 0BC7 +TAMIL SYLLABLE TAI; 0BA4 0BC8 +TAMIL SYLLABLE TO; 0BA4 0BCA +TAMIL SYLLABLE TOO; 0BA4 0BCB +TAMIL SYLLABLE TAU; 0BA4 0BCC + +TAMIL SYLLABLE NAA; 0BA8 0BBE +TAMIL SYLLABLE NI; 0BA8 0BBF +TAMIL SYLLABLE NII; 0BA8 0BC0 +TAMIL SYLLABLE NU; 0BA8 0BC1 +TAMIL SYLLABLE NUU; 0BA8 0BC2 +TAMIL SYLLABLE NE; 0BA8 0BC6 +TAMIL SYLLABLE NEE; 0BA8 0BC7 +TAMIL SYLLABLE NAI; 0BA8 0BC8 +TAMIL SYLLABLE NO; 0BA8 0BCA +TAMIL SYLLABLE NOO; 0BA8 0BCB +TAMIL SYLLABLE NAU; 0BA8 0BCC + +TAMIL SYLLABLE PAA; 0BAA 0BBE +TAMIL SYLLABLE PI; 0BAA 0BBF +TAMIL SYLLABLE PII; 0BAA 0BC0 +TAMIL SYLLABLE PU; 0BAA 0BC1 +TAMIL SYLLABLE PUU; 0BAA 0BC2 +TAMIL SYLLABLE PE; 0BAA 0BC6 +TAMIL SYLLABLE PEE; 0BAA 0BC7 +TAMIL SYLLABLE PAI; 0BAA 0BC8 +TAMIL SYLLABLE PO; 0BAA 0BCA +TAMIL SYLLABLE POO; 0BAA 0BCB +TAMIL SYLLABLE PAU; 0BAA 0BCC + +TAMIL SYLLABLE MAA; 0BAE 0BBE +TAMIL SYLLABLE MI; 0BAE 0BBF +TAMIL SYLLABLE MII; 0BAE 0BC0 +TAMIL SYLLABLE MU; 0BAE 0BC1 +TAMIL SYLLABLE MUU; 0BAE 0BC2 +TAMIL SYLLABLE ME; 0BAE 0BC6 +TAMIL SYLLABLE MEE; 0BAE 0BC7 +TAMIL SYLLABLE MAI; 0BAE 0BC8 +TAMIL SYLLABLE MO; 0BAE 0BCA +TAMIL SYLLABLE MOO; 0BAE 0BCB +TAMIL SYLLABLE MAU; 0BAE 0BCC + +TAMIL SYLLABLE YAA; 0BAF 0BBE +TAMIL SYLLABLE YI; 0BAF 0BBF +TAMIL SYLLABLE YII; 0BAF 0BC0 +TAMIL SYLLABLE YU; 0BAF 0BC1 +TAMIL SYLLABLE YUU; 0BAF 0BC2 +TAMIL SYLLABLE YE; 0BAF 0BC6 +TAMIL SYLLABLE YEE; 0BAF 0BC7 +TAMIL SYLLABLE YAI; 0BAF 0BC8 +TAMIL SYLLABLE YO; 0BAF 0BCA +TAMIL SYLLABLE YOO; 0BAF 0BCB +TAMIL SYLLABLE YAU; 0BAF 0BCC + +TAMIL SYLLABLE RAA; 0BB0 0BBE +TAMIL SYLLABLE RI; 0BB0 0BBF +TAMIL SYLLABLE RII; 0BB0 0BC0 +TAMIL SYLLABLE RU; 0BB0 0BC1 +TAMIL SYLLABLE RUU; 0BB0 0BC2 +TAMIL SYLLABLE RE; 0BB0 0BC6 +TAMIL SYLLABLE REE; 0BB0 0BC7 +TAMIL SYLLABLE RAI; 0BB0 0BC8 +TAMIL SYLLABLE RO; 0BB0 0BCA +TAMIL SYLLABLE ROO; 0BB0 0BCB +TAMIL SYLLABLE RAU; 0BB0 0BCC + +TAMIL SYLLABLE LAA; 0BB2 0BBE +TAMIL SYLLABLE LI; 0BB2 0BBF +TAMIL SYLLABLE LII; 0BB2 0BC0 +TAMIL SYLLABLE LU; 0BB2 0BC1 +TAMIL SYLLABLE LUU; 0BB2 0BC2 +TAMIL SYLLABLE LE; 0BB2 0BC6 +TAMIL SYLLABLE LEE; 0BB2 0BC7 +TAMIL SYLLABLE LAI; 0BB2 0BC8 +TAMIL SYLLABLE LO; 0BB2 0BCA +TAMIL SYLLABLE LOO; 0BB2 0BCB +TAMIL SYLLABLE LAU; 0BB2 0BCC + +TAMIL SYLLABLE VAA; 0BB5 0BBE +TAMIL SYLLABLE VI; 0BB5 0BBF +TAMIL SYLLABLE VII; 0BB5 0BC0 +TAMIL SYLLABLE VU; 0BB5 0BC1 +TAMIL SYLLABLE VUU; 0BB5 0BC2 +TAMIL SYLLABLE VE; 0BB5 0BC6 +TAMIL SYLLABLE VEE; 0BB5 0BC7 +TAMIL SYLLABLE VAI; 0BB5 0BC8 +TAMIL SYLLABLE VO; 0BB5 0BCA +TAMIL SYLLABLE VOO; 0BB5 0BCB +TAMIL SYLLABLE VAU; 0BB5 0BCC + +TAMIL SYLLABLE LLLAA; 0BB4 0BBE +TAMIL SYLLABLE LLLI; 0BB4 0BBF +TAMIL SYLLABLE LLLII; 0BB4 0BC0 +TAMIL SYLLABLE LLLU; 0BB4 0BC1 +TAMIL SYLLABLE LLLUU; 0BB4 0BC2 +TAMIL SYLLABLE LLLE; 0BB4 0BC6 +TAMIL SYLLABLE LLLEE; 0BB4 0BC7 +TAMIL SYLLABLE LLLAI; 0BB4 0BC8 +TAMIL SYLLABLE LLLO; 0BB4 0BCA +TAMIL SYLLABLE LLLOO; 0BB4 0BCB +TAMIL SYLLABLE LLLAU; 0BB4 0BCC + +TAMIL SYLLABLE LLAA; 0BB3 0BBE +TAMIL SYLLABLE LLI; 0BB3 0BBF +TAMIL SYLLABLE LLII; 0BB3 0BC0 +TAMIL SYLLABLE LLU; 0BB3 0BC1 +TAMIL SYLLABLE LLUU; 0BB3 0BC2 +TAMIL SYLLABLE LLE; 0BB3 0BC6 +TAMIL SYLLABLE LLEE; 0BB3 0BC7 +TAMIL SYLLABLE LLAI; 0BB3 0BC8 +TAMIL SYLLABLE LLO; 0BB3 0BCA +TAMIL SYLLABLE LLOO; 0BB3 0BCB +TAMIL SYLLABLE LLAU; 0BB3 0BCC + +TAMIL SYLLABLE RRAA; 0BB1 0BBE +TAMIL SYLLABLE RRI; 0BB1 0BBF +TAMIL SYLLABLE RRII; 0BB1 0BC0 +TAMIL SYLLABLE RRU; 0BB1 0BC1 +TAMIL SYLLABLE RRUU; 0BB1 0BC2 +TAMIL SYLLABLE RRE; 0BB1 0BC6 +TAMIL SYLLABLE RREE; 0BB1 0BC7 +TAMIL SYLLABLE RRAI; 0BB1 0BC8 +TAMIL SYLLABLE RRO; 0BB1 0BCA +TAMIL SYLLABLE RROO; 0BB1 0BCB +TAMIL SYLLABLE RRAU; 0BB1 0BCC + +TAMIL SYLLABLE NNNAA; 0BA9 0BBE +TAMIL SYLLABLE NNNI; 0BA9 0BBF +TAMIL SYLLABLE NNNII; 0BA9 0BC0 +TAMIL SYLLABLE NNNU; 0BA9 0BC1 +TAMIL SYLLABLE NNNUU; 0BA9 0BC2 +TAMIL SYLLABLE NNNE; 0BA9 0BC6 +TAMIL SYLLABLE NNNEE; 0BA9 0BC7 +TAMIL SYLLABLE NNNAI; 0BA9 0BC8 +TAMIL SYLLABLE NNNO; 0BA9 0BCA +TAMIL SYLLABLE NNNOO; 0BA9 0BCB +TAMIL SYLLABLE NNNAU; 0BA9 0BCC + +TAMIL SYLLABLE JAA; 0B9C 0BBE +TAMIL SYLLABLE JI; 0B9C 0BBF +TAMIL SYLLABLE JII; 0B9C 0BC0 +TAMIL SYLLABLE JU; 0B9C 0BC1 +TAMIL SYLLABLE JUU; 0B9C 0BC2 +TAMIL SYLLABLE JE; 0B9C 0BC6 +TAMIL SYLLABLE JEE; 0B9C 0BC7 +TAMIL SYLLABLE JAI; 0B9C 0BC8 +TAMIL SYLLABLE JO; 0B9C 0BCA +TAMIL SYLLABLE JOO; 0B9C 0BCB +TAMIL SYLLABLE JAU; 0B9C 0BCC + +TAMIL SYLLABLE SHAA; 0BB6 0BBE +TAMIL SYLLABLE SHI; 0BB6 0BBF +TAMIL SYLLABLE SHII; 0BB6 0BC0 +TAMIL SYLLABLE SHU; 0BB6 0BC1 +TAMIL SYLLABLE SHUU; 0BB6 0BC2 +TAMIL SYLLABLE SHE; 0BB6 0BC6 +TAMIL SYLLABLE SHEE; 0BB6 0BC7 +TAMIL SYLLABLE SHAI; 0BB6 0BC8 +TAMIL SYLLABLE SHO; 0BB6 0BCA +TAMIL SYLLABLE SHOO; 0BB6 0BCB +TAMIL SYLLABLE SHAU; 0BB6 0BCC + +TAMIL SYLLABLE SSAA; 0BB7 0BBE +TAMIL SYLLABLE SSI; 0BB7 0BBF +TAMIL SYLLABLE SSII; 0BB7 0BC0 +TAMIL SYLLABLE SSU; 0BB7 0BC1 +TAMIL SYLLABLE SSUU; 0BB7 0BC2 +TAMIL SYLLABLE SSE; 0BB7 0BC6 +TAMIL SYLLABLE SSEE; 0BB7 0BC7 +TAMIL SYLLABLE SSAI; 0BB7 0BC8 +TAMIL SYLLABLE SSO; 0BB7 0BCA +TAMIL SYLLABLE SSOO; 0BB7 0BCB +TAMIL SYLLABLE SSAU; 0BB7 0BCC + +TAMIL SYLLABLE SAA; 0BB8 0BBE +TAMIL SYLLABLE SI; 0BB8 0BBF +TAMIL SYLLABLE SII; 0BB8 0BC0 +TAMIL SYLLABLE SU; 0BB8 0BC1 +TAMIL SYLLABLE SUU; 0BB8 0BC2 +TAMIL SYLLABLE SE; 0BB8 0BC6 +TAMIL SYLLABLE SEE; 0BB8 0BC7 +TAMIL SYLLABLE SAI; 0BB8 0BC8 +TAMIL SYLLABLE SO; 0BB8 0BCA +TAMIL SYLLABLE SOO; 0BB8 0BCB +TAMIL SYLLABLE SAU; 0BB8 0BCC + +TAMIL SYLLABLE HAA; 0BB9 0BBE +TAMIL SYLLABLE HI; 0BB9 0BBF +TAMIL SYLLABLE HII; 0BB9 0BC0 +TAMIL SYLLABLE HU; 0BB9 0BC1 +TAMIL SYLLABLE HUU; 0BB9 0BC2 +TAMIL SYLLABLE HE; 0BB9 0BC6 +TAMIL SYLLABLE HEE; 0BB9 0BC7 +TAMIL SYLLABLE HAI; 0BB9 0BC8 +TAMIL SYLLABLE HO; 0BB9 0BCA +TAMIL SYLLABLE HOO; 0BB9 0BCB +TAMIL SYLLABLE HAU; 0BB9 0BCC + +TAMIL SYLLABLE KSSA; 0B95 0BCD 0BB7 +TAMIL SYLLABLE KSSAA; 0B95 0BCD 0BB7 0BBE +TAMIL SYLLABLE KSSI; 0B95 0BCD 0BB7 0BBF +TAMIL SYLLABLE KSSII; 0B95 0BCD 0BB7 0BC0 +TAMIL SYLLABLE KSSU; 0B95 0BCD 0BB7 0BC1 +TAMIL SYLLABLE KSSUU; 0B95 0BCD 0BB7 0BC2 +TAMIL SYLLABLE KSSE; 0B95 0BCD 0BB7 0BC6 +TAMIL SYLLABLE KSSEE; 0B95 0BCD 0BB7 0BC7 +TAMIL SYLLABLE KSSAI; 0B95 0BCD 0BB7 0BC8 +TAMIL SYLLABLE KSSO; 0B95 0BCD 0BB7 0BCA +TAMIL SYLLABLE KSSOO; 0B95 0BCD 0BB7 0BCB +TAMIL SYLLABLE KSSAU; 0B95 0BCD 0BB7 0BCC + +TAMIL SYLLABLE SHRII; 0BB6 0BCD 0BB0 0BC0 + +GEORGIAN LETTER U-BRJGU;10E3 0302 +KHMER CONSONANT SIGN COENG KA;17D2 1780 +KHMER CONSONANT SIGN COENG KHA;17D2 1781 +KHMER CONSONANT SIGN COENG KO;17D2 1782 +KHMER CONSONANT SIGN COENG KHO;17D2 1783 +KHMER CONSONANT SIGN COENG NGO;17D2 1784 +KHMER CONSONANT SIGN COENG CA;17D2 1785 +KHMER CONSONANT SIGN COENG CHA;17D2 1786 +KHMER CONSONANT SIGN COENG CO;17D2 1787 +KHMER CONSONANT SIGN COENG CHO;17D2 1788 +KHMER CONSONANT SIGN COENG NYO;17D2 1789 +KHMER CONSONANT SIGN COENG DA;17D2 178A +KHMER CONSONANT SIGN COENG TTHA;17D2 178B +KHMER CONSONANT SIGN COENG DO;17D2 178C +KHMER CONSONANT SIGN COENG TTHO;17D2 178D +KHMER CONSONANT SIGN COENG NA;17D2 178E +KHMER CONSONANT SIGN COENG TA;17D2 178F +KHMER CONSONANT SIGN COENG THA;17D2 1790 +KHMER CONSONANT SIGN COENG TO;17D2 1791 +KHMER CONSONANT SIGN COENG THO;17D2 1792 +KHMER CONSONANT SIGN COENG NO;17D2 1793 +KHMER CONSONANT SIGN COENG BA;17D2 1794 +KHMER CONSONANT SIGN COENG PHA;17D2 1795 +KHMER CONSONANT SIGN COENG PO;17D2 1796 +KHMER CONSONANT SIGN COENG PHO;17D2 1797 +KHMER CONSONANT SIGN COENG MO;17D2 1798 +KHMER CONSONANT SIGN COENG YO;17D2 1799 +KHMER CONSONANT SIGN COENG RO;17D2 179A +KHMER CONSONANT SIGN COENG LO;17D2 179B +KHMER CONSONANT SIGN COENG VO;17D2 179C +KHMER CONSONANT SIGN COENG SHA;17D2 179D +KHMER CONSONANT SIGN COENG SSA;17D2 179E +KHMER CONSONANT SIGN COENG SA;17D2 179F +KHMER CONSONANT SIGN COENG HA;17D2 17A0 +KHMER CONSONANT SIGN COENG LA;17D2 17A1 +KHMER VOWEL SIGN COENG QA;17D2 17A2 +KHMER INDEPENDENT VOWEL SIGN COENG QU;17D2 17A7 +KHMER INDEPENDENT VOWEL SIGN COENG RY;17D2 17AB +KHMER INDEPENDENT VOWEL SIGN COENG RYY;17D2 17AC +KHMER INDEPENDENT VOWEL SIGN COENG QE;17D2 17AF +KHMER VOWEL SIGN OM;17BB 17C6 +KHMER VOWEL SIGN AAM;17B6 17C6 +KATAKANA LETTER AINU P;31F7 309A +MODIFIER LETTER EXTRA-HIGH EXTRA-LOW CONTOUR TONE BAR;02E5 02E9 diff --git a/rpython/rlib/unicodedata/NamedSequences-6.0.0.txt b/rpython/rlib/unicodedata/NamedSequences-6.0.0.txt new file mode 100644 --- /dev/null +++ b/rpython/rlib/unicodedata/NamedSequences-6.0.0.txt @@ -0,0 +1,495 @@ +# NamedSequences-6.0.0.txt +# Date: 2010-05-18, 10:48:00 PDT [KW] +# +# Unicode Character Database +# Copyright (c) 1991-2010 Unicode, Inc. +# For terms of use, see http://www.unicode.org/terms_of_use.html +# For documentation, see http://www.unicode.org/reports/tr44/ +# +# Format: +# Name of Sequence; Code Point Sequence for USI +# +# Code point sequences in the UCD use spaces as delimiters. +# The corresponding format for a USI in ISO/IEC 10646 uses +# comma delimitation and angle brackets. Thus, a named sequence +# of the form: +# +# EXAMPLE NAME;1000 1001 1002 +# +# in this data file, would correspond to a 10646 USI as follows: +# +# <1000, 1001, 1002> +# +# Note: The order of entries in this file is not significant. +# However, entries are generally in script order corresponding +# to block order in the Unicode Standard, to make it easier +# to find entries in the list. + +# ================================================ + +LATIN CAPITAL LETTER A WITH MACRON AND GRAVE;0100 0300 +LATIN SMALL LETTER A WITH MACRON AND GRAVE;0101 0300 +LATIN CAPITAL LETTER E WITH VERTICAL LINE BELOW;0045 0329 +LATIN SMALL LETTER E WITH VERTICAL LINE BELOW;0065 0329 +LATIN CAPITAL LETTER E WITH VERTICAL LINE BELOW AND GRAVE;00C8 0329 +LATIN SMALL LETTER E WITH VERTICAL LINE BELOW AND GRAVE;00E8 0329 +LATIN CAPITAL LETTER E WITH VERTICAL LINE BELOW AND ACUTE;00C9 0329 +LATIN SMALL LETTER E WITH VERTICAL LINE BELOW AND ACUTE;00E9 0329 +LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND MACRON;00CA 0304 +LATIN SMALL LETTER E WITH CIRCUMFLEX AND MACRON;00EA 0304 +LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND CARON;00CA 030C +LATIN SMALL LETTER E WITH CIRCUMFLEX AND CARON;00EA 030C +LATIN CAPITAL LETTER I WITH MACRON AND GRAVE;012A 0300 +LATIN SMALL LETTER I WITH MACRON AND GRAVE;012B 0300 +LATIN SMALL LETTER I WITH DOT ABOVE AND ACUTE;0069 0307 0301 +LATIN SMALL LETTER NG WITH TILDE ABOVE;006E 0360 0067 +LATIN CAPITAL LETTER O WITH VERTICAL LINE BELOW;004F 0329 +LATIN SMALL LETTER O WITH VERTICAL LINE BELOW;006F 0329 +LATIN CAPITAL LETTER O WITH VERTICAL LINE BELOW AND GRAVE;00D2 0329 +LATIN SMALL LETTER O WITH VERTICAL LINE BELOW AND GRAVE;00F2 0329 +LATIN CAPITAL LETTER O WITH VERTICAL LINE BELOW AND ACUTE;00D3 0329 +LATIN SMALL LETTER O WITH VERTICAL LINE BELOW AND ACUTE;00F3 0329 +LATIN CAPITAL LETTER S WITH VERTICAL LINE BELOW;0053 0329 +LATIN SMALL LETTER S WITH VERTICAL LINE BELOW;0073 0329 +LATIN CAPITAL LETTER U WITH MACRON AND GRAVE;016A 0300 +LATIN SMALL LETTER U WITH MACRON AND GRAVE;016B 0300 + +# Additions for Lithuanian. Provisional 2006-05-18, Approved 2007-10-19 + +LATIN CAPITAL LETTER A WITH OGONEK AND ACUTE;0104 0301 +LATIN SMALL LETTER A WITH OGONEK AND ACUTE;0105 0301 +LATIN CAPITAL LETTER A WITH OGONEK AND TILDE;0104 0303 +LATIN SMALL LETTER A WITH OGONEK AND TILDE;0105 0303 +LATIN CAPITAL LETTER E WITH OGONEK AND ACUTE;0118 0301 +LATIN SMALL LETTER E WITH OGONEK AND ACUTE;0119 0301 +LATIN CAPITAL LETTER E WITH OGONEK AND TILDE;0118 0303 +LATIN SMALL LETTER E WITH OGONEK AND TILDE;0119 0303 +LATIN CAPITAL LETTER E WITH DOT ABOVE AND ACUTE;0116 0301 +LATIN SMALL LETTER E WITH DOT ABOVE AND ACUTE;0117 0301 +LATIN CAPITAL LETTER E WITH DOT ABOVE AND TILDE;0116 0303 +LATIN SMALL LETTER E WITH DOT ABOVE AND TILDE;0117 0303 +LATIN SMALL LETTER I WITH DOT ABOVE AND GRAVE;0069 0307 0300 +LATIN SMALL LETTER I WITH DOT ABOVE AND TILDE;0069 0307 0303 +LATIN CAPITAL LETTER I WITH OGONEK AND ACUTE;012E 0301 +LATIN SMALL LETTER I WITH OGONEK AND DOT ABOVE AND ACUTE;012F 0307 0301 +LATIN CAPITAL LETTER I WITH OGONEK AND TILDE;012E 0303 +LATIN SMALL LETTER I WITH OGONEK AND DOT ABOVE AND TILDE;012F 0307 0303 +LATIN CAPITAL LETTER J WITH TILDE;004A 0303 +LATIN SMALL LETTER J WITH DOT ABOVE AND TILDE;006A 0307 0303 +LATIN CAPITAL LETTER L WITH TILDE;004C 0303 +LATIN SMALL LETTER L WITH TILDE;006C 0303 +LATIN CAPITAL LETTER M WITH TILDE;004D 0303 +LATIN SMALL LETTER M WITH TILDE;006D 0303 +LATIN CAPITAL LETTER R WITH TILDE;0052 0303 +LATIN SMALL LETTER R WITH TILDE;0072 0303 +LATIN CAPITAL LETTER U WITH OGONEK AND ACUTE;0172 0301 +LATIN SMALL LETTER U WITH OGONEK AND ACUTE;0173 0301 +LATIN CAPITAL LETTER U WITH OGONEK AND TILDE;0172 0303 +LATIN SMALL LETTER U WITH OGONEK AND TILDE;0173 0303 +LATIN CAPITAL LETTER U WITH MACRON AND ACUTE;016A 0301 +LATIN SMALL LETTER U WITH MACRON AND ACUTE;016B 0301 +LATIN CAPITAL LETTER U WITH MACRON AND TILDE;016A 0303 +LATIN SMALL LETTER U WITH MACRON AND TILDE;016B 0303 + +# Entries for JIS X 0213 compatibility mapping. +# Provisional: 2008-11-07, Approved 2010-05-14 + +LATIN SMALL LETTER AE WITH GRAVE;00E6 0300 +LATIN SMALL LETTER OPEN O WITH GRAVE;0254 0300 +LATIN SMALL LETTER OPEN O WITH ACUTE;0254 0301 +LATIN SMALL LETTER TURNED V WITH GRAVE;028C 0300 +LATIN SMALL LETTER TURNED V WITH ACUTE;028C 0301 +LATIN SMALL LETTER SCHWA WITH GRAVE;0259 0300 +LATIN SMALL LETTER SCHWA WITH ACUTE;0259 0301 +LATIN SMALL LETTER HOOKED SCHWA WITH GRAVE;025A 0300 +LATIN SMALL LETTER HOOKED SCHWA WITH ACUTE;025A 0301 + +# Entry for a Bangla entity. +# Provisional: 2009-08-10, Approved 2010-05-14 + +BENGALI LETTER KHINYA;0995 09CD 09B7 + +# Additions for Tamil. Provisional 2008-02-08, Approved 2009-08-14 +# +# A visual display of the Tamil named sequences is available +# in the documentation for Unicode 5.2. See: +# http://www.unicode.org/versions/Unicode5.2.0/ + +TAMIL CONSONANT K; 0B95 0BCD +TAMIL CONSONANT NG; 0B99 0BCD +TAMIL CONSONANT C; 0B9A 0BCD +TAMIL CONSONANT NY; 0B9E 0BCD +TAMIL CONSONANT TT; 0B9F 0BCD +TAMIL CONSONANT NN; 0BA3 0BCD +TAMIL CONSONANT T; 0BA4 0BCD +TAMIL CONSONANT N; 0BA8 0BCD +TAMIL CONSONANT P; 0BAA 0BCD +TAMIL CONSONANT M; 0BAE 0BCD +TAMIL CONSONANT Y; 0BAF 0BCD +TAMIL CONSONANT R; 0BB0 0BCD +TAMIL CONSONANT L; 0BB2 0BCD +TAMIL CONSONANT V; 0BB5 0BCD +TAMIL CONSONANT LLL;0BB4 0BCD +TAMIL CONSONANT LL; 0BB3 0BCD +TAMIL CONSONANT RR; 0BB1 0BCD +TAMIL CONSONANT NNN;0BA9 0BCD +TAMIL CONSONANT J; 0B9C 0BCD +TAMIL CONSONANT SH; 0BB6 0BCD +TAMIL CONSONANT SS; 0BB7 0BCD +TAMIL CONSONANT S; 0BB8 0BCD +TAMIL CONSONANT H; 0BB9 0BCD +TAMIL CONSONANT KSS;0B95 0BCD 0BB7 0BCD + +TAMIL SYLLABLE KAA; 0B95 0BBE +TAMIL SYLLABLE KI; 0B95 0BBF +TAMIL SYLLABLE KII; 0B95 0BC0 +TAMIL SYLLABLE KU; 0B95 0BC1 +TAMIL SYLLABLE KUU; 0B95 0BC2 +TAMIL SYLLABLE KE; 0B95 0BC6 +TAMIL SYLLABLE KEE; 0B95 0BC7 +TAMIL SYLLABLE KAI; 0B95 0BC8 +TAMIL SYLLABLE KO; 0B95 0BCA +TAMIL SYLLABLE KOO; 0B95 0BCB +TAMIL SYLLABLE KAU; 0B95 0BCC + +TAMIL SYLLABLE NGAA; 0B99 0BBE +TAMIL SYLLABLE NGI; 0B99 0BBF +TAMIL SYLLABLE NGII; 0B99 0BC0 +TAMIL SYLLABLE NGU; 0B99 0BC1 +TAMIL SYLLABLE NGUU; 0B99 0BC2 +TAMIL SYLLABLE NGE; 0B99 0BC6 +TAMIL SYLLABLE NGEE; 0B99 0BC7 +TAMIL SYLLABLE NGAI; 0B99 0BC8 +TAMIL SYLLABLE NGO; 0B99 0BCA +TAMIL SYLLABLE NGOO; 0B99 0BCB +TAMIL SYLLABLE NGAU; 0B99 0BCC + +TAMIL SYLLABLE CAA; 0B9A 0BBE +TAMIL SYLLABLE CI; 0B9A 0BBF +TAMIL SYLLABLE CII; 0B9A 0BC0 +TAMIL SYLLABLE CU; 0B9A 0BC1 +TAMIL SYLLABLE CUU; 0B9A 0BC2 +TAMIL SYLLABLE CE; 0B9A 0BC6 +TAMIL SYLLABLE CEE; 0B9A 0BC7 +TAMIL SYLLABLE CAI; 0B9A 0BC8 +TAMIL SYLLABLE CO; 0B9A 0BCA +TAMIL SYLLABLE COO; 0B9A 0BCB +TAMIL SYLLABLE CAU; 0B9A 0BCC + +TAMIL SYLLABLE NYAA; 0B9E 0BBE +TAMIL SYLLABLE NYI; 0B9E 0BBF +TAMIL SYLLABLE NYII; 0B9E 0BC0 +TAMIL SYLLABLE NYU; 0B9E 0BC1 +TAMIL SYLLABLE NYUU; 0B9E 0BC2 +TAMIL SYLLABLE NYE; 0B9E 0BC6 +TAMIL SYLLABLE NYEE; 0B9E 0BC7 +TAMIL SYLLABLE NYAI; 0B9E 0BC8 +TAMIL SYLLABLE NYO; 0B9E 0BCA +TAMIL SYLLABLE NYOO; 0B9E 0BCB +TAMIL SYLLABLE NYAU; 0B9E 0BCC + +TAMIL SYLLABLE TTAA; 0B9F 0BBE +TAMIL SYLLABLE TTI; 0B9F 0BBF +TAMIL SYLLABLE TTII; 0B9F 0BC0 +TAMIL SYLLABLE TTU; 0B9F 0BC1 +TAMIL SYLLABLE TTUU; 0B9F 0BC2 +TAMIL SYLLABLE TTE; 0B9F 0BC6 +TAMIL SYLLABLE TTEE; 0B9F 0BC7 +TAMIL SYLLABLE TTAI; 0B9F 0BC8 +TAMIL SYLLABLE TTO; 0B9F 0BCA +TAMIL SYLLABLE TTOO; 0B9F 0BCB +TAMIL SYLLABLE TTAU; 0B9F 0BCC + +TAMIL SYLLABLE NNAA; 0BA3 0BBE +TAMIL SYLLABLE NNI; 0BA3 0BBF +TAMIL SYLLABLE NNII; 0BA3 0BC0 +TAMIL SYLLABLE NNU; 0BA3 0BC1 +TAMIL SYLLABLE NNUU; 0BA3 0BC2 +TAMIL SYLLABLE NNE; 0BA3 0BC6 +TAMIL SYLLABLE NNEE; 0BA3 0BC7 +TAMIL SYLLABLE NNAI; 0BA3 0BC8 +TAMIL SYLLABLE NNO; 0BA3 0BCA +TAMIL SYLLABLE NNOO; 0BA3 0BCB +TAMIL SYLLABLE NNAU; 0BA3 0BCC + +TAMIL SYLLABLE TAA; 0BA4 0BBE +TAMIL SYLLABLE TI; 0BA4 0BBF +TAMIL SYLLABLE TII; 0BA4 0BC0 +TAMIL SYLLABLE TU; 0BA4 0BC1 +TAMIL SYLLABLE TUU; 0BA4 0BC2 +TAMIL SYLLABLE TE; 0BA4 0BC6 +TAMIL SYLLABLE TEE; 0BA4 0BC7 +TAMIL SYLLABLE TAI; 0BA4 0BC8 +TAMIL SYLLABLE TO; 0BA4 0BCA +TAMIL SYLLABLE TOO; 0BA4 0BCB +TAMIL SYLLABLE TAU; 0BA4 0BCC + +TAMIL SYLLABLE NAA; 0BA8 0BBE +TAMIL SYLLABLE NI; 0BA8 0BBF +TAMIL SYLLABLE NII; 0BA8 0BC0 +TAMIL SYLLABLE NU; 0BA8 0BC1 +TAMIL SYLLABLE NUU; 0BA8 0BC2 +TAMIL SYLLABLE NE; 0BA8 0BC6 +TAMIL SYLLABLE NEE; 0BA8 0BC7 +TAMIL SYLLABLE NAI; 0BA8 0BC8 +TAMIL SYLLABLE NO; 0BA8 0BCA +TAMIL SYLLABLE NOO; 0BA8 0BCB +TAMIL SYLLABLE NAU; 0BA8 0BCC + +TAMIL SYLLABLE PAA; 0BAA 0BBE +TAMIL SYLLABLE PI; 0BAA 0BBF +TAMIL SYLLABLE PII; 0BAA 0BC0 +TAMIL SYLLABLE PU; 0BAA 0BC1 +TAMIL SYLLABLE PUU; 0BAA 0BC2 +TAMIL SYLLABLE PE; 0BAA 0BC6 +TAMIL SYLLABLE PEE; 0BAA 0BC7 +TAMIL SYLLABLE PAI; 0BAA 0BC8 +TAMIL SYLLABLE PO; 0BAA 0BCA +TAMIL SYLLABLE POO; 0BAA 0BCB +TAMIL SYLLABLE PAU; 0BAA 0BCC + +TAMIL SYLLABLE MAA; 0BAE 0BBE +TAMIL SYLLABLE MI; 0BAE 0BBF +TAMIL SYLLABLE MII; 0BAE 0BC0 +TAMIL SYLLABLE MU; 0BAE 0BC1 +TAMIL SYLLABLE MUU; 0BAE 0BC2 +TAMIL SYLLABLE ME; 0BAE 0BC6 +TAMIL SYLLABLE MEE; 0BAE 0BC7 +TAMIL SYLLABLE MAI; 0BAE 0BC8 +TAMIL SYLLABLE MO; 0BAE 0BCA +TAMIL SYLLABLE MOO; 0BAE 0BCB +TAMIL SYLLABLE MAU; 0BAE 0BCC + +TAMIL SYLLABLE YAA; 0BAF 0BBE +TAMIL SYLLABLE YI; 0BAF 0BBF +TAMIL SYLLABLE YII; 0BAF 0BC0 +TAMIL SYLLABLE YU; 0BAF 0BC1 +TAMIL SYLLABLE YUU; 0BAF 0BC2 +TAMIL SYLLABLE YE; 0BAF 0BC6 +TAMIL SYLLABLE YEE; 0BAF 0BC7 +TAMIL SYLLABLE YAI; 0BAF 0BC8 +TAMIL SYLLABLE YO; 0BAF 0BCA +TAMIL SYLLABLE YOO; 0BAF 0BCB +TAMIL SYLLABLE YAU; 0BAF 0BCC + +TAMIL SYLLABLE RAA; 0BB0 0BBE +TAMIL SYLLABLE RI; 0BB0 0BBF +TAMIL SYLLABLE RII; 0BB0 0BC0 +TAMIL SYLLABLE RU; 0BB0 0BC1 +TAMIL SYLLABLE RUU; 0BB0 0BC2 +TAMIL SYLLABLE RE; 0BB0 0BC6 +TAMIL SYLLABLE REE; 0BB0 0BC7 +TAMIL SYLLABLE RAI; 0BB0 0BC8 +TAMIL SYLLABLE RO; 0BB0 0BCA +TAMIL SYLLABLE ROO; 0BB0 0BCB +TAMIL SYLLABLE RAU; 0BB0 0BCC + +TAMIL SYLLABLE LAA; 0BB2 0BBE +TAMIL SYLLABLE LI; 0BB2 0BBF +TAMIL SYLLABLE LII; 0BB2 0BC0 +TAMIL SYLLABLE LU; 0BB2 0BC1 +TAMIL SYLLABLE LUU; 0BB2 0BC2 +TAMIL SYLLABLE LE; 0BB2 0BC6 +TAMIL SYLLABLE LEE; 0BB2 0BC7 +TAMIL SYLLABLE LAI; 0BB2 0BC8 +TAMIL SYLLABLE LO; 0BB2 0BCA +TAMIL SYLLABLE LOO; 0BB2 0BCB +TAMIL SYLLABLE LAU; 0BB2 0BCC + +TAMIL SYLLABLE VAA; 0BB5 0BBE +TAMIL SYLLABLE VI; 0BB5 0BBF +TAMIL SYLLABLE VII; 0BB5 0BC0 +TAMIL SYLLABLE VU; 0BB5 0BC1 +TAMIL SYLLABLE VUU; 0BB5 0BC2 +TAMIL SYLLABLE VE; 0BB5 0BC6 +TAMIL SYLLABLE VEE; 0BB5 0BC7 +TAMIL SYLLABLE VAI; 0BB5 0BC8 +TAMIL SYLLABLE VO; 0BB5 0BCA +TAMIL SYLLABLE VOO; 0BB5 0BCB +TAMIL SYLLABLE VAU; 0BB5 0BCC + +TAMIL SYLLABLE LLLAA; 0BB4 0BBE +TAMIL SYLLABLE LLLI; 0BB4 0BBF +TAMIL SYLLABLE LLLII; 0BB4 0BC0 +TAMIL SYLLABLE LLLU; 0BB4 0BC1 +TAMIL SYLLABLE LLLUU; 0BB4 0BC2 +TAMIL SYLLABLE LLLE; 0BB4 0BC6 +TAMIL SYLLABLE LLLEE; 0BB4 0BC7 +TAMIL SYLLABLE LLLAI; 0BB4 0BC8 +TAMIL SYLLABLE LLLO; 0BB4 0BCA +TAMIL SYLLABLE LLLOO; 0BB4 0BCB +TAMIL SYLLABLE LLLAU; 0BB4 0BCC + +TAMIL SYLLABLE LLAA; 0BB3 0BBE +TAMIL SYLLABLE LLI; 0BB3 0BBF +TAMIL SYLLABLE LLII; 0BB3 0BC0 +TAMIL SYLLABLE LLU; 0BB3 0BC1 +TAMIL SYLLABLE LLUU; 0BB3 0BC2 +TAMIL SYLLABLE LLE; 0BB3 0BC6 +TAMIL SYLLABLE LLEE; 0BB3 0BC7 +TAMIL SYLLABLE LLAI; 0BB3 0BC8 +TAMIL SYLLABLE LLO; 0BB3 0BCA +TAMIL SYLLABLE LLOO; 0BB3 0BCB +TAMIL SYLLABLE LLAU; 0BB3 0BCC + +TAMIL SYLLABLE RRAA; 0BB1 0BBE +TAMIL SYLLABLE RRI; 0BB1 0BBF +TAMIL SYLLABLE RRII; 0BB1 0BC0 +TAMIL SYLLABLE RRU; 0BB1 0BC1 +TAMIL SYLLABLE RRUU; 0BB1 0BC2 +TAMIL SYLLABLE RRE; 0BB1 0BC6 +TAMIL SYLLABLE RREE; 0BB1 0BC7 +TAMIL SYLLABLE RRAI; 0BB1 0BC8 +TAMIL SYLLABLE RRO; 0BB1 0BCA +TAMIL SYLLABLE RROO; 0BB1 0BCB +TAMIL SYLLABLE RRAU; 0BB1 0BCC + +TAMIL SYLLABLE NNNAA; 0BA9 0BBE +TAMIL SYLLABLE NNNI; 0BA9 0BBF +TAMIL SYLLABLE NNNII; 0BA9 0BC0 +TAMIL SYLLABLE NNNU; 0BA9 0BC1 +TAMIL SYLLABLE NNNUU; 0BA9 0BC2 +TAMIL SYLLABLE NNNE; 0BA9 0BC6 +TAMIL SYLLABLE NNNEE; 0BA9 0BC7 +TAMIL SYLLABLE NNNAI; 0BA9 0BC8 +TAMIL SYLLABLE NNNO; 0BA9 0BCA +TAMIL SYLLABLE NNNOO; 0BA9 0BCB +TAMIL SYLLABLE NNNAU; 0BA9 0BCC + +TAMIL SYLLABLE JAA; 0B9C 0BBE +TAMIL SYLLABLE JI; 0B9C 0BBF +TAMIL SYLLABLE JII; 0B9C 0BC0 +TAMIL SYLLABLE JU; 0B9C 0BC1 +TAMIL SYLLABLE JUU; 0B9C 0BC2 +TAMIL SYLLABLE JE; 0B9C 0BC6 +TAMIL SYLLABLE JEE; 0B9C 0BC7 +TAMIL SYLLABLE JAI; 0B9C 0BC8 +TAMIL SYLLABLE JO; 0B9C 0BCA +TAMIL SYLLABLE JOO; 0B9C 0BCB +TAMIL SYLLABLE JAU; 0B9C 0BCC + +TAMIL SYLLABLE SHAA; 0BB6 0BBE +TAMIL SYLLABLE SHI; 0BB6 0BBF +TAMIL SYLLABLE SHII; 0BB6 0BC0 +TAMIL SYLLABLE SHU; 0BB6 0BC1 +TAMIL SYLLABLE SHUU; 0BB6 0BC2 +TAMIL SYLLABLE SHE; 0BB6 0BC6 +TAMIL SYLLABLE SHEE; 0BB6 0BC7 +TAMIL SYLLABLE SHAI; 0BB6 0BC8 +TAMIL SYLLABLE SHO; 0BB6 0BCA +TAMIL SYLLABLE SHOO; 0BB6 0BCB +TAMIL SYLLABLE SHAU; 0BB6 0BCC + +TAMIL SYLLABLE SSAA; 0BB7 0BBE +TAMIL SYLLABLE SSI; 0BB7 0BBF +TAMIL SYLLABLE SSII; 0BB7 0BC0 +TAMIL SYLLABLE SSU; 0BB7 0BC1 +TAMIL SYLLABLE SSUU; 0BB7 0BC2 +TAMIL SYLLABLE SSE; 0BB7 0BC6 +TAMIL SYLLABLE SSEE; 0BB7 0BC7 +TAMIL SYLLABLE SSAI; 0BB7 0BC8 +TAMIL SYLLABLE SSO; 0BB7 0BCA +TAMIL SYLLABLE SSOO; 0BB7 0BCB +TAMIL SYLLABLE SSAU; 0BB7 0BCC + +TAMIL SYLLABLE SAA; 0BB8 0BBE +TAMIL SYLLABLE SI; 0BB8 0BBF +TAMIL SYLLABLE SII; 0BB8 0BC0 +TAMIL SYLLABLE SU; 0BB8 0BC1 +TAMIL SYLLABLE SUU; 0BB8 0BC2 +TAMIL SYLLABLE SE; 0BB8 0BC6 +TAMIL SYLLABLE SEE; 0BB8 0BC7 +TAMIL SYLLABLE SAI; 0BB8 0BC8 +TAMIL SYLLABLE SO; 0BB8 0BCA +TAMIL SYLLABLE SOO; 0BB8 0BCB +TAMIL SYLLABLE SAU; 0BB8 0BCC + +TAMIL SYLLABLE HAA; 0BB9 0BBE +TAMIL SYLLABLE HI; 0BB9 0BBF +TAMIL SYLLABLE HII; 0BB9 0BC0 +TAMIL SYLLABLE HU; 0BB9 0BC1 +TAMIL SYLLABLE HUU; 0BB9 0BC2 +TAMIL SYLLABLE HE; 0BB9 0BC6 +TAMIL SYLLABLE HEE; 0BB9 0BC7 +TAMIL SYLLABLE HAI; 0BB9 0BC8 +TAMIL SYLLABLE HO; 0BB9 0BCA +TAMIL SYLLABLE HOO; 0BB9 0BCB +TAMIL SYLLABLE HAU; 0BB9 0BCC + +TAMIL SYLLABLE KSSA; 0B95 0BCD 0BB7 +TAMIL SYLLABLE KSSAA; 0B95 0BCD 0BB7 0BBE +TAMIL SYLLABLE KSSI; 0B95 0BCD 0BB7 0BBF +TAMIL SYLLABLE KSSII; 0B95 0BCD 0BB7 0BC0 +TAMIL SYLLABLE KSSU; 0B95 0BCD 0BB7 0BC1 +TAMIL SYLLABLE KSSUU; 0B95 0BCD 0BB7 0BC2 +TAMIL SYLLABLE KSSE; 0B95 0BCD 0BB7 0BC6 +TAMIL SYLLABLE KSSEE; 0B95 0BCD 0BB7 0BC7 +TAMIL SYLLABLE KSSAI; 0B95 0BCD 0BB7 0BC8 +TAMIL SYLLABLE KSSO; 0B95 0BCD 0BB7 0BCA +TAMIL SYLLABLE KSSOO; 0B95 0BCD 0BB7 0BCB +TAMIL SYLLABLE KSSAU; 0B95 0BCD 0BB7 0BCC + +TAMIL SYLLABLE SHRII; 0BB6 0BCD 0BB0 0BC0 + +GEORGIAN LETTER U-BRJGU;10E3 0302 +KHMER CONSONANT SIGN COENG KA;17D2 1780 +KHMER CONSONANT SIGN COENG KHA;17D2 1781 +KHMER CONSONANT SIGN COENG KO;17D2 1782 +KHMER CONSONANT SIGN COENG KHO;17D2 1783 +KHMER CONSONANT SIGN COENG NGO;17D2 1784 +KHMER CONSONANT SIGN COENG CA;17D2 1785 +KHMER CONSONANT SIGN COENG CHA;17D2 1786 +KHMER CONSONANT SIGN COENG CO;17D2 1787 +KHMER CONSONANT SIGN COENG CHO;17D2 1788 +KHMER CONSONANT SIGN COENG NYO;17D2 1789 +KHMER CONSONANT SIGN COENG DA;17D2 178A +KHMER CONSONANT SIGN COENG TTHA;17D2 178B +KHMER CONSONANT SIGN COENG DO;17D2 178C +KHMER CONSONANT SIGN COENG TTHO;17D2 178D +KHMER CONSONANT SIGN COENG NA;17D2 178E +KHMER CONSONANT SIGN COENG TA;17D2 178F +KHMER CONSONANT SIGN COENG THA;17D2 1790 +KHMER CONSONANT SIGN COENG TO;17D2 1791 +KHMER CONSONANT SIGN COENG THO;17D2 1792 +KHMER CONSONANT SIGN COENG NO;17D2 1793 +KHMER CONSONANT SIGN COENG BA;17D2 1794 +KHMER CONSONANT SIGN COENG PHA;17D2 1795 +KHMER CONSONANT SIGN COENG PO;17D2 1796 +KHMER CONSONANT SIGN COENG PHO;17D2 1797 +KHMER CONSONANT SIGN COENG MO;17D2 1798 +KHMER CONSONANT SIGN COENG YO;17D2 1799 +KHMER CONSONANT SIGN COENG RO;17D2 179A +KHMER CONSONANT SIGN COENG LO;17D2 179B +KHMER CONSONANT SIGN COENG VO;17D2 179C +KHMER CONSONANT SIGN COENG SHA;17D2 179D +KHMER CONSONANT SIGN COENG SSA;17D2 179E +KHMER CONSONANT SIGN COENG SA;17D2 179F +KHMER CONSONANT SIGN COENG HA;17D2 17A0 +KHMER CONSONANT SIGN COENG LA;17D2 17A1 +KHMER VOWEL SIGN COENG QA;17D2 17A2 +KHMER INDEPENDENT VOWEL SIGN COENG QU;17D2 17A7 +KHMER INDEPENDENT VOWEL SIGN COENG RY;17D2 17AB +KHMER INDEPENDENT VOWEL SIGN COENG RYY;17D2 17AC +KHMER INDEPENDENT VOWEL SIGN COENG QE;17D2 17AF +KHMER VOWEL SIGN OM;17BB 17C6 +KHMER VOWEL SIGN AAM;17B6 17C6 + +# Entries for JIS X 0213 compatibility mapping. +# Provisional: 2008-11-07, Approved 2010-05-14 + +HIRAGANA LETTER BIDAKUON NGA;304B 309A +HIRAGANA LETTER BIDAKUON NGI;304D 309A +HIRAGANA LETTER BIDAKUON NGU;304F 309A +HIRAGANA LETTER BIDAKUON NGE;3051 309A +HIRAGANA LETTER BIDAKUON NGO;3053 309A +KATAKANA LETTER BIDAKUON NGA;30AB 309A +KATAKANA LETTER BIDAKUON NGI;30AD 309A +KATAKANA LETTER BIDAKUON NGU;30AF 309A +KATAKANA LETTER BIDAKUON NGE;30B1 309A +KATAKANA LETTER BIDAKUON NGO;30B3 309A +KATAKANA LETTER AINU CE;30BB 309A +KATAKANA LETTER AINU TU;30C4 309A +KATAKANA LETTER AINU TO;30C8 309A +KATAKANA LETTER AINU P;31F7 309A +MODIFIER LETTER EXTRA-HIGH EXTRA-LOW CONTOUR TONE BAR;02E5 02E9 +MODIFIER LETTER EXTRA-LOW EXTRA-HIGH CONTOUR TONE BAR;02E9 02E5 diff --git a/rpython/rlib/unicodedata/NamedSequences-6.2.0.txt b/rpython/rlib/unicodedata/NamedSequences-6.2.0.txt new file mode 100644 --- /dev/null +++ b/rpython/rlib/unicodedata/NamedSequences-6.2.0.txt @@ -0,0 +1,504 @@ +# NamedSequences-6.2.0.txt +# Date: 2012-05-15, 21:23:00 GMT [KW] +# +# Unicode Character Database +# Copyright (c) 1991-2012 Unicode, Inc. +# For terms of use, see http://www.unicode.org/terms_of_use.html +# For documentation, see http://www.unicode.org/reports/tr44/ +# +# Format: +# Name of Sequence; Code Point Sequence for USI +# +# Code point sequences in the UCD use spaces as delimiters. +# The corresponding format for a USI in ISO/IEC 10646 uses +# comma delimitation and angle brackets. Thus, a named sequence +# of the form: +# +# EXAMPLE NAME;1000 1001 1002 +# +# in this data file, would correspond to a 10646 USI as follows: +# +# <1000, 1001, 1002> +# +# Note: The order of entries in this file is not significant. +# However, entries are generally in script order corresponding +# to block order in the Unicode Standard, to make it easier +# to find entries in the list. + +# ================================================ + +LATIN CAPITAL LETTER A WITH MACRON AND GRAVE;0100 0300 +LATIN SMALL LETTER A WITH MACRON AND GRAVE;0101 0300 +LATIN CAPITAL LETTER E WITH VERTICAL LINE BELOW;0045 0329 +LATIN SMALL LETTER E WITH VERTICAL LINE BELOW;0065 0329 +LATIN CAPITAL LETTER E WITH VERTICAL LINE BELOW AND GRAVE;00C8 0329 +LATIN SMALL LETTER E WITH VERTICAL LINE BELOW AND GRAVE;00E8 0329 +LATIN CAPITAL LETTER E WITH VERTICAL LINE BELOW AND ACUTE;00C9 0329 +LATIN SMALL LETTER E WITH VERTICAL LINE BELOW AND ACUTE;00E9 0329 +LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND MACRON;00CA 0304 +LATIN SMALL LETTER E WITH CIRCUMFLEX AND MACRON;00EA 0304 +LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND CARON;00CA 030C +LATIN SMALL LETTER E WITH CIRCUMFLEX AND CARON;00EA 030C +LATIN CAPITAL LETTER I WITH MACRON AND GRAVE;012A 0300 +LATIN SMALL LETTER I WITH MACRON AND GRAVE;012B 0300 +LATIN SMALL LETTER I WITH DOT ABOVE AND ACUTE;0069 0307 0301 +LATIN SMALL LETTER NG WITH TILDE ABOVE;006E 0360 0067 +LATIN CAPITAL LETTER O WITH VERTICAL LINE BELOW;004F 0329 +LATIN SMALL LETTER O WITH VERTICAL LINE BELOW;006F 0329 +LATIN CAPITAL LETTER O WITH VERTICAL LINE BELOW AND GRAVE;00D2 0329 +LATIN SMALL LETTER O WITH VERTICAL LINE BELOW AND GRAVE;00F2 0329 +LATIN CAPITAL LETTER O WITH VERTICAL LINE BELOW AND ACUTE;00D3 0329 +LATIN SMALL LETTER O WITH VERTICAL LINE BELOW AND ACUTE;00F3 0329 +LATIN CAPITAL LETTER S WITH VERTICAL LINE BELOW;0053 0329 +LATIN SMALL LETTER S WITH VERTICAL LINE BELOW;0073 0329 +LATIN CAPITAL LETTER U WITH MACRON AND GRAVE;016A 0300 +LATIN SMALL LETTER U WITH MACRON AND GRAVE;016B 0300 + +# Additions for Lithuanian. Provisional 2006-05-18, Approved 2007-10-19 + +LATIN CAPITAL LETTER A WITH OGONEK AND ACUTE;0104 0301 +LATIN SMALL LETTER A WITH OGONEK AND ACUTE;0105 0301 +LATIN CAPITAL LETTER A WITH OGONEK AND TILDE;0104 0303 +LATIN SMALL LETTER A WITH OGONEK AND TILDE;0105 0303 +LATIN CAPITAL LETTER E WITH OGONEK AND ACUTE;0118 0301 +LATIN SMALL LETTER E WITH OGONEK AND ACUTE;0119 0301 +LATIN CAPITAL LETTER E WITH OGONEK AND TILDE;0118 0303 +LATIN SMALL LETTER E WITH OGONEK AND TILDE;0119 0303 +LATIN CAPITAL LETTER E WITH DOT ABOVE AND ACUTE;0116 0301 +LATIN SMALL LETTER E WITH DOT ABOVE AND ACUTE;0117 0301 +LATIN CAPITAL LETTER E WITH DOT ABOVE AND TILDE;0116 0303 +LATIN SMALL LETTER E WITH DOT ABOVE AND TILDE;0117 0303 +LATIN SMALL LETTER I WITH DOT ABOVE AND GRAVE;0069 0307 0300 +LATIN SMALL LETTER I WITH DOT ABOVE AND TILDE;0069 0307 0303 +LATIN CAPITAL LETTER I WITH OGONEK AND ACUTE;012E 0301 +LATIN SMALL LETTER I WITH OGONEK AND DOT ABOVE AND ACUTE;012F 0307 0301 +LATIN CAPITAL LETTER I WITH OGONEK AND TILDE;012E 0303 +LATIN SMALL LETTER I WITH OGONEK AND DOT ABOVE AND TILDE;012F 0307 0303 +LATIN CAPITAL LETTER J WITH TILDE;004A 0303 +LATIN SMALL LETTER J WITH DOT ABOVE AND TILDE;006A 0307 0303 +LATIN CAPITAL LETTER L WITH TILDE;004C 0303 +LATIN SMALL LETTER L WITH TILDE;006C 0303 +LATIN CAPITAL LETTER M WITH TILDE;004D 0303 +LATIN SMALL LETTER M WITH TILDE;006D 0303 +LATIN CAPITAL LETTER R WITH TILDE;0052 0303 +LATIN SMALL LETTER R WITH TILDE;0072 0303 +LATIN CAPITAL LETTER U WITH OGONEK AND ACUTE;0172 0301 +LATIN SMALL LETTER U WITH OGONEK AND ACUTE;0173 0301 +LATIN CAPITAL LETTER U WITH OGONEK AND TILDE;0172 0303 +LATIN SMALL LETTER U WITH OGONEK AND TILDE;0173 0303 +LATIN CAPITAL LETTER U WITH MACRON AND ACUTE;016A 0301 +LATIN SMALL LETTER U WITH MACRON AND ACUTE;016B 0301 +LATIN CAPITAL LETTER U WITH MACRON AND TILDE;016A 0303 +LATIN SMALL LETTER U WITH MACRON AND TILDE;016B 0303 + +# Entries for JIS X 0213 compatibility mapping. +# Provisional: 2008-11-07, Approved 2010-05-14 + +LATIN SMALL LETTER AE WITH GRAVE;00E6 0300 +LATIN SMALL LETTER OPEN O WITH GRAVE;0254 0300 +LATIN SMALL LETTER OPEN O WITH ACUTE;0254 0301 +LATIN SMALL LETTER TURNED V WITH GRAVE;028C 0300 +LATIN SMALL LETTER TURNED V WITH ACUTE;028C 0301 +LATIN SMALL LETTER SCHWA WITH GRAVE;0259 0300 +LATIN SMALL LETTER SCHWA WITH ACUTE;0259 0301 +LATIN SMALL LETTER HOOKED SCHWA WITH GRAVE;025A 0300 +LATIN SMALL LETTER HOOKED SCHWA WITH ACUTE;025A 0301 + +# Entry for a Bangla entity. +# Provisional: 2009-08-10, Approved 2010-05-14 + +BENGALI LETTER KHINYA;0995 09CD 09B7 + +# Additions for Tamil. Provisional 2008-02-08, Approved 2009-08-14 +# +# A visual display of the Tamil named sequences is available +# in the documentation for the Unicode Standard. See Section 9.6, Tamil in +# http://www.unicode.org/versions/latest/ + +TAMIL CONSONANT K; 0B95 0BCD +TAMIL CONSONANT NG; 0B99 0BCD +TAMIL CONSONANT C; 0B9A 0BCD +TAMIL CONSONANT NY; 0B9E 0BCD +TAMIL CONSONANT TT; 0B9F 0BCD +TAMIL CONSONANT NN; 0BA3 0BCD +TAMIL CONSONANT T; 0BA4 0BCD +TAMIL CONSONANT N; 0BA8 0BCD +TAMIL CONSONANT P; 0BAA 0BCD +TAMIL CONSONANT M; 0BAE 0BCD +TAMIL CONSONANT Y; 0BAF 0BCD +TAMIL CONSONANT R; 0BB0 0BCD +TAMIL CONSONANT L; 0BB2 0BCD +TAMIL CONSONANT V; 0BB5 0BCD +TAMIL CONSONANT LLL;0BB4 0BCD +TAMIL CONSONANT LL; 0BB3 0BCD +TAMIL CONSONANT RR; 0BB1 0BCD +TAMIL CONSONANT NNN;0BA9 0BCD +TAMIL CONSONANT J; 0B9C 0BCD +TAMIL CONSONANT SH; 0BB6 0BCD +TAMIL CONSONANT SS; 0BB7 0BCD +TAMIL CONSONANT S; 0BB8 0BCD +TAMIL CONSONANT H; 0BB9 0BCD +TAMIL CONSONANT KSS;0B95 0BCD 0BB7 0BCD + +TAMIL SYLLABLE KAA; 0B95 0BBE +TAMIL SYLLABLE KI; 0B95 0BBF +TAMIL SYLLABLE KII; 0B95 0BC0 +TAMIL SYLLABLE KU; 0B95 0BC1 +TAMIL SYLLABLE KUU; 0B95 0BC2 +TAMIL SYLLABLE KE; 0B95 0BC6 +TAMIL SYLLABLE KEE; 0B95 0BC7 +TAMIL SYLLABLE KAI; 0B95 0BC8 +TAMIL SYLLABLE KO; 0B95 0BCA +TAMIL SYLLABLE KOO; 0B95 0BCB +TAMIL SYLLABLE KAU; 0B95 0BCC + +TAMIL SYLLABLE NGAA; 0B99 0BBE +TAMIL SYLLABLE NGI; 0B99 0BBF +TAMIL SYLLABLE NGII; 0B99 0BC0 +TAMIL SYLLABLE NGU; 0B99 0BC1 +TAMIL SYLLABLE NGUU; 0B99 0BC2 +TAMIL SYLLABLE NGE; 0B99 0BC6 +TAMIL SYLLABLE NGEE; 0B99 0BC7 +TAMIL SYLLABLE NGAI; 0B99 0BC8 +TAMIL SYLLABLE NGO; 0B99 0BCA +TAMIL SYLLABLE NGOO; 0B99 0BCB +TAMIL SYLLABLE NGAU; 0B99 0BCC + +TAMIL SYLLABLE CAA; 0B9A 0BBE +TAMIL SYLLABLE CI; 0B9A 0BBF +TAMIL SYLLABLE CII; 0B9A 0BC0 +TAMIL SYLLABLE CU; 0B9A 0BC1 +TAMIL SYLLABLE CUU; 0B9A 0BC2 +TAMIL SYLLABLE CE; 0B9A 0BC6 +TAMIL SYLLABLE CEE; 0B9A 0BC7 +TAMIL SYLLABLE CAI; 0B9A 0BC8 +TAMIL SYLLABLE CO; 0B9A 0BCA +TAMIL SYLLABLE COO; 0B9A 0BCB +TAMIL SYLLABLE CAU; 0B9A 0BCC + +TAMIL SYLLABLE NYAA; 0B9E 0BBE +TAMIL SYLLABLE NYI; 0B9E 0BBF +TAMIL SYLLABLE NYII; 0B9E 0BC0 +TAMIL SYLLABLE NYU; 0B9E 0BC1 +TAMIL SYLLABLE NYUU; 0B9E 0BC2 +TAMIL SYLLABLE NYE; 0B9E 0BC6 +TAMIL SYLLABLE NYEE; 0B9E 0BC7 +TAMIL SYLLABLE NYAI; 0B9E 0BC8 +TAMIL SYLLABLE NYO; 0B9E 0BCA +TAMIL SYLLABLE NYOO; 0B9E 0BCB +TAMIL SYLLABLE NYAU; 0B9E 0BCC + +TAMIL SYLLABLE TTAA; 0B9F 0BBE +TAMIL SYLLABLE TTI; 0B9F 0BBF +TAMIL SYLLABLE TTII; 0B9F 0BC0 +TAMIL SYLLABLE TTU; 0B9F 0BC1 +TAMIL SYLLABLE TTUU; 0B9F 0BC2 +TAMIL SYLLABLE TTE; 0B9F 0BC6 +TAMIL SYLLABLE TTEE; 0B9F 0BC7 +TAMIL SYLLABLE TTAI; 0B9F 0BC8 +TAMIL SYLLABLE TTO; 0B9F 0BCA +TAMIL SYLLABLE TTOO; 0B9F 0BCB +TAMIL SYLLABLE TTAU; 0B9F 0BCC + +TAMIL SYLLABLE NNAA; 0BA3 0BBE +TAMIL SYLLABLE NNI; 0BA3 0BBF +TAMIL SYLLABLE NNII; 0BA3 0BC0 +TAMIL SYLLABLE NNU; 0BA3 0BC1 +TAMIL SYLLABLE NNUU; 0BA3 0BC2 +TAMIL SYLLABLE NNE; 0BA3 0BC6 +TAMIL SYLLABLE NNEE; 0BA3 0BC7 +TAMIL SYLLABLE NNAI; 0BA3 0BC8 +TAMIL SYLLABLE NNO; 0BA3 0BCA +TAMIL SYLLABLE NNOO; 0BA3 0BCB +TAMIL SYLLABLE NNAU; 0BA3 0BCC + +TAMIL SYLLABLE TAA; 0BA4 0BBE +TAMIL SYLLABLE TI; 0BA4 0BBF +TAMIL SYLLABLE TII; 0BA4 0BC0 +TAMIL SYLLABLE TU; 0BA4 0BC1 +TAMIL SYLLABLE TUU; 0BA4 0BC2 +TAMIL SYLLABLE TE; 0BA4 0BC6 +TAMIL SYLLABLE TEE; 0BA4 0BC7 +TAMIL SYLLABLE TAI; 0BA4 0BC8 +TAMIL SYLLABLE TO; 0BA4 0BCA +TAMIL SYLLABLE TOO; 0BA4 0BCB +TAMIL SYLLABLE TAU; 0BA4 0BCC + +TAMIL SYLLABLE NAA; 0BA8 0BBE +TAMIL SYLLABLE NI; 0BA8 0BBF +TAMIL SYLLABLE NII; 0BA8 0BC0 +TAMIL SYLLABLE NU; 0BA8 0BC1 +TAMIL SYLLABLE NUU; 0BA8 0BC2 +TAMIL SYLLABLE NE; 0BA8 0BC6 +TAMIL SYLLABLE NEE; 0BA8 0BC7 +TAMIL SYLLABLE NAI; 0BA8 0BC8 +TAMIL SYLLABLE NO; 0BA8 0BCA +TAMIL SYLLABLE NOO; 0BA8 0BCB +TAMIL SYLLABLE NAU; 0BA8 0BCC + +TAMIL SYLLABLE PAA; 0BAA 0BBE +TAMIL SYLLABLE PI; 0BAA 0BBF +TAMIL SYLLABLE PII; 0BAA 0BC0 +TAMIL SYLLABLE PU; 0BAA 0BC1 +TAMIL SYLLABLE PUU; 0BAA 0BC2 +TAMIL SYLLABLE PE; 0BAA 0BC6 +TAMIL SYLLABLE PEE; 0BAA 0BC7 +TAMIL SYLLABLE PAI; 0BAA 0BC8 +TAMIL SYLLABLE PO; 0BAA 0BCA +TAMIL SYLLABLE POO; 0BAA 0BCB +TAMIL SYLLABLE PAU; 0BAA 0BCC + +TAMIL SYLLABLE MAA; 0BAE 0BBE +TAMIL SYLLABLE MI; 0BAE 0BBF +TAMIL SYLLABLE MII; 0BAE 0BC0 +TAMIL SYLLABLE MU; 0BAE 0BC1 +TAMIL SYLLABLE MUU; 0BAE 0BC2 +TAMIL SYLLABLE ME; 0BAE 0BC6 +TAMIL SYLLABLE MEE; 0BAE 0BC7 +TAMIL SYLLABLE MAI; 0BAE 0BC8 +TAMIL SYLLABLE MO; 0BAE 0BCA +TAMIL SYLLABLE MOO; 0BAE 0BCB +TAMIL SYLLABLE MAU; 0BAE 0BCC + +TAMIL SYLLABLE YAA; 0BAF 0BBE +TAMIL SYLLABLE YI; 0BAF 0BBF +TAMIL SYLLABLE YII; 0BAF 0BC0 +TAMIL SYLLABLE YU; 0BAF 0BC1 +TAMIL SYLLABLE YUU; 0BAF 0BC2 +TAMIL SYLLABLE YE; 0BAF 0BC6 +TAMIL SYLLABLE YEE; 0BAF 0BC7 +TAMIL SYLLABLE YAI; 0BAF 0BC8 +TAMIL SYLLABLE YO; 0BAF 0BCA +TAMIL SYLLABLE YOO; 0BAF 0BCB +TAMIL SYLLABLE YAU; 0BAF 0BCC + +TAMIL SYLLABLE RAA; 0BB0 0BBE +TAMIL SYLLABLE RI; 0BB0 0BBF +TAMIL SYLLABLE RII; 0BB0 0BC0 +TAMIL SYLLABLE RU; 0BB0 0BC1 +TAMIL SYLLABLE RUU; 0BB0 0BC2 +TAMIL SYLLABLE RE; 0BB0 0BC6 +TAMIL SYLLABLE REE; 0BB0 0BC7 +TAMIL SYLLABLE RAI; 0BB0 0BC8 +TAMIL SYLLABLE RO; 0BB0 0BCA +TAMIL SYLLABLE ROO; 0BB0 0BCB +TAMIL SYLLABLE RAU; 0BB0 0BCC + +TAMIL SYLLABLE LAA; 0BB2 0BBE +TAMIL SYLLABLE LI; 0BB2 0BBF +TAMIL SYLLABLE LII; 0BB2 0BC0 +TAMIL SYLLABLE LU; 0BB2 0BC1 +TAMIL SYLLABLE LUU; 0BB2 0BC2 +TAMIL SYLLABLE LE; 0BB2 0BC6 +TAMIL SYLLABLE LEE; 0BB2 0BC7 +TAMIL SYLLABLE LAI; 0BB2 0BC8 +TAMIL SYLLABLE LO; 0BB2 0BCA +TAMIL SYLLABLE LOO; 0BB2 0BCB +TAMIL SYLLABLE LAU; 0BB2 0BCC + +TAMIL SYLLABLE VAA; 0BB5 0BBE +TAMIL SYLLABLE VI; 0BB5 0BBF +TAMIL SYLLABLE VII; 0BB5 0BC0 +TAMIL SYLLABLE VU; 0BB5 0BC1 +TAMIL SYLLABLE VUU; 0BB5 0BC2 +TAMIL SYLLABLE VE; 0BB5 0BC6 +TAMIL SYLLABLE VEE; 0BB5 0BC7 +TAMIL SYLLABLE VAI; 0BB5 0BC8 +TAMIL SYLLABLE VO; 0BB5 0BCA +TAMIL SYLLABLE VOO; 0BB5 0BCB +TAMIL SYLLABLE VAU; 0BB5 0BCC + +TAMIL SYLLABLE LLLAA; 0BB4 0BBE +TAMIL SYLLABLE LLLI; 0BB4 0BBF +TAMIL SYLLABLE LLLII; 0BB4 0BC0 +TAMIL SYLLABLE LLLU; 0BB4 0BC1 +TAMIL SYLLABLE LLLUU; 0BB4 0BC2 +TAMIL SYLLABLE LLLE; 0BB4 0BC6 +TAMIL SYLLABLE LLLEE; 0BB4 0BC7 +TAMIL SYLLABLE LLLAI; 0BB4 0BC8 +TAMIL SYLLABLE LLLO; 0BB4 0BCA +TAMIL SYLLABLE LLLOO; 0BB4 0BCB +TAMIL SYLLABLE LLLAU; 0BB4 0BCC + +TAMIL SYLLABLE LLAA; 0BB3 0BBE +TAMIL SYLLABLE LLI; 0BB3 0BBF +TAMIL SYLLABLE LLII; 0BB3 0BC0 +TAMIL SYLLABLE LLU; 0BB3 0BC1 +TAMIL SYLLABLE LLUU; 0BB3 0BC2 +TAMIL SYLLABLE LLE; 0BB3 0BC6 +TAMIL SYLLABLE LLEE; 0BB3 0BC7 +TAMIL SYLLABLE LLAI; 0BB3 0BC8 +TAMIL SYLLABLE LLO; 0BB3 0BCA +TAMIL SYLLABLE LLOO; 0BB3 0BCB +TAMIL SYLLABLE LLAU; 0BB3 0BCC + +TAMIL SYLLABLE RRAA; 0BB1 0BBE +TAMIL SYLLABLE RRI; 0BB1 0BBF +TAMIL SYLLABLE RRII; 0BB1 0BC0 +TAMIL SYLLABLE RRU; 0BB1 0BC1 +TAMIL SYLLABLE RRUU; 0BB1 0BC2 +TAMIL SYLLABLE RRE; 0BB1 0BC6 +TAMIL SYLLABLE RREE; 0BB1 0BC7 +TAMIL SYLLABLE RRAI; 0BB1 0BC8 +TAMIL SYLLABLE RRO; 0BB1 0BCA +TAMIL SYLLABLE RROO; 0BB1 0BCB +TAMIL SYLLABLE RRAU; 0BB1 0BCC + +TAMIL SYLLABLE NNNAA; 0BA9 0BBE +TAMIL SYLLABLE NNNI; 0BA9 0BBF +TAMIL SYLLABLE NNNII; 0BA9 0BC0 +TAMIL SYLLABLE NNNU; 0BA9 0BC1 +TAMIL SYLLABLE NNNUU; 0BA9 0BC2 +TAMIL SYLLABLE NNNE; 0BA9 0BC6 +TAMIL SYLLABLE NNNEE; 0BA9 0BC7 +TAMIL SYLLABLE NNNAI; 0BA9 0BC8 +TAMIL SYLLABLE NNNO; 0BA9 0BCA +TAMIL SYLLABLE NNNOO; 0BA9 0BCB +TAMIL SYLLABLE NNNAU; 0BA9 0BCC + +TAMIL SYLLABLE JAA; 0B9C 0BBE +TAMIL SYLLABLE JI; 0B9C 0BBF +TAMIL SYLLABLE JII; 0B9C 0BC0 +TAMIL SYLLABLE JU; 0B9C 0BC1 +TAMIL SYLLABLE JUU; 0B9C 0BC2 +TAMIL SYLLABLE JE; 0B9C 0BC6 +TAMIL SYLLABLE JEE; 0B9C 0BC7 +TAMIL SYLLABLE JAI; 0B9C 0BC8 +TAMIL SYLLABLE JO; 0B9C 0BCA +TAMIL SYLLABLE JOO; 0B9C 0BCB +TAMIL SYLLABLE JAU; 0B9C 0BCC + +TAMIL SYLLABLE SHAA; 0BB6 0BBE +TAMIL SYLLABLE SHI; 0BB6 0BBF +TAMIL SYLLABLE SHII; 0BB6 0BC0 +TAMIL SYLLABLE SHU; 0BB6 0BC1 +TAMIL SYLLABLE SHUU; 0BB6 0BC2 +TAMIL SYLLABLE SHE; 0BB6 0BC6 +TAMIL SYLLABLE SHEE; 0BB6 0BC7 +TAMIL SYLLABLE SHAI; 0BB6 0BC8 +TAMIL SYLLABLE SHO; 0BB6 0BCA +TAMIL SYLLABLE SHOO; 0BB6 0BCB +TAMIL SYLLABLE SHAU; 0BB6 0BCC From noreply at buildbot.pypy.org Mon Mar 16 01:16:26 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 16 Mar 2015 01:16:26 +0100 (CET) Subject: [pypy-commit] pypy py3.3: Unicodedb: Add support for Aliases. Message-ID: <20150316001626.E1ED61C00EF@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3.3 Changeset: r76391:06f9a5ad6287 Date: 2015-03-16 01:15 +0100 http://bitbucket.org/pypy/pypy/changeset/06f9a5ad6287/ Log: Unicodedb: Add support for Aliases. diff too long, truncating to 2000 out of 172399 lines diff --git a/pypy/module/unicodedata/interp_ucd.py b/pypy/module/unicodedata/interp_ucd.py --- a/pypy/module/unicodedata/interp_ucd.py +++ b/pypy/module/unicodedata/interp_ucd.py @@ -75,7 +75,7 @@ class UCD(W_Root): def __init__(self, unicodedb): - self._lookup = unicodedb.lookup + self._lookup = unicodedb.lookup_with_alias self._lookup_named_sequence = unicodedb.lookup_named_sequence self._name = unicodedb.name self._decimal = unicodedb.decimal diff --git a/pypy/module/unicodedata/test/test_unicodedata.py b/pypy/module/unicodedata/test/test_unicodedata.py --- a/pypy/module/unicodedata/test/test_unicodedata.py +++ b/pypy/module/unicodedata/test/test_unicodedata.py @@ -107,6 +107,27 @@ import unicodedata raises(TypeError, unicodedata.bidirectional, 'xx') + def test_aliases(self): + import unicodedata + aliases = [ + ('LATIN CAPITAL LETTER GHA', 0x01A2), + ('LATIN SMALL LETTER GHA', 0x01A3), + ('KANNADA LETTER LLLA', 0x0CDE), + ('LAO LETTER FO FON', 0x0E9D), + ('LAO LETTER FO FAY', 0x0E9F), + ('LAO LETTER RO', 0x0EA3), + ('LAO LETTER LO', 0x0EA5), + ('TIBETAN MARK BKA- SHOG GI MGO RGYAN', 0x0FD0), + ('YI SYLLABLE ITERATION MARK', 0xA015), + ('PRESENTATION FORM FOR VERTICAL RIGHT WHITE LENTICULAR BRACKET', 0xFE18), + ('BYZANTINE MUSICAL SYMBOL FTHORA SKLIRON CHROMA VASIS', 0x1D0C5) + ] + for alias, codepoint in aliases: + name = unicodedata.name(chr(codepoint)) + assert name != alias + assert unicodedata.lookup(alias) == unicodedata.lookup(name) + raises(KeyError, unicodedata.ucd_3_2_0.lookup, alias) + def test_named_sequences(self): import unicodedata sequences = [ diff --git a/rpython/rlib/unicodedata/generate_unicodedb.py b/rpython/rlib/unicodedata/generate_unicodedb.py --- a/rpython/rlib/unicodedata/generate_unicodedb.py +++ b/rpython/rlib/unicodedata/generate_unicodedb.py @@ -88,10 +88,11 @@ class UnicodeData(object): # we use this range of PUA_15 to store name aliases and named sequences NAME_ALIASES_START = 0xF0000 - NAMED_SEQUENCES_START = 0xF0100 + NAMED_SEQUENCES_START = 0xF0200 def __init__(self): self.table = [None] * (MAXUNICODE + 1) + self.aliases = [] self.named_sequences = [] def add_char(self, code, char): @@ -149,6 +150,12 @@ self.table[code].canonical_decomp = result return self.table[code].canonical_decomp + def add_alias(self, name, char): + pua_index = self.NAME_ALIASES_START + len(self.aliases) + self.aliases.append((name, char)) + # also store the name in the PUA 1 + self.table[pua_index].name = name + def add_named_sequence(self, name, chars): pua_index = self.NAMED_SEQUENCES_START + len(self.named_sequences) self.named_sequences.append((name, chars)) @@ -262,6 +269,16 @@ table.get_canonical_decomposition(code) table.get_compat_decomposition(code) + # Name aliases + for line in files['name_aliases']: + line = line.strip() + if not line or line.startswith('#'): + continue + items = line.split(';') + char = int(items[0], 16) + name = items[1] + table.add_alias(name, char) + # Named sequences for line in files['named_sequences']: line = line.strip() @@ -786,7 +803,21 @@ return None ''' % dict(start=table.NAMED_SEQUENCES_START) - + # aliases + print >> outfile, '_name_aliases = [' + for name, char in table.aliases: + print >> outfile, "%s," % (char,) + print >> outfile, ']' + print >> outfile, ''' + +def lookup_with_alias(name): + code = lookup(name) + if 0 <= code - %(start)s < len(_name_aliases): + return _name_aliases[code - %(start)s] + else: + return code +''' % dict(start=table.NAME_ALIASES_START) + def main(): import sys diff --git a/rpython/rlib/unicodedata/unicodedb_3_2_0.py b/rpython/rlib/unicodedata/unicodedb_3_2_0.py --- a/rpython/rlib/unicodedata/unicodedb_3_2_0.py +++ b/rpython/rlib/unicodedata/unicodedb_3_2_0.py @@ -493,6 +493,7 @@ 6679: None, 6682: None, 6680: None, +983050: None, 6322: None, 6321: None, 6387: None, @@ -3668,7 +3669,7 @@ 9965: None, 4346: None, 4345: None, -983646: None, +983902: None, 11520: None, 11521: None, 11546: None, @@ -4439,13 +4440,14 @@ 69815: None, 69811: None, 69812: None, +983042: None, 3261: None, 3313: None, 3260: None, 3314: None, 3298: None, 3299: None, -983688: None, +983944: None, 43272: None, 43269: None, 43268: None, @@ -4559,44 +4561,44 @@ 68102: None, 68098: None, 68099: None, -983667: None, -983652: None, -983653: None, -983655: None, -983654: None, -983657: None, -983659: None, -983679: None, -983647: None, -983648: None, -983650: None, -983649: None, -983680: None, -983674: None, -983671: None, -983661: None, -983651: None, -983666: None, -983656: None, -983668: None, -983670: None, -983669: None, -983673: None, -983678: None, -983676: None, -983677: None, -983662: None, -983663: None, -983665: None, -983664: None, -983658: None, -983660: None, -983675: None, -983672: None, -983685: None, -983682: None, -983683: None, -983684: None, +983923: None, +983908: None, +983909: None, +983911: None, +983910: None, +983913: None, +983915: None, +983935: None, +983903: None, +983904: None, +983906: None, +983905: None, +983936: None, +983930: None, +983927: None, +983917: None, +983907: None, +983922: None, +983912: None, +983924: None, +983926: None, +983925: None, +983929: None, +983934: None, +983932: None, +983933: None, +983918: None, +983919: None, +983921: None, +983920: None, +983914: None, +983916: None, +983931: None, +983928: None, +983941: None, +983938: None, +983939: None, +983940: None, 6109: None, 6627: None, 6643: None, @@ -4640,16 +4642,20 @@ 6631: None, 6647: None, 6640: None, -983687: None, -983681: None, -983686: None, +983943: None, +983937: None, +983942: None, +983044: None, +983043: None, +983046: None, +983045: None, 68413: None, 68415: None, 68412: None, 68414: None, -983296: None, -983322: None, -983324: None, +983552: None, +983578: None, +983580: None, 570: None, 42802: None, 11373: None, @@ -4664,26 +4670,27 @@ 42862: None, 42796: None, 42798: None, -983306: None, -983304: None, -983330: None, -983332: None, -983326: None, -983328: None, +983562: None, +983560: None, +983586: None, +983588: None, +983582: None, +983584: None, 582: None, -983298: None, -983302: None, -983300: None, +983554: None, +983558: None, +983556: None, 42788: None, 42786: None, 42858: None, +983040: None, 577: None, 11367: None, 11381: None, 42790: None, -983308: None, -983336: None, -983338: None, +983564: None, +983592: None, +983594: None, 42873: None, 42875: None, 42877: None, @@ -4692,7 +4699,7 @@ 42886: None, 42860: None, 584: None, -983340: None, +983596: None, 11369: None, 42818: None, 42816: None, @@ -4701,16 +4708,16 @@ 11360: None, 42824: None, 11362: None, -983342: None, +983598: None, 11374: None, -983344: None, +983600: None, 7930: None, 7932: None, 42826: None, 42828: None, -983312: None, -983316: None, -983314: None, +983568: None, +983572: None, +983570: None, 42830: None, 42834: None, 42836: None, @@ -4721,11 +4728,11 @@ 42842: None, 588: None, 11364: None, -983346: None, +983602: None, 42814: None, 42844: None, 11390: None, -983318: None, +983574: None, 42891: None, 7838: None, 586: None, @@ -4740,11 +4747,11 @@ 581: None, 42792: None, 580: None, -983352: None, -983320: None, -983354: None, -983348: None, -983350: None, +983608: None, +983576: None, +983610: None, +983604: None, +983606: None, 42846: None, 42856: None, 42850: None, @@ -4791,9 +4798,9 @@ 7460: None, 7547: None, 7550: None, -983297: None, -983323: None, -983325: None, +983553: None, +983579: None, +983581: None, 7567: None, 11365: None, 42803: None, @@ -4819,18 +4826,18 @@ 7839: None, 567: None, 42865: None, -983307: None, -983305: None, -983331: None, -983333: None, +983563: None, +983561: None, +983587: None, +983589: None, 11384: None, -983327: None, -983329: None, +983583: None, +983585: None, 7570: None, 583: None, -983299: None, -983303: None, -983301: None, +983555: None, +983559: None, +983557: None, 42789: None, 42787: None, 7563: None, @@ -4840,16 +4847,17 @@ 7534: None, 7554: None, 7555: None, +983041: None, 578: None, 11368: None, 11382: None, 42791: None, -983310: None, -983334: None, -983335: None, -983309: None, -983337: None, -983339: None, +983566: None, +983590: None, +983591: None, +983565: None, +983593: None, +983595: None, 7574: None, 42874: None, 42876: None, @@ -4859,7 +4867,7 @@ 42887: None, 7548: None, 42861: None, -983341: None, +983597: None, 585: None, 11370: None, 42819: None, @@ -4870,27 +4878,27 @@ 11361: None, 42825: None, 7557: None, -983343: None, +983599: None, 7836: None, 7837: None, 42866: None, 7535: None, 7558: None, -983345: None, +983601: None, 7931: None, 7933: None, 42867: None, 565: None, 7536: None, 7559: None, -983311: None, +983567: None, 42868: None, 42827: None, 42829: None, 11386: None, -983313: None, -983317: None, -983315: None, +983569: None, +983573: None, +983571: None, 42831: None, 7571: None, 7575: None, @@ -4909,7 +4917,7 @@ 7538: None, 7561: None, 589: None, -983347: None, +983603: None, 8580: None, 42815: None, 7572: None, @@ -4918,7 +4926,7 @@ 7540: None, 7562: None, 575: None, -983319: None, +983575: None, 42892: None, 7573: None, 7454: None, @@ -4948,11 +4956,11 @@ 7432: None, 11385: None, 42793: None, -983353: None, -983321: None, -983355: None, -983349: None, -983351: None, +983609: None, +983577: None, +983611: None, +983605: None, +983607: None, 7577: None, 7531: None, 42872: None, @@ -5661,7 +5669,7 @@ 764: None, 42765: None, 42760: None, -983689: None, +983945: None, 42770: None, 42769: None, 42764: None, @@ -6483,6 +6491,7 @@ 65047: None, 65046: None, 65096: None, +983049: None, 65048: None, 65044: None, 9915: None, @@ -7220,30 +7229,30 @@ 43699: None, 43705: None, 3064: None, -983358: None, -983378: None, -983374: None, -983356: None, -983379: None, -983368: None, -983371: None, -983370: None, -983365: None, -983363: None, -983357: None, -983361: None, -983373: None, -983359: None, -983364: None, -983367: None, -983372: None, -983377: None, -983375: None, -983376: None, -983362: None, -983360: None, -983369: None, -983366: None, +983614: None, +983634: None, +983630: None, +983612: None, +983635: None, +983624: None, +983627: None, +983626: None, +983621: None, +983619: None, +983613: None, +983617: None, +983629: None, +983615: None, +983620: None, +983623: None, +983628: None, +983633: None, +983631: None, +983632: None, +983618: None, +983616: None, +983625: None, +983622: None, 3063: None, 3059: None, 3062: None, @@ -7253,272 +7262,272 @@ 3066: None, 3024: None, 3065: None, -983402: None, -983409: None, -983412: None, -983407: None, -983408: None, -983403: None, -983404: None, -983410: None, -983411: None, -983405: None, -983406: None, -983622: None, -983629: None, -983632: None, -983627: None, -983628: None, -983623: None, -983624: None, -983630: None, -983631: None, -983625: None, -983626: None, -983578: None, -983585: None, -983588: None, -983583: None, -983584: None, -983579: None, -983580: None, -983586: None, -983587: None, -983581: None, -983582: None, -983380: None, -983387: None, -983390: None, -983385: None, -983386: None, -983381: None, -983382: None, -983388: None, -983389: None, -983633: None, -983634: None, +983658: None, +983665: None, +983668: None, +983663: None, +983664: None, +983659: None, +983660: None, +983666: None, +983667: None, +983661: None, +983662: None, +983878: None, +983885: None, +983888: None, +983883: None, +983884: None, +983879: None, +983880: None, +983886: None, +983887: None, +983881: None, +983882: None, +983834: None, +983841: None, +983844: None, +983839: None, +983840: None, +983835: None, +983836: None, +983842: None, +983843: None, +983837: None, +983838: None, +983636: None, +983643: None, +983646: None, 983641: None, +983642: None, +983637: None, +983638: None, 983644: None, +983645: None, +983889: None, +983890: None, +983897: None, +983900: None, +983895: None, +983896: None, +983891: None, +983892: None, +983898: None, +983899: None, +983893: None, +983894: None, 983639: None, 983640: None, -983635: None, -983636: None, -983642: None, -983643: None, -983637: None, -983638: None, -983383: None, -983384: None, -983512: None, -983519: None, -983522: None, -983517: None, -983518: None, -983513: None, -983514: None, -983545: None, -983552: None, -983555: None, -983550: None, -983551: None, -983546: None, -983547: None, -983534: None, -983541: None, -983544: None, -983539: None, -983540: None, -983535: None, -983536: None, -983542: None, -983543: None, -983537: None, -983538: None, -983553: None, -983554: None, -983548: None, -983549: None, -983520: None, -983521: None, -983515: None, -983516: None, -983479: None, -983486: None, -983489: None, -983484: None, -983485: None, -983480: None, -983481: None, -983487: None, -983488: None, -983482: None, -983483: None, -983457: None, -983464: None, -983467: None, -983462: None, -983463: None, -983391: None, -983398: None, -983401: None, -983396: None, -983397: None, -983392: None, -983393: None, -983399: None, -983400: None, -983394: None, -983395: None, -983458: None, -983459: None, -983435: None, -983442: None, -983445: None, -983440: None, -983441: None, -983436: None, -983437: None, -983567: None, -983574: None, -983577: None, -983572: None, -983573: None, -983568: None, -983569: None, -983575: None, -983576: None, -983570: None, -983571: None, -983443: None, -983444: None, -983438: None, -983439: None, -983465: None, -983466: None, -983460: None, -983461: None, -983413: None, -983420: None, -983423: None, -983418: None, -983419: None, -983414: None, -983415: None, -983421: None, -983422: None, -983416: None, -983417: None, -983468: None, -983475: None, -983478: None, -983473: None, -983474: None, -983469: None, -983470: None, -983476: None, -983477: None, -983471: None, -983472: None, -983501: None, -983508: None, -983511: None, -983506: None, -983507: None, -983502: None, -983503: None, -983509: None, -983510: None, -983556: None, -983563: None, -983566: None, -983561: None, -983562: None, -983557: None, -983558: None, -983564: None, -983565: None, -983559: None, -983560: None, -983504: None, -983505: None, -983611: None, -983618: None, -983621: None, -983616: None, -983617: None, -983589: None, -983596: None, -983599: None, -983594: None, -983595: None, -983590: None, -983591: None, -983597: None, -983598: None, -983645: None, -983592: None, -983593: None, -983612: None, -983613: None, -983619: None, -983620: None, -983600: None, -983607: None, -983610: None, -983605: None, -983606: None, -983601: None, -983602: None, -983608: None, -983609: None, -983603: None, -983604: None, -983614: None, -983615: None, -983446: None, -983453: None, -983456: None, -983451: None, -983452: None, -983447: None, -983448: None, -983454: None, -983455: None, -983424: None, -983431: None, -983434: None, -983429: None, -983430: None, -983425: None, -983426: None, -983432: None, -983433: None, -983427: None, -983428: None, -983449: None, -983450: None, -983523: None, -983530: None, -983533: None, -983528: None, -983529: None, -983524: None, -983525: None, -983531: None, -983532: None, -983526: None, -983527: None, -983490: None, -983497: None, -983500: None, -983495: None, -983496: None, -983491: None, -983492: None, -983498: None, -983499: None, -983493: None, -983494: None, +983768: None, +983775: None, +983778: None, +983773: None, +983774: None, +983769: None, +983770: None, +983801: None, +983808: None, +983811: None, +983806: None, +983807: None, +983802: None, +983803: None, +983790: None, +983797: None, +983800: None, +983795: None, +983796: None, +983791: None, +983792: None, +983798: None, +983799: None, +983793: None, +983794: None, +983809: None, +983810: None, +983804: None, +983805: None, +983776: None, +983777: None, +983771: None, +983772: None, +983735: None, +983742: None, +983745: None, +983740: None, +983741: None, +983736: None, +983737: None, +983743: None, +983744: None, +983738: None, +983739: None, +983713: None, +983720: None, +983723: None, +983718: None, +983719: None, +983647: None, +983654: None, +983657: None, +983652: None, +983653: None, +983648: None, +983649: None, +983655: None, +983656: None, +983650: None, +983651: None, +983714: None, +983715: None, +983691: None, +983698: None, +983701: None, +983696: None, +983697: None, +983692: None, +983693: None, +983823: None, +983830: None, +983833: None, +983828: None, +983829: None, +983824: None, +983825: None, +983831: None, +983832: None, +983826: None, +983827: None, +983699: None, +983700: None, +983694: None, +983695: None, +983721: None, +983722: None, +983716: None, +983717: None, +983669: None, +983676: None, +983679: None, +983674: None, +983675: None, +983670: None, +983671: None, +983677: None, +983678: None, +983672: None, +983673: None, +983724: None, +983731: None, +983734: None, +983729: None, +983730: None, +983725: None, +983726: None, +983732: None, +983733: None, +983727: None, +983728: None, +983757: None, +983764: None, +983767: None, +983762: None, +983763: None, +983758: None, +983759: None, +983765: None, +983766: None, +983812: None, +983819: None, +983822: None, +983817: None, +983818: None, +983813: None, +983814: None, +983820: None, +983821: None, +983815: None, +983816: None, +983760: None, +983761: None, +983867: None, +983874: None, +983877: None, +983872: None, +983873: None, +983845: None, +983852: None, +983855: None, +983850: None, +983851: None, +983846: None, +983847: None, +983853: None, +983854: None, +983901: None, +983848: None, +983849: None, +983868: None, +983869: None, +983875: None, +983876: None, +983856: None, +983863: None, +983866: None, +983861: None, +983862: None, +983857: None, +983858: None, +983864: None, +983865: None, +983859: None, +983860: None, +983870: None, +983871: None, +983702: None, +983709: None, +983712: None, +983707: None, +983708: None, +983703: None, +983704: None, +983710: None, +983711: None, +983680: None, +983687: None, +983690: None, +983685: None, +983686: None, +983681: None, +983682: None, +983688: None, +983689: None, +983683: None, +983684: None, +983705: None, +983706: None, +983779: None, +983786: None, +983789: None, +983784: None, +983785: None, +983780: None, +983781: None, +983787: None, +983788: None, +983782: None, +983783: None, +983746: None, +983753: None, +983756: None, +983751: None, +983752: None, +983747: None, +983748: None, +983754: None, +983755: None, +983749: None, +983750: None, 3061: None, 3196: None, 3193: None, @@ -7624,6 +7633,7 @@ 9928: None, 3947: None, 3948: None, +983047: None, 4048: None, 4052: None, 4051: None, @@ -8358,6 +8368,7 @@ 11055: None, 11038: None, 11825: None, +983048: None, } _code_by_name = { } @@ -8844,6 +8855,7 @@ 'BUGINESE VOWEL SIGN I': None, 'BUGINESE VOWEL SIGN O': None, 'BUGINESE VOWEL SIGN U': None, +'BYZANTINE MUSICAL SYMBOL FTHORA SKLIRON CHROMA VASIS': None, 'CANADIAN SYLLABICS AAY': None, 'CANADIAN SYLLABICS AY': None, 'CANADIAN SYLLABICS BEAVER DENE L': None, @@ -12790,6 +12802,7 @@ 'KAITHI VOWEL SIGN O': None, 'KAITHI VOWEL SIGN U': None, 'KAITHI VOWEL SIGN UU': None, +'KANNADA LETTER LLLA': None, 'KANNADA SIGN AVAGRAHA': None, 'KANNADA SIGN JIHVAMULIYA': None, 'KANNADA SIGN NUKTA': None, @@ -12994,6 +13007,10 @@ 'KHMER VOWEL SIGN AAM': None, 'KHMER VOWEL SIGN COENG QA': None, 'KHMER VOWEL SIGN OM': None, +'LAO LETTER FO FAY': None, +'LAO LETTER FO FON': None, +'LAO LETTER LO': None, +'LAO LETTER RO': None, 'LARGE ONE DOT OVER TWO DOTS PUNCTUATION': None, 'LARGE ONE RING OVER TWO RINGS PUNCTUATION': None, 'LARGE TWO DOTS OVER ONE DOT PUNCTUATION': None, @@ -13028,6 +13045,7 @@ 'LATIN CAPITAL LETTER EGYPTOLOGICAL AIN': None, 'LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF': None, 'LATIN CAPITAL LETTER ET': None, +'LATIN CAPITAL LETTER GHA': None, 'LATIN CAPITAL LETTER GLOTTAL STOP': None, 'LATIN CAPITAL LETTER H WITH DESCENDER': None, 'LATIN CAPITAL LETTER HALF H': None, @@ -13191,6 +13209,7 @@ 'LATIN SMALL LETTER F WITH MIDDLE TILDE': None, 'LATIN SMALL LETTER F WITH PALATAL HOOK': None, 'LATIN SMALL LETTER G WITH PALATAL HOOK': None, +'LATIN SMALL LETTER GHA': None, 'LATIN SMALL LETTER GLOTTAL STOP': None, 'LATIN SMALL LETTER H WITH DESCENDER': None, 'LATIN SMALL LETTER HALF H': None, @@ -14834,6 +14853,7 @@ 'PRESENTATION FORM FOR VERTICAL LEFT WHITE LENTICULAR BRACKET': None, 'PRESENTATION FORM FOR VERTICAL QUESTION MARK': None, 'PRESENTATION FORM FOR VERTICAL RIGHT SQUARE BRACKET': None, +'PRESENTATION FORM FOR VERTICAL RIGHT WHITE LENTICULAR BRACKET': None, 'PRESENTATION FORM FOR VERTICAL RIGHT WHITE LENTICULAR BRAKCET': None, 'PRESENTATION FORM FOR VERTICAL SEMICOLON': None, 'QUINCUNX': None, @@ -15975,6 +15995,7 @@ 'THUNDER CLOUD AND RAIN': None, 'TIBETAN LETTER KKA': None, 'TIBETAN LETTER RRA': None, +'TIBETAN MARK BKA- SHOG GI MGO RGYAN': None, 'TIBETAN MARK BSKA- SHOG GI MGO RGYAN': None, 'TIBETAN MARK CLOSING BRDA RNYING YIG MGO SGAB MA': None, 'TIBETAN MARK INITIAL BRDA RNYING YIG MGO MDUN MA': None, @@ -16709,6 +16730,7 @@ 'WHITE VERTICAL ELLIPSE': None, 'WHITE VERY SMALL SQUARE': None, 'WORD SEPARATOR MIDDLE DOT': None, +'YI SYLLABLE ITERATION MARK': None, } _cjk_prefix = "CJK UNIFIED IDEOGRAPH-" @@ -21257,8 +21279,19 @@ def lookup_named_sequence(code): - if 0 <= code - 983296 < len(_named_sequences): - return _named_sequences[code - 983296] + if 0 <= code - 983552 < len(_named_sequences): + return _named_sequences[code - 983552] else: return None +_name_aliases = [ +] + + +def lookup_with_alias(name): + code = lookup(name) + if 0 <= code - 983040 < len(_name_aliases): + return _name_aliases[code - 983040] + else: + return code + diff --git a/rpython/rlib/unicodedata/unicodedb_5_2_0.py b/rpython/rlib/unicodedata/unicodedb_5_2_0.py --- a/rpython/rlib/unicodedata/unicodedb_5_2_0.py +++ b/rpython/rlib/unicodedata/unicodedb_5_2_0.py @@ -190,6 +190,7 @@ '\x07 KEFULA' '\x08 KEMBANG' '\x0e KISIM5 TIMES ' +'\x02 L' '\x05 LACA' '\x11 LAGAB TIMES ASH2' '\x11 LAGAR OVER LAGAR' @@ -1349,7 +1350,7 @@ '\x04CHOR' '\tCHOSEONG ' '\x06CHRIVI' -'\rCHROMA SYNAFI' +'\x07CHROMA ' '\rCHRYSANTHEMUM' '\x07CHU CAN' '\x05CHULA' @@ -1377,6 +1378,7 @@ '\x02CK' '\x07CK MARK' ')CK-TILTED SHADOWED WHITE RIGHTWARDS ARROW' +'\x04CKET' '\x05CKING' '\x06CKNESS' '\x07CKWISE ' @@ -1932,6 +1934,7 @@ '\x07ER THAN' '\x08ER TRUTH' '\x08ERAL URN' +'\x0cERATION MARK' '\x0bERCENT SIGN' '\x07ERCIAL ' '\tERCIAL AT' @@ -2251,6 +2254,7 @@ '\x04GHWA' '\x02GI' '\x07GI GUNU' +'\x0cGI MGO RGYAN' '\x04GIBA' '\x06GICAL ' '\x04GIDA' @@ -2972,7 +2976,7 @@ '\nK2 PLUS BU' '\x02K4' '\x02KA' -'\x10KA- SHOG YIG MGO' +'\tKA- SHOG ' '\x04KAAF' '\x03KAB' '\tKABA TENU' @@ -3003,6 +3007,7 @@ '\x06KAYAH ' '\x07KAYANNA' '\x12KBAR ISOLATED FORM' +'\x04KCET' '\x02KE' '\x06KE PHO' '\x16KEEPING STILL MOUNTAIN' @@ -3187,8 +3192,8 @@ '\nLENDED YUS' '\x0bLENGTH MARK' '\x05LENIS' +'\x0eLENTICULAR BRA' '\x12LENTICULAR BRACKET' -'\x12LENTICULAR BRAKCET' '\x04LEPH' '\x0cLER CONSTANT' '\x08LESS SHA' @@ -3276,7 +3281,6 @@ '\x0eLMOST EQUAL TO' '\x0fLMOST EQUAL TO ' '\x02LO' -'\x04LO L' '\rLOCATION SIGN' '\x08LOCATIVE' '\tLOCKWISE ' @@ -5041,6 +5045,7 @@ '\x0bSYMBOL FOR ' '\x12SYMMETRIC SWAPPING' '\x16SYMPTOTICALLY EQUAL TO' +'\x06SYNAFI' '\x07SYNAGMA' '\rSYNDESMOS NEO' '\tSYNTHETON' @@ -5598,6 +5603,7 @@ '\x13VARIANT WITH SQUARE' '\x13VARIATION INDICATOR' '\x0bVARYS ICHOS' +'\x05VASIS' '\x13VASTNESS OR WASTING' '\x03VAV' '\x07VAV YOD' @@ -5907,6 +5913,7 @@ '\x06YGISMA' '\x02YI' '\x08YIDDISH ' +'\x07YIG MGO' '\x08YIG MGO ' '\x0fYIG MGO MDUN MA' '\x14YIG MGO PHUR SHAD MA' @@ -5998,6313 +6005,6313 @@ '\x05ZYGOS' ) _charnodes =[70758, - -54016, + -54013, -1, 132371, - 28772, + 28800, -1, 197694, - 78442, + 78444, -1, 262727, - 136032, + 136035, -1, 327957, - 202072, + 202075, -1, 393238, - 282255, + 282270, -1, -65529, - 347791, + 347806, 195071, -65528, - 409765, + 409767, 195070, -65527, - 472618, + 472620, 195069, -65526, - 535808, + 535811, 195068, -65525, - 599704, + 599707, 195067, -65524, - 660875, + 660878, 195066, -65523, - 726360, + 726363, 195065, -65522, - 791844, + 791847, 195064, -65521, - 857309, + 857312, 195063, -65520, - 922776, + 922779, 195062, -65519, - 988192, + 988195, 195061, -65518, - 1053651, + 1053654, 195060, -65517, - 1119075, + 1119078, 195059, -65516, - 1184446, + 1184449, 195058, -65515, - 1249830, + 1249833, 195057, -1, - 1315238, + 1315241, 195056, 1507367, - 344229, + 344231, -1, -65512, - 1461903, + 1461918, 195055, -65511, - 1523877, + 1523879, 195054, -65510, - 1586730, + 1586732, 195053, -65509, - 1649920, + 1649923, 195052, -65508, - 1713816, + 1713819, 195051, -65507, - 1774987, + 1774990, 195050, -65506, - 1840472, + 1840475, 195049, -65505, - 1905956, + 1905959, 195048, -65504, - 1971421, + 1971424, 195047, -65503, - 2036888, + 2036891, 195046, -65502, - 2102304, + 2102307, 195045, -65501, - 2167763, + 2167766, 195044, -65500, - 2233187, + 2233190, 195043, -65499, - 2298558, + 2298561, 195042, -65498, - 2363942, + 2363945, 195041, -1, - 2429350, + 2429353, 195040, 2621496, - 1455658, + 1455660, -1, -65495, - 2576015, + 2576030, 195039, -65494, - 2637989, + 2637991, 195038, -65493, - 2700842, + 2700844, 195037, -65492, - 2764032, + 2764035, 195036, -65491, - 2827928, + 2827931, 195035, -65490, - 2889099, + 2889102, 195034, -65489, - 2954584, + 2954587, 195033, -65488, - 3020068, + 3020071, 195032, -65487, - 3085533, + 3085536, 195031, -65486, - 3151000, + 3151003, 195030, -65485, - 3216416, + 3216419, 195029, -65484, - 3281875, + 3281878, 195028, -65483, - 3347299, + 3347302, 195027, -65482, - 3412670, + 3412673, 195026, -65481, - 3478054, + 3478057, 195025, -1, - 3543462, + 3543465, 195024, 3735625, - 2567424, + 2567427, -1, -65478, - 3690127, + 3690142, 195023, -65477, - 3752101, + 3752103, 195022, -65476, - 3814954, + 3814956, 195021, -65475, - 3878144, + 3878147, 195020, -65474, - 3942040, + 3942043, 195019, -65473, - 4003211, + 4003214, 195018, -65472, - 4068696, + 4068699, 195017, -65471, - 4134180, + 4134183, 195016, -65470, - 4199645, + 4199648, 195015, -65469, - 4265112, + 4265115, 195014, -65468, - 4330528, + 4330531, 195013, -65467, - 4395987, + 4395990, 195012, -65466, - 4461411, + 4461414, 195011, -65465, - 4526782, + 4526785, 195010, -65464, - 4592166, + 4592169, 195009, -1, - 4657574, + 4657577, 195008, 4849754, - 3679896, + 3679899, -1, -65461, - 4804239, + 4804254, 195007, -65460, - 4866213, + 4866215, 195006, -65459, - 4929066, + 4929068, 195005, -65458, - 4992256, + 4992259, 195004, -65457, - 5056152, + 5056155, 195003, -65456, - 5117323, + 5117326, 195002, -65455, - 5182808, + 5182811, 195001, -65454, - 5248292, + 5248295, 195000, -65453, - 5313757, + 5313760, 194999, -65452, - 5379224, + 5379227, 194998, -65451, - 5444640, + 5444643, 194997, -65450, - 5510099, + 5510102, 194996, -65449, - 5575523, + 5575526, 194995, -65448, - 5640894, + 5640897, 194994, -65447, - 5706278, + 5706281, 194993, -1, - 5771686, + 5771689, 194992, 5963883, - 4789643, + 4789646, -1, -65444, - 5918351, + 5918366, 194991, -65443, - 5980325, + 5980327, 194990, -65442, - 6043178, + 6043180, 194989, -65441, - 6106368, + 6106371, 194988, -65440, - 6170264, + 6170267, 194987, -65439, - 6231435, + 6231438, 194986, -65438, - 6296920, + 6296923, 194985, -65437, - 6362404, + 6362407, 194984, -65436, - 6427869, + 6427872, 194983, -65435, - 6493336, + 6493339, 194982, -65434, - 6558752, + 6558755, 194981, -65433, - 6624211, + 6624214, 194980, -65432, - 6689635, + 6689638, 194979, -65431, - 6755006, + 6755009, 194978, -65430, - 6820390, + 6820393, 194977, -1, - 6885798, + 6885801, 194976, 7078012, - 5903704, + 5903707, -1, -65427, - 7032463, + 7032478, 194975, -65426, - 7094437, + 7094439, 194974, -65425, - 7157290, + 7157292, 194973, -65424, - 7220480, + 7220483, 194972, -65423, - 7284376, + 7284379, 194971, -65422, - 7345547, + 7345550, 194970, -65421, - 7411032, + 7411035, 194969, -65420, - 7476516, + 7476519, 194968, -65419, - 7541981, + 7541984, 194967, -65418, - 7607448, + 7607451, 194966, -65417, - 7672864, + 7672867, 194965, -65416, - 7738323, + 7738326, 194964, -65415, - 7803747, + 7803750, 194963, -65414, - 7869118, + 7869121, 194962, -65413, - 7934502, + 7934505, 194961, -1, - 7999910, + 7999913, 194960, 8192141, - 7017764, + 7017767, -1, -65410, - 8146575, + 8146590, 194959, -65409, - 8208549, + 8208551, 194958, -65408, - 8271402, + 8271404, 194957, -65407, - 8334592, + 8334595, 194956, -65406, - 8398488, + 8398491, 194955, -65405, - 8459659, + 8459662, 194954, -65404, - 8525144, + 8525147, 194953, -65403, - 8590628, + 8590631, 194952, -65402, - 8656093, + 8656096, 194951, -65401, - 8721560, + 8721563, 194950, -65400, - 8786976, + 8786979, 194949, -65399, - 8852435, + 8852438, 194948, -65398, - 8917859, + 8917862, 194947, -65397, - 8983230, + 8983233, 194946, -65396, - 9048614, + 9048617, 194945, -1, - 9114022, + 9114025, 194944, 9306270, - 8131805, + 8131808, -1, -65393, - 9260687, + 9260702, 194943, -65392, - 9322661, + 9322663, 194942, -65391, - 9385514, + 9385516, 194941, -65390, - 9448704, + 9448707, 194940, -65389, - 9512600, + 9512603, 194939, -65388, - 9573771, + 9573774, 194938, -65387, - 9639256, + 9639259, 194937, -65386, - 9704740, + 9704743, 194936, -65385, - 9770205, + 9770208, 194935, -65384, - 9835672, + 9835675, 194934, -65383, - 9901088, + 9901091, 194933, -65382, - 9966547, + 9966550, 194932, -65381, - 10031971, + 10031974, 194931, -65380, - 10097342, + 10097345, 194930, -65379, - 10162726, + 10162729, 194929, -1, - 10228134, + 10228137, 194928, 10420399, - 9245848, + 9245851, -1, -65376, - 10374799, + 10374814, 194927, -65375, - 10436773, + 10436775, 194926, -65374, - 10499626, + 10499628, 194925, -65373, - 10562816, + 10562819, 194924, -65372, - 10626712, + 10626715, 194923, -65371, - 10687883, + 10687886, 194922, -65370, - 10753368, + 10753371, 194921, -65369, - 10818852, + 10818855, 194920, -65368, - 10884317, + 10884320, 194919, -65367, - 10949784, + 10949787, 194918, -65366, - 11015200, + 11015203, From noreply at buildbot.pypy.org Mon Mar 16 10:03:13 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 16 Mar 2015 10:03:13 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Re-export these names from pypystm Message-ID: <20150316090313.098FD1C14C2@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76392:e3dad9a57eca Date: 2015-03-16 09:20 +0100 http://bitbucket.org/pypy/pypy/changeset/e3dad9a57eca/ Log: Re-export these names from pypystm diff --git a/lib_pypy/transaction.py b/lib_pypy/transaction.py --- a/lib_pypy/transaction.py +++ b/lib_pypy/transaction.py @@ -37,24 +37,20 @@ signals_enabled = _SignalsEnabled() try: - from pypystm import hint_commit_soon + from pypystm import hint_commit_soon, getsegmentlimit + from pypystm import hashtable, stmset, stmdict + from pypystm import local, time, clock except ImportError: # Not a STM-enabled PyPy. def hint_commit_soon(): return None - -try: - from pypystm import getsegmentlimit -except ImportError: - # Not a STM-enabled PyPy. def getsegmentlimit(): return 1 - -try: - from pypystm import hashtable -except ImportError: - # Not a STM-enabled PyPy. hashtable = dict + stmset = set + stmdict = dict + local = thread._local + from time import time, clock class stmidset(object): def __init__(self): From noreply at buildbot.pypy.org Mon Mar 16 10:03:14 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 16 Mar 2015 10:03:14 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Remove "local" here: it is the same thing as "thread._local". Message-ID: <20150316090314.30B341C14CA@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76393:6e0a8051f34b Date: 2015-03-16 09:48 +0100 http://bitbucket.org/pypy/pypy/changeset/6e0a8051f34b/ Log: Remove "local" here: it is the same thing as "thread._local". diff --git a/lib_pypy/transaction.py b/lib_pypy/transaction.py --- a/lib_pypy/transaction.py +++ b/lib_pypy/transaction.py @@ -49,7 +49,6 @@ hashtable = dict stmset = set stmdict = dict - local = thread._local from time import time, clock class stmidset(object): From noreply at buildbot.pypy.org Mon Mar 16 10:03:15 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 16 Mar 2015 10:03:15 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Change the argument to threadlocalproperty() from a default value Message-ID: <20150316090315.60B731C14C2@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76394:87258f56326e Date: 2015-03-16 10:02 +0100 http://bitbucket.org/pypy/pypy/changeset/87258f56326e/ Log: Change the argument to threadlocalproperty() from a default value to a default value factory (like "defaultdict"). diff --git a/lib_pypy/pypy_test/test_transaction.py b/lib_pypy/pypy_test/test_transaction.py --- a/lib_pypy/pypy_test/test_transaction.py +++ b/lib_pypy/pypy_test/test_transaction.py @@ -192,6 +192,39 @@ assert d.setdefault(key2) is None assert d[key2] is None +def test_stmdict(): + d = transaction.stmdict() + d["abc"] = "def" + assert list(d.iterkeys()) == ["abc"] + +def test_stmset(): + d = transaction.stmset() + d.add("abc") + assert list(d) == ["abc"] + +def test_time_clock(): + assert isinstance(transaction.time(), float) + assert isinstance(transaction.clock(), float) + +def test_threadlocalproperty(): + class Foo(object): + x = transaction.threadlocalproperty() + y = transaction.threadlocalproperty(dict) + foo = Foo() + py.test.raises(AttributeError, "foo.x") + d = foo.y + assert d == {} + assert d is foo.y + foo.y['bar'] = 'baz' + foo.x = 42 + foo.y = 43 + assert foo.x == 42 + assert foo.y == 43 + del foo.x + del foo.y + py.test.raises(AttributeError, "foo.x") + assert foo.y == {} + def run_tests(): for name in sorted(globals().keys()): diff --git a/lib_pypy/transaction.py b/lib_pypy/transaction.py --- a/lib_pypy/transaction.py +++ b/lib_pypy/transaction.py @@ -294,9 +294,9 @@ class threadlocalproperty(object): - def __init__(self, *default): - self.tl_default = default - self.tl_name = intern(str(id(self))) + def __init__(self, default_factory=None): + self.tl_default_factory = default_factory + self.tl_name = intern('tlprop.%d' % id(self)) def tl_get(self, obj): try: @@ -308,7 +308,14 @@ def __get__(self, obj, cls=None): if obj is None: return self - return getattr(self.tl_get(obj), self.tl_name, *self.tl_default) + try: + return getattr(self.tl_get(obj), self.tl_name) + except AttributeError: + if self.tl_default_factory is None: + raise + result = self.tl_default_factory() + setattr(self.tl_get(obj), self.tl_name, result) + return result def __set__(self, obj, value): setattr(self.tl_get(obj), self.tl_name, value) From noreply at buildbot.pypy.org Mon Mar 16 10:03:16 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 16 Mar 2015 10:03:16 +0100 (CET) Subject: [pypy-commit] pypy default: Finish the section about TransactionQueue Message-ID: <20150316090316.7B5FA1C14C2@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76395:6a184862d752 Date: 2015-03-16 10:03 +0100 http://bitbucket.org/pypy/pypy/changeset/6a184862d752/ Log: Finish the section about TransactionQueue diff --git a/pypy/doc/stm.rst b/pypy/doc/stm.rst --- a/pypy/doc/stm.rst +++ b/pypy/doc/stm.rst @@ -246,34 +246,85 @@ aborts one of the two transactions. In the example above this occurred 12412 times. -The two other conflict causes are ``STM_CONTENTION_INEVITABLE``, which -means that two transactions both tried to do an external operation, -like printing or reading from a socket or accessing an external array -of raw data; and ``STM_CONTENTION_WRITE_READ``, which means that one -transaction wrote to an object but the other one merely read it, not -wrote to it (in that case only the writing transaction is reported; -the location for the reads is not recorded because doing so is not -possible without a very large performance impact). +The two other conflict sources are ``STM_CONTENTION_INEVITABLE``, +which means that two transactions both tried to do an external +operation, like printing or reading from a socket or accessing an +external array of raw data; and ``STM_CONTENTION_WRITE_READ``, which +means that one transaction wrote to an object but the other one merely +read it, not wrote to it (in that case only the writing transaction is +reported; the location for the reads is not recorded because doing so +is not possible without a very large performance impact). + +Common causes of conflicts: + +* First of all, any I/O or raw manipulation of memory turns the + transaction inevitable ("must not abort"). There can be only one + inevitable transaction running at any time. A common case is if + each transaction starts with sending data to a log file. You should + refactor this case so that it occurs either near the end of the + transaction (which can then mostly run in non-inevitable mode), or + even delegate it to a separate thread. + +* Writing to a list or a dictionary conflicts with any read from the + same list or dictionary, even one done with a different key. For + dictionaries and sets, you can try the types ``transaction.stmdict`` + and ``transaction.stmset``, which behave mostly like ``dict`` and + ``set`` but allow concurrent access to different keys. (What is + missing from them so far is lazy iteration: for example, + ``stmdict.iterkeys()`` is implemented as ``iter(stmdict.keys())``; + and, unlike PyPy's dictionaries and sets, the STM versions are not + ordered.) There are also experimental ``stmiddict`` and + ``stmidset`` classes using the identity of the key. + +* ``time.time()`` and ``time.clock()`` turn the transaction inevitable + in order to guarantee that a call that appears to be later will + really return a higher number. If getting slightly unordered + results is fine, use ``transaction.time()`` or + ``transaction.clock()``. + +* ``transaction.threadlocalproperty`` can be used as class-level:: + + class Foo(object): # must be a new-style class! + x = transaction.threadlocalproperty() + y = transaction.threadlocalproperty(dict) + + This declares that instances of ``Foo`` have two attributes ``x`` + and ``y`` that are thread-local: reading or writing them from + concurrently-running transactions will return independent results. + (Any other attributes of ``Foo`` instances will be globally visible + from all threads, as usual.) The optional argument to + ``threadlocalproperty()`` is the default value factory: in case no + value was assigned in the current thread yet, the factory is called + and its result becomes the value in that thread (like + ``collections.defaultdict``). If no default value factory is + specified, uninitialized reads raise ``AttributeError``. Note that + with ``TransactionQueue`` you get a pool of a fixed number of + threads, each running the transactions one after the other; such + thread-local properties will have the value last stored in them in + the same thread,, which may come from a random previous transaction. + ``threadlocalproperty`` is still useful to avoid conflicts from + cache-like data structures. Note that Python is a complicated language; there are a number of less common cases that may cause conflict (of any type) where we might not expect it at priori. In many of these cases it could be fixed; please -report any case that you don't understand. One known example so far -is creating weakrefs to objects: the new weakref is xxx - +report any case that you don't understand. (For example, so far, +creating a weakref to an object requires attaching an auxiliary +internal object to that object, and so it can cause write-write +conflicts.) Atomic sections --------------- -PyPy supports *atomic sections,* which are blocks of code which you -want to execute without "releasing the GIL". In STM terms, this means -blocks of code that are executed while guaranteeing that the -transaction is not interrupted in the middle. *This is experimental -and may be removed in the future* if `lock elision`_ is ever -implemented. +The ``TransactionQueue`` class described above is based on *atomic +sections,* which are blocks of code which you want to execute without +"releasing the GIL". In STM terms, this means blocks of code that are +executed while guaranteeing that the transaction is not interrupted in +the middle. *This is experimental and may be removed in the future* +if `lock elision`_ is ever implemented. -Here is a usage example:: +Here is a direct usage example:: with transaction.atomic: assert len(lst1) == 10 From noreply at buildbot.pypy.org Mon Mar 16 10:51:24 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 16 Mar 2015 10:51:24 +0100 (CET) Subject: [pypy-commit] pypy default: Finish refactoring this document Message-ID: <20150316095124.2EF471C046C@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76396:c892964d02b7 Date: 2015-03-16 10:51 +0100 http://bitbucket.org/pypy/pypy/changeset/c892964d02b7/ Log: Finish refactoring this document diff --git a/pypy/doc/stm.rst b/pypy/doc/stm.rst --- a/pypy/doc/stm.rst +++ b/pypy/doc/stm.rst @@ -46,13 +46,14 @@ multiple cores. * ``pypy-stm`` provides (but does not impose) a special API to the - user in the pure Python module `transaction`_. This module is based - on the lower-level module `pypystm`_, but also provides some + user in the pure Python module ``transaction``. This module is based + on the lower-level module ``pypystm``, but also provides some compatibily with non-STM PyPy's or CPython's. * Building on top of the way the GIL is removed, we will talk - about `Atomic sections, Transactions, etc.: a better way to write - parallel programs`_. + about `How to write multithreaded programs: the 10'000-feet view`_ + and `transaction.TransactionQueue`_. + Getting Started @@ -89,7 +90,7 @@ Current status (stmgc-c7) ------------------------- -* It seems to work fine, without crashing any more. Please `report +* **NEW:** It seems to work fine, without crashing any more. Please `report any crash`_ you find (or other bugs). * It runs with an overhead as low as 20% on examples like "richards". @@ -97,24 +98,35 @@ 2x for "translate.py"-- which we are still trying to understand. One suspect is our partial GC implementation, see below. +* **NEW:** the ``PYPYSTM`` environment variable and the + ``pypy/stm/print_stm_log.py`` script let you know exactly which + "conflicts" occurred. This is described in the section + `transaction.TransactionQueue`_ below. + +* **NEW:** special transaction-friendly APIs (like ``stmdict``), + described in the section `transaction.TransactionQueue`_ below. The + old API changed again, mostly moving to different modules. Sorry + about that. I feel it's a better idea to change the API early + instead of being stuck with a bad one later... + * Currently limited to 1.5 GB of RAM (this is just a parameter in `core.h`__ -- theoretically. In practice, increase it too much and clang crashes again). Memory overflows are not correctly handled; they cause segfaults. -* The JIT warm-up time improved recently but is still bad. In order to - produce machine code, the JIT needs to enter a special single-threaded - mode for now. This means that you will get bad performance results if - your program doesn't run for several seconds, where *several* can mean - *many.* When trying benchmarks, be sure to check that you have - reached the warmed state, i.e. the performance is not improving any - more. This should be clear from the fact that as long as it's - producing more machine code, ``pypy-stm`` will run on a single core. +* **NEW:** The JIT warm-up time improved again, but is still + relatively large. In order to produce machine code, the JIT needs + to enter "inevitable" mode. This means that you will get bad + performance results if your program doesn't run for several seconds, + where *several* can mean *many.* When trying benchmarks, be sure to + check that you have reached the warmed state, i.e. the performance + is not improving any more. * The GC is new; although clearly inspired by PyPy's regular GC, it misses a number of optimizations for now. Programs allocating large numbers of small objects that don't immediately die (surely a common - situation) suffer from these missing optimizations. + situation) suffer from these missing optimizations. (The bleeding + edge ``stmgc-c8`` is better at that.) * Weakrefs might appear to work a bit strangely for now, sometimes staying alive throught ``gc.collect()``, or even dying but then @@ -122,8 +134,7 @@ * The STM system is based on very efficient read/write barriers, which are mostly done (their placement could be improved a bit in - JIT-generated machine code). But the overall bookkeeping logic could - see more improvements (see `Low-level statistics`_ below). + JIT-generated machine code). * Forking the process is slow because the complete memory needs to be copied manually. A warning is printed to this effect. @@ -132,7 +143,8 @@ crash on an assertion error because of a non-implemented overflow of an internal 28-bit counter. -.. _`report bugs`: https://bugs.pypy.org/ + +.. _`report any crash`: https://bitbucket.org/pypy/pypy/issues?status=new&status=open .. __: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/rpython/translator/stm/src_stm/stm/core.h @@ -155,7 +167,6 @@ interpreter and other ones might have slightly different needs. - User Guide ========== @@ -181,8 +192,33 @@ order. -A better way to write parallel programs ---------------------------------------- +How to write multithreaded programs: the 10'000-feet view +--------------------------------------------------------- + +PyPy-STM offers two ways to write multithreaded programs: + +* the traditional way, using the ``thread`` or ``threading`` modules. + +* using ``TransactionQueue``, described next__, as a way to hide the + low-level notion of threads. + +.. __: `transaction.TransactionQueue`_ + +``TransactionQueue`` hides the hard multithreading-related issues that +we typically encounter when using low-level threads. This is not the +first alternative approach to avoid dealing with low-level threads; +for example, OpenMP_ is one. However, it is one of the first ones +which does not require the code to be organized in a particular +fashion. Instead, it works on any Python program which has got +*latent* and *imperfect* parallelism. Ideally, it only requires that +the end programmer identifies where this parallelism is likely to be +found, and communicates it to the system using a simple API. + +.. _OpenMP: http://en.wikipedia.org/wiki/OpenMP + + +transaction.TransactionQueue +---------------------------- In CPU-hungry programs, we can often easily identify outermost loops over some data structure, or other repetitive algorithm, where each @@ -322,7 +358,7 @@ "releasing the GIL". In STM terms, this means blocks of code that are executed while guaranteeing that the transaction is not interrupted in the middle. *This is experimental and may be removed in the future* -if `lock elision`_ is ever implemented. +if `Software lock elision`_ is ever implemented. Here is a direct usage example:: @@ -369,7 +405,8 @@ including with a ``print`` to standard output. If one thread tries to acquire a lock while running in an atomic block, and another thread has got the same lock at that point, then the former may fail with a -``thread.error``. The reason is that "waiting" for some condition to +``thread.error``. (Don't rely on it; it may also deadlock.) +The reason is that "waiting" for some condition to become true --while running in an atomic block-- does not really make sense. For now you can work around it by making sure that, say, all your prints are either in an ``atomic`` block or none of them are. @@ -428,106 +465,38 @@ .. _`software lock elision`: https://www.repository.cam.ac.uk/handle/1810/239410 -Atomic sections, Transactions, etc.: a better way to write parallel programs ----------------------------------------------------------------------------- +Miscellaneous functions +----------------------- -(This section is based on locks as we plan to implement them, but also -works with the existing atomic sections.) - -In the cases where elision works, the block of code can run in parallel -with other blocks of code *even if they are protected by the same lock.* -You still get the illusion that the blocks are run sequentially. This -works even for multiple threads that run each a series of such blocks -and nothing else, protected by one single global lock. This is -basically the Python application-level equivalent of what was done with -the interpreter in ``pypy-stm``: while you think you are writing -thread-unfriendly code because of this global lock, actually the -underlying system is able to make it run on multiple cores anyway. - -This capability can be hidden in a library or in the framework you use; -the end user's code does not need to be explicitly aware of using -threads. For a simple example of this, there is `transaction.py`_ in -``lib_pypy``. The idea is that you write, or already have, some program -where the function ``f(key, value)`` runs on every item of some big -dictionary, say:: - - for key, value in bigdict.items(): - f(key, value) - -Then you simply replace the loop with:: - - for key, value in bigdict.items(): - transaction.add(f, key, value) - transaction.run() - -This code runs the various calls to ``f(key, value)`` using a thread -pool, but every single call is executed under the protection of a unique -lock. The end result is that the behavior is exactly equivalent --- in -fact it makes little sense to do it in this way on a non-STM PyPy or on -CPython. But on ``pypy-stm``, the various locked calls to ``f(key, -value)`` can tentatively be executed in parallel, even if the observable -result is as if they were executed in some serial order. - -This approach hides the notion of threads from the end programmer, -including all the hard multithreading-related issues. This is not the -first alternative approach to explicit threads; for example, OpenMP_ is -one. However, it is one of the first ones which does not require the -code to be organized in a particular fashion. Instead, it works on any -Python program which has got latent, imperfect parallelism. Ideally, it -only requires that the end programmer identifies where this parallelism -is likely to be found, and communicates it to the system, using for -example the ``transaction.add()`` scheme. - -.. _`transaction.py`: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/lib_pypy/transaction.py -.. _OpenMP: http://en.wikipedia.org/wiki/OpenMP - - -.. _`transactional_memory`: - -API of transactional_memory ---------------------------- - -The new pure Python module ``transactional_memory`` runs on both CPython -and PyPy, both with and without STM. It contains: - -* ``getsegmentlimit()``: return the number of "segments" in +* ``transaction.getsegmentlimit()``: return the number of "segments" in this pypy-stm. This is the limit above which more threads will not be able to execute on more cores. (Right now it is limited to 4 due to inter-segment overhead, but should be increased in the future. It should also be settable, and the default value should depend on the number of actual CPUs.) If STM is not available, this returns 1. -* ``print_abort_info(minimum_time=0.0)``: debugging help. Each thread - remembers the longest abort or pause it did because of cross-thread - contention_. This function prints it to ``stderr`` if the time lost - is greater than ``minimum_time`` seconds. The record is then - cleared, to make it ready for new events. This function returns - ``True`` if it printed a report, and ``False`` otherwise. +* ``__pypy__.thread.signals_enabled``: a context manager that runs its + block of code with signals enabled. By default, signals are only + enabled in the main thread; a non-main thread will not receive + signals (this is like CPython). Enabling signals in non-main + threads is useful for libraries where threads are hidden and the end + user is not expecting his code to run elsewhere than in the main + thread. +* ``pypystm.exclusive_atomic``: a context manager similar to + ``transaction.atomic`` but which complains if it is nested. -API of __pypy__.thread ----------------------- +* ``transaction.is_atomic()``: return True if called from an atomic + context. -The ``__pypy__.thread`` submodule is a built-in module of PyPy that -contains a few internal built-in functions used by the -``transactional_memory`` module, plus the following: +* ``pypystm.count()``: return a different positive integer every time + it is called. This works without generating conflicts. The + returned integers are only roughly in increasing order; this should + not be relied upon. -* ``__pypy__.thread.atomic``: a context manager to run a block in - fully atomic mode, without "releasing the GIL". (May be eventually - removed?) -* ``__pypy__.thread.signals_enabled``: a context manager that runs its - block with signals enabled. By default, signals are only enabled in - the main thread; a non-main thread will not receive signals (this is - like CPython). Enabling signals in non-main threads is useful for - libraries where threads are hidden and the end user is not expecting - his code to run elsewhere than in the main thread. - - -.. _contention: - -Conflicts ---------- +More details about conflicts +---------------------------- Based on Software Transactional Memory, the ``pypy-stm`` solution is prone to "conflicts". To repeat the basic idea, threads execute their code @@ -543,7 +512,7 @@ the transaction). If this occurs too often, parallelization fails. How much actual parallelization a multithreaded program can see is a bit -subtle. Basically, a program not using ``__pypy__.thread.atomic`` or +subtle. Basically, a program not using ``transaction.atomic`` or eliding locks, or doing so for very short amounts of time, will parallelize almost freely (as long as it's not some artificial example where, say, all threads try to increase the same global counter and do @@ -555,13 +524,14 @@ overview. Parallelization works as long as two principles are respected. The -first one is that the transactions must not *conflict* with each other. -The most obvious sources of conflicts are threads that all increment a -global shared counter, or that all store the result of their -computations into the same list --- or, more subtly, that all ``pop()`` -the work to do from the same list, because that is also a mutation of -the list. (It is expected that some STM-aware library will eventually -be designed to help with conflict problems, like a STM-aware queue.) +first one is that the transactions must not *conflict* with each +other. The most obvious sources of conflicts are threads that all +increment a global shared counter, or that all store the result of +their computations into the same list --- or, more subtly, that all +``pop()`` the work to do from the same list, because that is also a +mutation of the list. (You can work around it with +``transaction.stmdict``, but for that specific example, some STM-aware +queue should eventually be designed.) A conflict occurs as follows: when a transaction commits (i.e. finishes successfully) it may cause other transactions that are still in progress @@ -577,22 +547,23 @@ Another issue is that of avoiding long-running so-called "inevitable" transactions ("inevitable" is taken in the sense of "which cannot be avoided", i.e. transactions which cannot abort any more). Transactions -like that should only occur if you use ``__pypy__.thread.atomic``, -generally become of I/O in atomic blocks. They work, but the +like that should only occur if you use ``atomic``, +generally because of I/O in atomic blocks. They work, but the transaction is turned inevitable before the I/O is performed. For all the remaining execution time of the atomic block, they will impede parallel work. The best is to organize the code so that such operations -are done completely outside ``__pypy__.thread.atomic``. +are done completely outside ``atomic``. -(This is related to the fact that blocking I/O operations are +(This is not unrelated to the fact that blocking I/O operations are discouraged with Twisted, and if you really need them, you should do them on their own separate thread.) -In case of lock elision, we don't get long-running inevitable -transactions, but a different problem can occur: doing I/O cancels lock -elision, and the lock turns into a real lock, preventing other threads -from committing if they also need this lock. (More about it when lock -elision is implemented and tested.) +In case lock elision eventually replaces atomic sections, we wouldn't +get long-running inevitable transactions, but the same problem occurs +in a different way: doing I/O cancels lock elision, and the lock turns +into a real lock. This prevents other threads from committing if they +also need this lock. (More about it when lock elision is implemented +and tested.) @@ -602,56 +573,18 @@ XXX this section mostly empty for now -Low-level statistics --------------------- - -When a non-main thread finishes, you get low-level statistics printed to -stderr, looking like that:: - - thread 0x7f73377fe600: - outside transaction 42182 0.506 s - run current 85466 0.000 s - run committed 34262 3.178 s - run aborted write write 6982 0.083 s - run aborted write read 550 0.005 s - run aborted inevitable 388 0.010 s - run aborted other 0 0.000 s - wait free segment 0 0.000 s - wait write read 78 0.027 s - wait inevitable 887 0.490 s - wait other 0 0.000 s - sync commit soon 1 0.000 s - bookkeeping 51418 0.606 s - minor gc 162970 1.135 s - major gc 1 0.019 s - sync pause 59173 1.738 s - longest recordered marker 0.000826 s - "File "x.py", line 5, in f" - -On each line, the first number is a counter, and the second number gives -the associated time --- the amount of real time that the thread was in -this state. The sum of all the times should be equal to the total time -between the thread's start and the thread's end. The most important -points are "run committed", which gives the amount of useful work, and -"outside transaction", which should give the time spent e.g. in library -calls (right now it seems to be larger than that; to investigate). The -various "run aborted" and "wait" entries are time lost due to -conflicts_. Everything else is overhead of various forms. (Short-, -medium- and long-term future work involves reducing this overhead :-) - -The last two lines are special; they are an internal marker read by -``transactional_memory.print_abort_info()``. - - Reference to implementation details ----------------------------------- -The core of the implementation is in a separate C library called stmgc_, -in the c7_ subdirectory. Please see the `README.txt`_ for more -information. In particular, the notion of segment is discussed there. +The core of the implementation is in a separate C library called +stmgc_, in the c7_ subdirectory (current version of pypy-stm) and in +the c8_ subdirectory (bleeding edge version). Please see the +`README.txt`_ for more information. In particular, the notion of +segment is discussed there. .. _stmgc: https://bitbucket.org/pypy/stmgc/src/default/ .. _c7: https://bitbucket.org/pypy/stmgc/src/default/c7/ +.. _c8: https://bitbucket.org/pypy/stmgc/src/default/c8/ .. _`README.txt`: https://bitbucket.org/pypy/stmgc/raw/default/c7/README.txt PyPy itself adds on top of it the automatic placement of read__ and write__ From noreply at buildbot.pypy.org Mon Mar 16 12:24:53 2015 From: noreply at buildbot.pypy.org (plan_rich) Date: Mon, 16 Mar 2015 12:24:53 +0100 (CET) Subject: [pypy-commit] pypy default: rpython doc: started documentation of the trace optimizeopt module Message-ID: <20150316112453.0FA5F1C0579@cobra.cs.uni-duesseldorf.de> Author: Richard Plangger Branch: Changeset: r76397:e284a3db9a81 Date: 2015-03-16 10:33 +0100 http://bitbucket.org/pypy/pypy/changeset/e284a3db9a81/ Log: rpython doc: started documentation of the trace optimizeopt module diff --git a/rpython/doc/index.rst b/rpython/doc/index.rst --- a/rpython/doc/index.rst +++ b/rpython/doc/index.rst @@ -63,6 +63,7 @@ translation rtyper garbage_collection + optimizer/overview Indices and tables From noreply at buildbot.pypy.org Mon Mar 16 12:24:54 2015 From: noreply at buildbot.pypy.org (plan_rich) Date: Mon, 16 Mar 2015 12:24:54 +0100 (CET) Subject: [pypy-commit] pypy default: added missing description file Message-ID: <20150316112454.32E211C0579@cobra.cs.uni-duesseldorf.de> Author: Richard Plangger Branch: Changeset: r76398:86ab64843181 Date: 2015-03-16 10:58 +0100 http://bitbucket.org/pypy/pypy/changeset/86ab64843181/ Log: added missing description file diff --git a/rpython/doc/optimizer/overview.rst b/rpython/doc/optimizer/overview.rst new file mode 100644 --- /dev/null +++ b/rpython/doc/optimizer/overview.rst @@ -0,0 +1,193 @@ +.. _trace_optimizer: + +Trace Optimzier +=============== + +Traces of user programs are not directly translated into machine code. +The optimizer module implements several different semantic preserving +transformations that either allow operations to be swept from the trace +or convert them to operations that need less time or space. + +When you try to make sense of this module, this page might get you started. + +Before some optimizations are explained in more detail, it is essential to +understand how traces look like. +The optimizer comes with a test suit. It contains many trace +examples and you might want to take a look at it +(in `rpython/jit/metainterp/optimizeopt/test/*.py`). +The allowed operations can be found in `rpython/jit/metainterp/resoperation.py`. +Here is an example of a trace:: + + [p0,i0,i1] + label(p0, i0, i1) + i2 = getarray_item_raw(p0, i0, descr=) + i3 = int_add(i1,i2) + i4 = int_add(i0,1) + i5 = int_le(i4, 100) # less equal + guard_true(i5) + jump(p0, i4, i3) + +At the beginning it might be clumsy to read but it makes sense when you start +to compare the Python code that constructed the trace:: + + from array import array + a = array('i',range(101)) + sum = 0; i = 0 + while i <= 100: # can be seen as label + sum += a[i] + i += 1 + # jumps back to the while header + +There are better ways to the sum from ``[0..100]``, but it gives a better intuition on how +traces are constructed than ``sum(range(101))``. +Note that the trace syntax is the one used in the test suit. It is also very +similar a printed trace at runtime. The first operation is called ``input``, the +second a ``label``, the last is the backwards ``jump``. + +These instruction mention earlier are special: + +* ``input`` defines the input parameter type and name to enter the trace. +* ``label`` is the instruction a ``jump`` can target. Label instructions have + a ``JitCellToken`` associated that uniquely identifies the label. Any jump + has a target token of a label. + +The token is saved in a so called `descriptor` of the instruction. It is +not written explicitly because it is not done in the tests either. But +the test suit creates a dummy token for each trace and adds it as descriptor +to ``label`` and ``jump``. Of course the optimizer does the same at runtime, +but using real values. +The sample trace includes a descriptor in ``getarrayitem_raw``. Here it +annotates the type of the array. It is a signed integer array. + +High level overview +------------------- + +Before the JIT backend transforms any trace into a machine code, it tries to +transform the trace into an equivalent trace that executes faster. The method +`optimize_trace` in `rpython/jit/metainterp/optimizeopt/__init__.py` is the +main entry point. + +Optimizations are applied in a sequence one after another and the base +sequence is as follows:: + + intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll + +Each of the colon separated name has a class attached. It is later +instantiated as a subclass of `Optimization`. The `Optimizer` class +derives from the `Optimization` class and implements the control logic for +the optimization. Most of the optimizations only require a single forward pass. +The trace is 'propagated' in to each optimization using the method +`propagate_forward`. Instruction by instruction then flows from the +first optimization to the last optimization. The method `emit_operation` +is called for every operation that is passed to the next optimizer. + +A frequently encountered pattern +-------------------------------- + +To find potential optimization targets it is necessary to know the instruction +type. Simple solution is to switch using the operation number (= type):: + + for op in operations: + if op.getopnum() == rop.INT_ADD: + # handle this instruction + pass + elif op.getopnum() == rop.INT_FLOOR_DIV: + pass + # and many more + +Things get worse if you start to match the arguments +(is argument one constant and two variable or vice versa?). The pattern to tackle +this code bloat is to move it to a separate method using +`make_dispatcher_method`. It associates methods with instruction types:: + + class OptX(Optimization): + def prefix_INT_ADD(self, op): + pass # emit, transform, ... + + dispatch_opt = make_dispatcher_method(OptX, 'prefix_', + default=OptX.emit_operation) + OptX.propagate_forward = dispatch_opt + + optX = OptX() + for op in operations: + optX.propagate_forward(op) + +``propagate_forward`` searches for the method that is able to handle the instruction +type. As an example `INT_ADD` will invoke `prefix_INT_ADD`. If there is no function +for the instruction, it is routed to the default implementation (``emit_operation`` +in this example). + +Rewrite optimization +-------------------- + +The second optimization is called 'rewrite' an is commonly also known as +strength reduction. A simple example would be that an integer multiplied +by 2 is equivalent to the bits shifted to the left once +(e.g. ``x * 2 == x << 1``). Not only strength reduction is done in this +optimization but also boolean or arithmetic simplifications. Other examples +would be: ``x & 0 == 0``, ``x - 0 == x`` + +Whenever such an operation is encountered (e.g. ``x & 0``), no operation is +emitted. Instead the variable of x is made equal to 0 +(= ``make_equal_to(op.result, 0)``). The variables found in a trace are +instances of Box classes that can be found in +`rpython/jit/metainterp/history.py`. `OptValue` wraps those variables again +and maps the boxes to the optimization values in the optimizer. When a +value is made equal, the box in the opt. value. This renders a new value +to any further access. +As a result the optimizer must provide the means to access the ``Box`` +instances. The instance method `make_args_key` returns the boxed value. + +**NOTE: that OptValue is likely to to be replaced in near future.** + +Pure optimization +----------------- + +Is interwoven into the basic optimizer. It saves operations, results, +arguments to be known to have pure semantics. + +Pure is free of side effects and it is referentially transparent +(the operation can be replaced with its value without changing the program +semantics). The operations marked as ALWAYS_PURE in `resoperation.py` is a +subset of the SIDEEFFECT free operations. Operations such as new, new array, +getfield_(raw/gc) are marked SIDEEFFECT free but not as ALWAYS_PURE. + +This can be seen as memoization technique. Once an operation proved to +be 'pure' it is saved and should not be recomputed later. + +Unroll optimization +------------------- + +A detailed description can be found the document +`Loop-Aware Optimizations in PyPy's Tracing JIT`__ + +.. __: http://www2.maths.lth.se/matematiklth/vision/publdb/reports/pdf/ardo-bolz-etal-dls-12.pdf + +This optimization does not fall into the traditional scheme of one forward +pass only. In a nutshell it unrolls the trace _once_, connects the two +traces (by inserting parameters into the jump and label of the peeled trace) +and uses information to iron out allocations, propagate constants and +do any other optimization currently present in the 'optimizeopt' module. + +It is prepended all optimizations and thus extends the Optimizer class +and unrolls the loop once before it proceeds. + + +What is missing? +---------------- + +* Guards are not explained +* Several optimizations + + +Further references +------------------ + +* `Allocation Removal by Partial Evaluation in a Tracing JIT`__ +* `Loop-Aware Optimizations in PyPy's Tracing JIT`__ + +.. __: http://www.stups.uni-duesseldorf.de/mediawiki/images/b/b0/Pub-BoCuFiLePeRi2011.pdf +.. __: http://www2.maths.lth.se/matematiklth/vision/publdb/reports/pdf/ardo-bolz-etal-dls-12.pdf + + + From noreply at buildbot.pypy.org Mon Mar 16 12:24:55 2015 From: noreply at buildbot.pypy.org (plan_rich) Date: Mon, 16 Mar 2015 12:24:55 +0100 (CET) Subject: [pypy-commit] pypy default: moved optimizer/overview.rst -> jit/optimizer.rst Message-ID: <20150316112455.53ECA1C0579@cobra.cs.uni-duesseldorf.de> Author: Richard Plangger Branch: Changeset: r76399:cea0639cfe6c Date: 2015-03-16 11:48 +0100 http://bitbucket.org/pypy/pypy/changeset/cea0639cfe6c/ Log: moved optimizer/overview.rst -> jit/optimizer.rst updated index.rst to link to it diff --git a/rpython/doc/index.rst b/rpython/doc/index.rst --- a/rpython/doc/index.rst +++ b/rpython/doc/index.rst @@ -63,7 +63,7 @@ translation rtyper garbage_collection - optimizer/overview + jit/optimizer Indices and tables diff --git a/rpython/doc/jit/optimizer.rst b/rpython/doc/jit/optimizer.rst new file mode 100644 --- /dev/null +++ b/rpython/doc/jit/optimizer.rst @@ -0,0 +1,193 @@ +.. _trace_optimizer: + +Trace Optimzier +=============== + +Traces of user programs are not directly translated into machine code. +The optimizer module implements several different semantic preserving +transformations that either allow operations to be swept from the trace +or convert them to operations that need less time or space. + +When you try to make sense of this module, this page might get you started. + +Before some optimizations are explained in more detail, it is essential to +understand how traces look like. +The optimizer comes with a test suit. It contains many trace +examples and you might want to take a look at it +(in `rpython/jit/metainterp/optimizeopt/test/*.py`). +The allowed operations can be found in `rpython/jit/metainterp/resoperation.py`. +Here is an example of a trace:: + + [p0,i0,i1] + label(p0, i0, i1) + i2 = getarray_item_raw(p0, i0, descr=) + i3 = int_add(i1,i2) + i4 = int_add(i0,1) + i5 = int_le(i4, 100) # less equal + guard_true(i5) + jump(p0, i4, i3) + +At the beginning it might be clumsy to read but it makes sense when you start +to compare the Python code that constructed the trace:: + + from array import array + a = array('i',range(101)) + sum = 0; i = 0 + while i <= 100: # can be seen as label + sum += a[i] + i += 1 + # jumps back to the while header + +There are better ways to the sum from ``[0..100]``, but it gives a better intuition on how +traces are constructed than ``sum(range(101))``. +Note that the trace syntax is the one used in the test suit. It is also very +similar a printed trace at runtime. The first operation is called ``input``, the +second a ``label``, the last is the backwards ``jump``. + +These instruction mention earlier are special: + +* ``input`` defines the input parameter type and name to enter the trace. +* ``label`` is the instruction a ``jump`` can target. Label instructions have + a ``JitCellToken`` associated that uniquely identifies the label. Any jump + has a target token of a label. + +The token is saved in a so called `descriptor` of the instruction. It is +not written explicitly because it is not done in the tests either. But +the test suit creates a dummy token for each trace and adds it as descriptor +to ``label`` and ``jump``. Of course the optimizer does the same at runtime, +but using real values. +The sample trace includes a descriptor in ``getarrayitem_raw``. Here it +annotates the type of the array. It is a signed integer array. + +High level overview +------------------- + +Before the JIT backend transforms any trace into a machine code, it tries to +transform the trace into an equivalent trace that executes faster. The method +`optimize_trace` in `rpython/jit/metainterp/optimizeopt/__init__.py` is the +main entry point. + +Optimizations are applied in a sequence one after another and the base +sequence is as follows:: + + intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll + +Each of the colon separated name has a class attached. It is later +instantiated as a subclass of `Optimization`. The `Optimizer` class +derives from the `Optimization` class and implements the control logic for +the optimization. Most of the optimizations only require a single forward pass. +The trace is 'propagated' in to each optimization using the method +`propagate_forward`. Instruction by instruction then flows from the +first optimization to the last optimization. The method `emit_operation` +is called for every operation that is passed to the next optimizer. + +A frequently encountered pattern +-------------------------------- + +To find potential optimization targets it is necessary to know the instruction +type. Simple solution is to switch using the operation number (= type):: + + for op in operations: + if op.getopnum() == rop.INT_ADD: + # handle this instruction + pass + elif op.getopnum() == rop.INT_FLOOR_DIV: + pass + # and many more + +Things get worse if you start to match the arguments +(is argument one constant and two variable or vice versa?). The pattern to tackle +this code bloat is to move it to a separate method using +`make_dispatcher_method`. It associates methods with instruction types:: + + class OptX(Optimization): + def prefix_INT_ADD(self, op): + pass # emit, transform, ... + + dispatch_opt = make_dispatcher_method(OptX, 'prefix_', + default=OptX.emit_operation) + OptX.propagate_forward = dispatch_opt + + optX = OptX() + for op in operations: + optX.propagate_forward(op) + +``propagate_forward`` searches for the method that is able to handle the instruction +type. As an example `INT_ADD` will invoke `prefix_INT_ADD`. If there is no function +for the instruction, it is routed to the default implementation (``emit_operation`` +in this example). + +Rewrite optimization +-------------------- + +The second optimization is called 'rewrite' an is commonly also known as +strength reduction. A simple example would be that an integer multiplied +by 2 is equivalent to the bits shifted to the left once +(e.g. ``x * 2 == x << 1``). Not only strength reduction is done in this +optimization but also boolean or arithmetic simplifications. Other examples +would be: ``x & 0 == 0``, ``x - 0 == x`` + +Whenever such an operation is encountered (e.g. ``x & 0``), no operation is +emitted. Instead the variable of x is made equal to 0 +(= ``make_equal_to(op.result, 0)``). The variables found in a trace are +instances of Box classes that can be found in +`rpython/jit/metainterp/history.py`. `OptValue` wraps those variables again +and maps the boxes to the optimization values in the optimizer. When a +value is made equal, the box in the opt. value. This renders a new value +to any further access. +As a result the optimizer must provide the means to access the ``Box`` +instances. The instance method `make_args_key` returns the boxed value. + +**NOTE: that OptValue is likely to to be replaced in near future.** + +Pure optimization +----------------- + +Is interwoven into the basic optimizer. It saves operations, results, +arguments to be known to have pure semantics. + +Pure is free of side effects and it is referentially transparent +(the operation can be replaced with its value without changing the program +semantics). The operations marked as ALWAYS_PURE in `resoperation.py` is a +subset of the SIDEEFFECT free operations. Operations such as new, new array, +getfield_(raw/gc) are marked SIDEEFFECT free but not as ALWAYS_PURE. + +This can be seen as memoization technique. Once an operation proved to +be 'pure' it is saved and should not be recomputed later. + +Unroll optimization +------------------- + +A detailed description can be found the document +`Loop-Aware Optimizations in PyPy's Tracing JIT`__ + +.. __: http://www2.maths.lth.se/matematiklth/vision/publdb/reports/pdf/ardo-bolz-etal-dls-12.pdf + +This optimization does not fall into the traditional scheme of one forward +pass only. In a nutshell it unrolls the trace _once_, connects the two +traces (by inserting parameters into the jump and label of the peeled trace) +and uses information to iron out allocations, propagate constants and +do any other optimization currently present in the 'optimizeopt' module. + +It is prepended all optimizations and thus extends the Optimizer class +and unrolls the loop once before it proceeds. + + +What is missing? +---------------- + +* Guards are not explained +* Several optimizations + + +Further references +------------------ + +* `Allocation Removal by Partial Evaluation in a Tracing JIT`__ +* `Loop-Aware Optimizations in PyPy's Tracing JIT`__ + +.. __: http://www.stups.uni-duesseldorf.de/mediawiki/images/b/b0/Pub-BoCuFiLePeRi2011.pdf +.. __: http://www2.maths.lth.se/matematiklth/vision/publdb/reports/pdf/ardo-bolz-etal-dls-12.pdf + + + From noreply at buildbot.pypy.org Mon Mar 16 12:24:56 2015 From: noreply at buildbot.pypy.org (plan_rich) Date: Mon, 16 Mar 2015 12:24:56 +0100 (CET) Subject: [pypy-commit] pypy default: removed old file Message-ID: <20150316112456.872C91C0579@cobra.cs.uni-duesseldorf.de> Author: Richard Plangger Branch: Changeset: r76400:6793c07d2ea7 Date: 2015-03-16 11:51 +0100 http://bitbucket.org/pypy/pypy/changeset/6793c07d2ea7/ Log: removed old file diff --git a/rpython/doc/optimizer/overview.rst b/rpython/doc/optimizer/overview.rst deleted file mode 100644 --- a/rpython/doc/optimizer/overview.rst +++ /dev/null @@ -1,193 +0,0 @@ -.. _trace_optimizer: - -Trace Optimzier -=============== - -Traces of user programs are not directly translated into machine code. -The optimizer module implements several different semantic preserving -transformations that either allow operations to be swept from the trace -or convert them to operations that need less time or space. - -When you try to make sense of this module, this page might get you started. - -Before some optimizations are explained in more detail, it is essential to -understand how traces look like. -The optimizer comes with a test suit. It contains many trace -examples and you might want to take a look at it -(in `rpython/jit/metainterp/optimizeopt/test/*.py`). -The allowed operations can be found in `rpython/jit/metainterp/resoperation.py`. -Here is an example of a trace:: - - [p0,i0,i1] - label(p0, i0, i1) - i2 = getarray_item_raw(p0, i0, descr=) - i3 = int_add(i1,i2) - i4 = int_add(i0,1) - i5 = int_le(i4, 100) # less equal - guard_true(i5) - jump(p0, i4, i3) - -At the beginning it might be clumsy to read but it makes sense when you start -to compare the Python code that constructed the trace:: - - from array import array - a = array('i',range(101)) - sum = 0; i = 0 - while i <= 100: # can be seen as label - sum += a[i] - i += 1 - # jumps back to the while header - -There are better ways to the sum from ``[0..100]``, but it gives a better intuition on how -traces are constructed than ``sum(range(101))``. -Note that the trace syntax is the one used in the test suit. It is also very -similar a printed trace at runtime. The first operation is called ``input``, the -second a ``label``, the last is the backwards ``jump``. - -These instruction mention earlier are special: - -* ``input`` defines the input parameter type and name to enter the trace. -* ``label`` is the instruction a ``jump`` can target. Label instructions have - a ``JitCellToken`` associated that uniquely identifies the label. Any jump - has a target token of a label. - -The token is saved in a so called `descriptor` of the instruction. It is -not written explicitly because it is not done in the tests either. But -the test suit creates a dummy token for each trace and adds it as descriptor -to ``label`` and ``jump``. Of course the optimizer does the same at runtime, -but using real values. -The sample trace includes a descriptor in ``getarrayitem_raw``. Here it -annotates the type of the array. It is a signed integer array. - -High level overview -------------------- - -Before the JIT backend transforms any trace into a machine code, it tries to -transform the trace into an equivalent trace that executes faster. The method -`optimize_trace` in `rpython/jit/metainterp/optimizeopt/__init__.py` is the -main entry point. - -Optimizations are applied in a sequence one after another and the base -sequence is as follows:: - - intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll - -Each of the colon separated name has a class attached. It is later -instantiated as a subclass of `Optimization`. The `Optimizer` class -derives from the `Optimization` class and implements the control logic for -the optimization. Most of the optimizations only require a single forward pass. -The trace is 'propagated' in to each optimization using the method -`propagate_forward`. Instruction by instruction then flows from the -first optimization to the last optimization. The method `emit_operation` -is called for every operation that is passed to the next optimizer. - -A frequently encountered pattern --------------------------------- - -To find potential optimization targets it is necessary to know the instruction -type. Simple solution is to switch using the operation number (= type):: - - for op in operations: - if op.getopnum() == rop.INT_ADD: - # handle this instruction - pass - elif op.getopnum() == rop.INT_FLOOR_DIV: - pass - # and many more - -Things get worse if you start to match the arguments -(is argument one constant and two variable or vice versa?). The pattern to tackle -this code bloat is to move it to a separate method using -`make_dispatcher_method`. It associates methods with instruction types:: - - class OptX(Optimization): - def prefix_INT_ADD(self, op): - pass # emit, transform, ... - - dispatch_opt = make_dispatcher_method(OptX, 'prefix_', - default=OptX.emit_operation) - OptX.propagate_forward = dispatch_opt - - optX = OptX() - for op in operations: - optX.propagate_forward(op) - -``propagate_forward`` searches for the method that is able to handle the instruction -type. As an example `INT_ADD` will invoke `prefix_INT_ADD`. If there is no function -for the instruction, it is routed to the default implementation (``emit_operation`` -in this example). - -Rewrite optimization --------------------- - -The second optimization is called 'rewrite' an is commonly also known as -strength reduction. A simple example would be that an integer multiplied -by 2 is equivalent to the bits shifted to the left once -(e.g. ``x * 2 == x << 1``). Not only strength reduction is done in this -optimization but also boolean or arithmetic simplifications. Other examples -would be: ``x & 0 == 0``, ``x - 0 == x`` - -Whenever such an operation is encountered (e.g. ``x & 0``), no operation is -emitted. Instead the variable of x is made equal to 0 -(= ``make_equal_to(op.result, 0)``). The variables found in a trace are -instances of Box classes that can be found in -`rpython/jit/metainterp/history.py`. `OptValue` wraps those variables again -and maps the boxes to the optimization values in the optimizer. When a -value is made equal, the box in the opt. value. This renders a new value -to any further access. -As a result the optimizer must provide the means to access the ``Box`` -instances. The instance method `make_args_key` returns the boxed value. - -**NOTE: that OptValue is likely to to be replaced in near future.** - -Pure optimization ------------------ - -Is interwoven into the basic optimizer. It saves operations, results, -arguments to be known to have pure semantics. - -Pure is free of side effects and it is referentially transparent -(the operation can be replaced with its value without changing the program -semantics). The operations marked as ALWAYS_PURE in `resoperation.py` is a -subset of the SIDEEFFECT free operations. Operations such as new, new array, -getfield_(raw/gc) are marked SIDEEFFECT free but not as ALWAYS_PURE. - -This can be seen as memoization technique. Once an operation proved to -be 'pure' it is saved and should not be recomputed later. - -Unroll optimization -------------------- - -A detailed description can be found the document -`Loop-Aware Optimizations in PyPy's Tracing JIT`__ - -.. __: http://www2.maths.lth.se/matematiklth/vision/publdb/reports/pdf/ardo-bolz-etal-dls-12.pdf - -This optimization does not fall into the traditional scheme of one forward -pass only. In a nutshell it unrolls the trace _once_, connects the two -traces (by inserting parameters into the jump and label of the peeled trace) -and uses information to iron out allocations, propagate constants and -do any other optimization currently present in the 'optimizeopt' module. - -It is prepended all optimizations and thus extends the Optimizer class -and unrolls the loop once before it proceeds. - - -What is missing? ----------------- - -* Guards are not explained -* Several optimizations - - -Further references ------------------- - -* `Allocation Removal by Partial Evaluation in a Tracing JIT`__ -* `Loop-Aware Optimizations in PyPy's Tracing JIT`__ - -.. __: http://www.stups.uni-duesseldorf.de/mediawiki/images/b/b0/Pub-BoCuFiLePeRi2011.pdf -.. __: http://www2.maths.lth.se/matematiklth/vision/publdb/reports/pdf/ardo-bolz-etal-dls-12.pdf - - - From noreply at buildbot.pypy.org Mon Mar 16 12:24:57 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 16 Mar 2015 12:24:57 +0100 (CET) Subject: [pypy-commit] pypy default: Put the link to jit/optimizer along with the links to the other files in Message-ID: <20150316112457.99EBE1C0579@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76401:9e1a77cd0e5c Date: 2015-03-16 12:05 +0100 http://bitbucket.org/pypy/pypy/changeset/9e1a77cd0e5c/ Log: Put the link to jit/optimizer along with the links to the other files in the jit/ subdirectory diff --git a/rpython/doc/index.rst b/rpython/doc/index.rst --- a/rpython/doc/index.rst +++ b/rpython/doc/index.rst @@ -63,7 +63,6 @@ translation rtyper garbage_collection - jit/optimizer Indices and tables diff --git a/rpython/doc/jit/index.rst b/rpython/doc/jit/index.rst --- a/rpython/doc/jit/index.rst +++ b/rpython/doc/jit/index.rst @@ -23,11 +23,15 @@ overview pyjitpl5 + optimizer virtualizable - :doc:`Overview `: motivating our approach - :doc:`Notes ` about the current work in PyPy +- :doc:`Optimizer `: the step between tracing and writing + machine code + - :doc:`Virtulizable ` how virtualizables work and what they are (in other words how to make frames more efficient). From noreply at buildbot.pypy.org Mon Mar 16 12:24:58 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 16 Mar 2015 12:24:58 +0100 (CET) Subject: [pypy-commit] pypy default: Typo-tracking, and some extra explaining here and there Message-ID: <20150316112458.B8AFE1C0579@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76402:6022cd1339ad Date: 2015-03-16 12:24 +0100 http://bitbucket.org/pypy/pypy/changeset/6022cd1339ad/ Log: Typo-tracking, and some extra explaining here and there diff --git a/rpython/doc/jit/optimizer.rst b/rpython/doc/jit/optimizer.rst --- a/rpython/doc/jit/optimizer.rst +++ b/rpython/doc/jit/optimizer.rst @@ -1,13 +1,14 @@ .. _trace_optimizer: -Trace Optimzier -=============== +Trace Optimizier +================ Traces of user programs are not directly translated into machine code. The optimizer module implements several different semantic preserving transformations that either allow operations to be swept from the trace or convert them to operations that need less time or space. +The optimizer is in `rpython/jit/metainterp/optimizeopt/`. When you try to make sense of this module, this page might get you started. Before some optimizations are explained in more detail, it is essential to @@ -23,7 +24,7 @@ i2 = getarray_item_raw(p0, i0, descr=) i3 = int_add(i1,i2) i4 = int_add(i0,1) - i5 = int_le(i4, 100) # less equal + i5 = int_le(i4, 100) # lower-or-equal guard_true(i5) jump(p0, i4, i3) @@ -38,22 +39,24 @@ i += 1 # jumps back to the while header -There are better ways to the sum from ``[0..100]``, but it gives a better intuition on how +There are better ways to compute the sum from ``[0..100]``, but it gives a better intuition on how traces are constructed than ``sum(range(101))``. -Note that the trace syntax is the one used in the test suit. It is also very -similar a printed trace at runtime. The first operation is called ``input``, the -second a ``label``, the last is the backwards ``jump``. +Note that the trace syntax is the one used in the test suite. It is also very +similar to traces printed at runtime by PYPYLOG_. The first line gives the input variables, the +second line is a ``label`` operation, the last one is the backwards ``jump`` operation. -These instruction mention earlier are special: +.. _PYPYLOG: logging.html -* ``input`` defines the input parameter type and name to enter the trace. +These instructions mentioned earlier are special: + +* the input defines the input parameter type and name to enter the trace. * ``label`` is the instruction a ``jump`` can target. Label instructions have a ``JitCellToken`` associated that uniquely identifies the label. Any jump has a target token of a label. The token is saved in a so called `descriptor` of the instruction. It is not written explicitly because it is not done in the tests either. But -the test suit creates a dummy token for each trace and adds it as descriptor +the test suite creates a dummy token for each trace and adds it as descriptor to ``label`` and ``jump``. Of course the optimizer does the same at runtime, but using real values. The sample trace includes a descriptor in ``getarrayitem_raw``. Here it @@ -62,7 +65,7 @@ High level overview ------------------- -Before the JIT backend transforms any trace into a machine code, it tries to +Before the JIT backend transforms any trace into machine code, it tries to transform the trace into an equivalent trace that executes faster. The method `optimize_trace` in `rpython/jit/metainterp/optimizeopt/__init__.py` is the main entry point. @@ -72,12 +75,12 @@ intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll -Each of the colon separated name has a class attached. It is later -instantiated as a subclass of `Optimization`. The `Optimizer` class +Each of the colon-separated name has a class attached, inheriting from +the `Optimization` class. The `Optimizer` class itself also derives from the `Optimization` class and implements the control logic for the optimization. Most of the optimizations only require a single forward pass. -The trace is 'propagated' in to each optimization using the method -`propagate_forward`. Instruction by instruction then flows from the +The trace is 'propagated' into each optimization using the method +`propagate_forward`. Instruction by instruction, it flows from the first optimization to the last optimization. The method `emit_operation` is called for every operation that is passed to the next optimizer. @@ -120,25 +123,23 @@ Rewrite optimization -------------------- -The second optimization is called 'rewrite' an is commonly also known as +The second optimization is called 'rewrite' and is commonly also known as strength reduction. A simple example would be that an integer multiplied by 2 is equivalent to the bits shifted to the left once (e.g. ``x * 2 == x << 1``). Not only strength reduction is done in this optimization but also boolean or arithmetic simplifications. Other examples would be: ``x & 0 == 0``, ``x - 0 == x`` -Whenever such an operation is encountered (e.g. ``x & 0``), no operation is -emitted. Instead the variable of x is made equal to 0 +Whenever such an operation is encountered (e.g. ``y = x & 0``), no operation is +emitted. Instead the variable y is made equal to 0 (= ``make_equal_to(op.result, 0)``). The variables found in a trace are instances of Box classes that can be found in `rpython/jit/metainterp/history.py`. `OptValue` wraps those variables again and maps the boxes to the optimization values in the optimizer. When a -value is made equal, the box in the opt. value. This renders a new value -to any further access. -As a result the optimizer must provide the means to access the ``Box`` -instances. The instance method `make_args_key` returns the boxed value. +value is made equal, the two variable's boxes are made to point to the same +`OptValue` instance. -**NOTE: that OptValue is likely to to be replaced in near future.** +**NOTE: this OptValue organization is currently being refactored in a branch.** Pure optimization ----------------- @@ -146,14 +147,19 @@ Is interwoven into the basic optimizer. It saves operations, results, arguments to be known to have pure semantics. -Pure is free of side effects and it is referentially transparent -(the operation can be replaced with its value without changing the program -semantics). The operations marked as ALWAYS_PURE in `resoperation.py` is a -subset of the SIDEEFFECT free operations. Operations such as new, new array, -getfield_(raw/gc) are marked SIDEEFFECT free but not as ALWAYS_PURE. +"Pure" here means the same as the ``jit.elidable`` decorator: +free of "observable" side effects and referentially transparent +(the operation can be replaced with its result without changing the program +semantics). The operations marked as ALWAYS_PURE in `resoperation.py` are a +subset of the NOSIDEEFFECT operations. Operations such as new, new array, +getfield_(raw/gc) are marked as NOSIDEEFFECT but not as ALWAYS_PURE. -This can be seen as memoization technique. Once an operation proved to -be 'pure' it is saved and should not be recomputed later. +Pure operations are optimized in two different ways. If their arguments +are constants, the operation is removed and the result is turned into a +constant. If not, we can still use a memoization technique: if, later, +we see the same operation on the same arguments again, we don't need to +recompute its result, but can simply reuse the previous operation's +result. Unroll optimization ------------------- @@ -169,15 +175,15 @@ and uses information to iron out allocations, propagate constants and do any other optimization currently present in the 'optimizeopt' module. -It is prepended all optimizations and thus extends the Optimizer class +It is prepended to all optimizations and thus extends the Optimizer class and unrolls the loop once before it proceeds. -What is missing? ----------------- +What is missing from this document +---------------------------------- * Guards are not explained -* Several optimizations +* Several optimizations are not explained Further references @@ -188,6 +194,3 @@ .. __: http://www.stups.uni-duesseldorf.de/mediawiki/images/b/b0/Pub-BoCuFiLePeRi2011.pdf .. __: http://www2.maths.lth.se/matematiklth/vision/publdb/reports/pdf/ardo-bolz-etal-dls-12.pdf - - - From noreply at buildbot.pypy.org Mon Mar 16 12:24:59 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 16 Mar 2015 12:24:59 +0100 (CET) Subject: [pypy-commit] pypy default: merge heads Message-ID: <20150316112459.D64FE1C0579@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76403:0488deef2308 Date: 2015-03-16 12:24 +0100 http://bitbucket.org/pypy/pypy/changeset/0488deef2308/ Log: merge heads diff --git a/pypy/doc/stm.rst b/pypy/doc/stm.rst --- a/pypy/doc/stm.rst +++ b/pypy/doc/stm.rst @@ -46,13 +46,14 @@ multiple cores. * ``pypy-stm`` provides (but does not impose) a special API to the - user in the pure Python module `transaction`_. This module is based - on the lower-level module `pypystm`_, but also provides some + user in the pure Python module ``transaction``. This module is based + on the lower-level module ``pypystm``, but also provides some compatibily with non-STM PyPy's or CPython's. * Building on top of the way the GIL is removed, we will talk - about `Atomic sections, Transactions, etc.: a better way to write - parallel programs`_. + about `How to write multithreaded programs: the 10'000-feet view`_ + and `transaction.TransactionQueue`_. + Getting Started @@ -89,7 +90,7 @@ Current status (stmgc-c7) ------------------------- -* It seems to work fine, without crashing any more. Please `report +* **NEW:** It seems to work fine, without crashing any more. Please `report any crash`_ you find (or other bugs). * It runs with an overhead as low as 20% on examples like "richards". @@ -97,24 +98,35 @@ 2x for "translate.py"-- which we are still trying to understand. One suspect is our partial GC implementation, see below. +* **NEW:** the ``PYPYSTM`` environment variable and the + ``pypy/stm/print_stm_log.py`` script let you know exactly which + "conflicts" occurred. This is described in the section + `transaction.TransactionQueue`_ below. + +* **NEW:** special transaction-friendly APIs (like ``stmdict``), + described in the section `transaction.TransactionQueue`_ below. The + old API changed again, mostly moving to different modules. Sorry + about that. I feel it's a better idea to change the API early + instead of being stuck with a bad one later... + * Currently limited to 1.5 GB of RAM (this is just a parameter in `core.h`__ -- theoretically. In practice, increase it too much and clang crashes again). Memory overflows are not correctly handled; they cause segfaults. -* The JIT warm-up time improved recently but is still bad. In order to - produce machine code, the JIT needs to enter a special single-threaded - mode for now. This means that you will get bad performance results if - your program doesn't run for several seconds, where *several* can mean - *many.* When trying benchmarks, be sure to check that you have - reached the warmed state, i.e. the performance is not improving any - more. This should be clear from the fact that as long as it's - producing more machine code, ``pypy-stm`` will run on a single core. +* **NEW:** The JIT warm-up time improved again, but is still + relatively large. In order to produce machine code, the JIT needs + to enter "inevitable" mode. This means that you will get bad + performance results if your program doesn't run for several seconds, + where *several* can mean *many.* When trying benchmarks, be sure to + check that you have reached the warmed state, i.e. the performance + is not improving any more. * The GC is new; although clearly inspired by PyPy's regular GC, it misses a number of optimizations for now. Programs allocating large numbers of small objects that don't immediately die (surely a common - situation) suffer from these missing optimizations. + situation) suffer from these missing optimizations. (The bleeding + edge ``stmgc-c8`` is better at that.) * Weakrefs might appear to work a bit strangely for now, sometimes staying alive throught ``gc.collect()``, or even dying but then @@ -122,8 +134,7 @@ * The STM system is based on very efficient read/write barriers, which are mostly done (their placement could be improved a bit in - JIT-generated machine code). But the overall bookkeeping logic could - see more improvements (see `Low-level statistics`_ below). + JIT-generated machine code). * Forking the process is slow because the complete memory needs to be copied manually. A warning is printed to this effect. @@ -132,7 +143,8 @@ crash on an assertion error because of a non-implemented overflow of an internal 28-bit counter. -.. _`report bugs`: https://bugs.pypy.org/ + +.. _`report any crash`: https://bitbucket.org/pypy/pypy/issues?status=new&status=open .. __: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/rpython/translator/stm/src_stm/stm/core.h @@ -155,7 +167,6 @@ interpreter and other ones might have slightly different needs. - User Guide ========== @@ -181,8 +192,33 @@ order. -A better way to write parallel programs ---------------------------------------- +How to write multithreaded programs: the 10'000-feet view +--------------------------------------------------------- + +PyPy-STM offers two ways to write multithreaded programs: + +* the traditional way, using the ``thread`` or ``threading`` modules. + +* using ``TransactionQueue``, described next__, as a way to hide the + low-level notion of threads. + +.. __: `transaction.TransactionQueue`_ + +``TransactionQueue`` hides the hard multithreading-related issues that +we typically encounter when using low-level threads. This is not the +first alternative approach to avoid dealing with low-level threads; +for example, OpenMP_ is one. However, it is one of the first ones +which does not require the code to be organized in a particular +fashion. Instead, it works on any Python program which has got +*latent* and *imperfect* parallelism. Ideally, it only requires that +the end programmer identifies where this parallelism is likely to be +found, and communicates it to the system using a simple API. + +.. _OpenMP: http://en.wikipedia.org/wiki/OpenMP + + +transaction.TransactionQueue +---------------------------- In CPU-hungry programs, we can often easily identify outermost loops over some data structure, or other repetitive algorithm, where each @@ -322,7 +358,7 @@ "releasing the GIL". In STM terms, this means blocks of code that are executed while guaranteeing that the transaction is not interrupted in the middle. *This is experimental and may be removed in the future* -if `lock elision`_ is ever implemented. +if `Software lock elision`_ is ever implemented. Here is a direct usage example:: @@ -369,7 +405,8 @@ including with a ``print`` to standard output. If one thread tries to acquire a lock while running in an atomic block, and another thread has got the same lock at that point, then the former may fail with a -``thread.error``. The reason is that "waiting" for some condition to +``thread.error``. (Don't rely on it; it may also deadlock.) +The reason is that "waiting" for some condition to become true --while running in an atomic block-- does not really make sense. For now you can work around it by making sure that, say, all your prints are either in an ``atomic`` block or none of them are. @@ -428,106 +465,38 @@ .. _`software lock elision`: https://www.repository.cam.ac.uk/handle/1810/239410 -Atomic sections, Transactions, etc.: a better way to write parallel programs ----------------------------------------------------------------------------- +Miscellaneous functions +----------------------- -(This section is based on locks as we plan to implement them, but also -works with the existing atomic sections.) - -In the cases where elision works, the block of code can run in parallel -with other blocks of code *even if they are protected by the same lock.* -You still get the illusion that the blocks are run sequentially. This -works even for multiple threads that run each a series of such blocks -and nothing else, protected by one single global lock. This is -basically the Python application-level equivalent of what was done with -the interpreter in ``pypy-stm``: while you think you are writing -thread-unfriendly code because of this global lock, actually the -underlying system is able to make it run on multiple cores anyway. - -This capability can be hidden in a library or in the framework you use; -the end user's code does not need to be explicitly aware of using -threads. For a simple example of this, there is `transaction.py`_ in -``lib_pypy``. The idea is that you write, or already have, some program -where the function ``f(key, value)`` runs on every item of some big -dictionary, say:: - - for key, value in bigdict.items(): - f(key, value) - -Then you simply replace the loop with:: - - for key, value in bigdict.items(): - transaction.add(f, key, value) - transaction.run() - -This code runs the various calls to ``f(key, value)`` using a thread -pool, but every single call is executed under the protection of a unique -lock. The end result is that the behavior is exactly equivalent --- in -fact it makes little sense to do it in this way on a non-STM PyPy or on -CPython. But on ``pypy-stm``, the various locked calls to ``f(key, -value)`` can tentatively be executed in parallel, even if the observable -result is as if they were executed in some serial order. - -This approach hides the notion of threads from the end programmer, -including all the hard multithreading-related issues. This is not the -first alternative approach to explicit threads; for example, OpenMP_ is -one. However, it is one of the first ones which does not require the -code to be organized in a particular fashion. Instead, it works on any -Python program which has got latent, imperfect parallelism. Ideally, it -only requires that the end programmer identifies where this parallelism -is likely to be found, and communicates it to the system, using for -example the ``transaction.add()`` scheme. - -.. _`transaction.py`: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/lib_pypy/transaction.py -.. _OpenMP: http://en.wikipedia.org/wiki/OpenMP - - -.. _`transactional_memory`: - -API of transactional_memory ---------------------------- - -The new pure Python module ``transactional_memory`` runs on both CPython -and PyPy, both with and without STM. It contains: - -* ``getsegmentlimit()``: return the number of "segments" in +* ``transaction.getsegmentlimit()``: return the number of "segments" in this pypy-stm. This is the limit above which more threads will not be able to execute on more cores. (Right now it is limited to 4 due to inter-segment overhead, but should be increased in the future. It should also be settable, and the default value should depend on the number of actual CPUs.) If STM is not available, this returns 1. -* ``print_abort_info(minimum_time=0.0)``: debugging help. Each thread - remembers the longest abort or pause it did because of cross-thread - contention_. This function prints it to ``stderr`` if the time lost - is greater than ``minimum_time`` seconds. The record is then - cleared, to make it ready for new events. This function returns - ``True`` if it printed a report, and ``False`` otherwise. +* ``__pypy__.thread.signals_enabled``: a context manager that runs its + block of code with signals enabled. By default, signals are only + enabled in the main thread; a non-main thread will not receive + signals (this is like CPython). Enabling signals in non-main + threads is useful for libraries where threads are hidden and the end + user is not expecting his code to run elsewhere than in the main + thread. +* ``pypystm.exclusive_atomic``: a context manager similar to + ``transaction.atomic`` but which complains if it is nested. -API of __pypy__.thread ----------------------- +* ``transaction.is_atomic()``: return True if called from an atomic + context. -The ``__pypy__.thread`` submodule is a built-in module of PyPy that -contains a few internal built-in functions used by the -``transactional_memory`` module, plus the following: +* ``pypystm.count()``: return a different positive integer every time + it is called. This works without generating conflicts. The + returned integers are only roughly in increasing order; this should + not be relied upon. -* ``__pypy__.thread.atomic``: a context manager to run a block in - fully atomic mode, without "releasing the GIL". (May be eventually - removed?) -* ``__pypy__.thread.signals_enabled``: a context manager that runs its - block with signals enabled. By default, signals are only enabled in - the main thread; a non-main thread will not receive signals (this is - like CPython). Enabling signals in non-main threads is useful for - libraries where threads are hidden and the end user is not expecting - his code to run elsewhere than in the main thread. - - -.. _contention: - -Conflicts ---------- +More details about conflicts +---------------------------- Based on Software Transactional Memory, the ``pypy-stm`` solution is prone to "conflicts". To repeat the basic idea, threads execute their code @@ -543,7 +512,7 @@ the transaction). If this occurs too often, parallelization fails. How much actual parallelization a multithreaded program can see is a bit -subtle. Basically, a program not using ``__pypy__.thread.atomic`` or +subtle. Basically, a program not using ``transaction.atomic`` or eliding locks, or doing so for very short amounts of time, will parallelize almost freely (as long as it's not some artificial example where, say, all threads try to increase the same global counter and do @@ -555,13 +524,14 @@ overview. Parallelization works as long as two principles are respected. The -first one is that the transactions must not *conflict* with each other. -The most obvious sources of conflicts are threads that all increment a -global shared counter, or that all store the result of their -computations into the same list --- or, more subtly, that all ``pop()`` -the work to do from the same list, because that is also a mutation of -the list. (It is expected that some STM-aware library will eventually -be designed to help with conflict problems, like a STM-aware queue.) +first one is that the transactions must not *conflict* with each +other. The most obvious sources of conflicts are threads that all +increment a global shared counter, or that all store the result of +their computations into the same list --- or, more subtly, that all +``pop()`` the work to do from the same list, because that is also a +mutation of the list. (You can work around it with +``transaction.stmdict``, but for that specific example, some STM-aware +queue should eventually be designed.) A conflict occurs as follows: when a transaction commits (i.e. finishes successfully) it may cause other transactions that are still in progress @@ -577,22 +547,23 @@ Another issue is that of avoiding long-running so-called "inevitable" transactions ("inevitable" is taken in the sense of "which cannot be avoided", i.e. transactions which cannot abort any more). Transactions -like that should only occur if you use ``__pypy__.thread.atomic``, -generally become of I/O in atomic blocks. They work, but the +like that should only occur if you use ``atomic``, +generally because of I/O in atomic blocks. They work, but the transaction is turned inevitable before the I/O is performed. For all the remaining execution time of the atomic block, they will impede parallel work. The best is to organize the code so that such operations -are done completely outside ``__pypy__.thread.atomic``. +are done completely outside ``atomic``. -(This is related to the fact that blocking I/O operations are +(This is not unrelated to the fact that blocking I/O operations are discouraged with Twisted, and if you really need them, you should do them on their own separate thread.) -In case of lock elision, we don't get long-running inevitable -transactions, but a different problem can occur: doing I/O cancels lock -elision, and the lock turns into a real lock, preventing other threads -from committing if they also need this lock. (More about it when lock -elision is implemented and tested.) +In case lock elision eventually replaces atomic sections, we wouldn't +get long-running inevitable transactions, but the same problem occurs +in a different way: doing I/O cancels lock elision, and the lock turns +into a real lock. This prevents other threads from committing if they +also need this lock. (More about it when lock elision is implemented +and tested.) @@ -602,56 +573,18 @@ XXX this section mostly empty for now -Low-level statistics --------------------- - -When a non-main thread finishes, you get low-level statistics printed to -stderr, looking like that:: - - thread 0x7f73377fe600: - outside transaction 42182 0.506 s - run current 85466 0.000 s - run committed 34262 3.178 s - run aborted write write 6982 0.083 s - run aborted write read 550 0.005 s - run aborted inevitable 388 0.010 s - run aborted other 0 0.000 s - wait free segment 0 0.000 s - wait write read 78 0.027 s - wait inevitable 887 0.490 s - wait other 0 0.000 s - sync commit soon 1 0.000 s - bookkeeping 51418 0.606 s - minor gc 162970 1.135 s - major gc 1 0.019 s - sync pause 59173 1.738 s - longest recordered marker 0.000826 s - "File "x.py", line 5, in f" - -On each line, the first number is a counter, and the second number gives -the associated time --- the amount of real time that the thread was in -this state. The sum of all the times should be equal to the total time -between the thread's start and the thread's end. The most important -points are "run committed", which gives the amount of useful work, and -"outside transaction", which should give the time spent e.g. in library -calls (right now it seems to be larger than that; to investigate). The -various "run aborted" and "wait" entries are time lost due to -conflicts_. Everything else is overhead of various forms. (Short-, -medium- and long-term future work involves reducing this overhead :-) - -The last two lines are special; they are an internal marker read by -``transactional_memory.print_abort_info()``. - - Reference to implementation details ----------------------------------- -The core of the implementation is in a separate C library called stmgc_, -in the c7_ subdirectory. Please see the `README.txt`_ for more -information. In particular, the notion of segment is discussed there. +The core of the implementation is in a separate C library called +stmgc_, in the c7_ subdirectory (current version of pypy-stm) and in +the c8_ subdirectory (bleeding edge version). Please see the +`README.txt`_ for more information. In particular, the notion of +segment is discussed there. .. _stmgc: https://bitbucket.org/pypy/stmgc/src/default/ .. _c7: https://bitbucket.org/pypy/stmgc/src/default/c7/ +.. _c8: https://bitbucket.org/pypy/stmgc/src/default/c8/ .. _`README.txt`: https://bitbucket.org/pypy/stmgc/raw/default/c7/README.txt PyPy itself adds on top of it the automatic placement of read__ and write__ From noreply at buildbot.pypy.org Mon Mar 16 16:27:52 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 16 Mar 2015 16:27:52 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: merge heads Message-ID: <20150316152752.29D001C0455@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76405:19a20e7c5d93 Date: 2015-03-16 16:22 +0100 http://bitbucket.org/pypy/pypy/changeset/19a20e7c5d93/ Log: merge heads diff --git a/pypy/module/pypystm/test_pypy_c/test_conflict.py b/pypy/module/pypystm/test_pypy_c/test_with_conflict.py rename from pypy/module/pypystm/test_pypy_c/test_conflict.py rename to pypy/module/pypystm/test_pypy_c/test_with_conflict.py From noreply at buildbot.pypy.org Mon Mar 16 16:27:53 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 16 Mar 2015 16:27:53 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Oops, bad move: reading the ll_raw_hashtable field of HASHTABLE_OBJ Message-ID: <20150316152753.53D151C0455@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76406:e4bdd177ed03 Date: 2015-03-16 16:26 +0100 http://bitbucket.org/pypy/pypy/changeset/e4bdd177ed03/ Log: Oops, bad move: reading the ll_raw_hashtable field of HASHTABLE_OBJ would be done with a stm_read(), even though it is immutable. Should fix test_hashtable_populate. diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py --- a/rpython/rlib/rstm.py +++ b/rpython/rlib/rstm.py @@ -244,6 +244,7 @@ _HASHTABLE_OBJ = lltype.GcStruct('HASHTABLE_OBJ', ('ll_raw_hashtable', _STM_HASHTABLE_P), + hints={'immutable': True}, rtti=True, adtmeths={'get': _ll_hashtable_get, 'set': _ll_hashtable_set, @@ -282,8 +283,7 @@ p = lltype.malloc(_STM_HASHTABLE_ENTRY) else: p = lltype.nullptr(_STM_HASHTABLE_ENTRY) - h = lltype.malloc(_HASHTABLE_OBJ) - h.ll_raw_hashtable = lltype.nullptr(_STM_HASHTABLE_P.TO) + h = lltype.malloc(_HASHTABLE_OBJ, zero=True) h.ll_raw_hashtable = llop.stm_hashtable_create(_STM_HASHTABLE_P, p) return h From noreply at buildbot.pypy.org Mon Mar 16 16:27:50 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 16 Mar 2015 16:27:50 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Rename the file, so that it is after "test_no_conflict" Message-ID: <20150316152750.E89D91C0455@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76404:ca29129dc9a1 Date: 2015-03-16 15:50 +0100 http://bitbucket.org/pypy/pypy/changeset/ca29129dc9a1/ Log: Rename the file, so that it is after "test_no_conflict" diff --git a/pypy/module/pypystm/test_pypy_c/test_conflict.py b/pypy/module/pypystm/test_pypy_c/test_with_conflict.py rename from pypy/module/pypystm/test_pypy_c/test_conflict.py rename to pypy/module/pypystm/test_pypy_c/test_with_conflict.py From noreply at buildbot.pypy.org Mon Mar 16 16:29:49 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 16 Mar 2015 16:29:49 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c8-hashtable: hg merge stmgc-c7 Message-ID: <20150316152949.188F51C0470@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c8-hashtable Changeset: r76407:8afb97abf085 Date: 2015-03-16 16:29 +0100 http://bitbucket.org/pypy/pypy/changeset/8afb97abf085/ Log: hg merge stmgc-c7 diff --git a/lib_pypy/pypy_test/test_transaction.py b/lib_pypy/pypy_test/test_transaction.py --- a/lib_pypy/pypy_test/test_transaction.py +++ b/lib_pypy/pypy_test/test_transaction.py @@ -192,6 +192,39 @@ assert d.setdefault(key2) is None assert d[key2] is None +def test_stmdict(): + d = transaction.stmdict() + d["abc"] = "def" + assert list(d.iterkeys()) == ["abc"] + +def test_stmset(): + d = transaction.stmset() + d.add("abc") + assert list(d) == ["abc"] + +def test_time_clock(): + assert isinstance(transaction.time(), float) + assert isinstance(transaction.clock(), float) + +def test_threadlocalproperty(): + class Foo(object): + x = transaction.threadlocalproperty() + y = transaction.threadlocalproperty(dict) + foo = Foo() + py.test.raises(AttributeError, "foo.x") + d = foo.y + assert d == {} + assert d is foo.y + foo.y['bar'] = 'baz' + foo.x = 42 + foo.y = 43 + assert foo.x == 42 + assert foo.y == 43 + del foo.x + del foo.y + py.test.raises(AttributeError, "foo.x") + assert foo.y == {} + def run_tests(): for name in sorted(globals().keys()): diff --git a/lib_pypy/transaction.py b/lib_pypy/transaction.py --- a/lib_pypy/transaction.py +++ b/lib_pypy/transaction.py @@ -37,24 +37,19 @@ signals_enabled = _SignalsEnabled() try: - from pypystm import hint_commit_soon + from pypystm import hint_commit_soon, getsegmentlimit + from pypystm import hashtable, stmset, stmdict + from pypystm import local, time, clock except ImportError: # Not a STM-enabled PyPy. def hint_commit_soon(): return None - -try: - from pypystm import getsegmentlimit -except ImportError: - # Not a STM-enabled PyPy. def getsegmentlimit(): return 1 - -try: - from pypystm import hashtable -except ImportError: - # Not a STM-enabled PyPy. hashtable = dict + stmset = set + stmdict = dict + from time import time, clock class stmidset(object): def __init__(self): @@ -299,9 +294,9 @@ class threadlocalproperty(object): - def __init__(self, *default): - self.tl_default = default - self.tl_name = intern(str(id(self))) + def __init__(self, default_factory=None): + self.tl_default_factory = default_factory + self.tl_name = intern('tlprop.%d' % id(self)) def tl_get(self, obj): try: @@ -313,7 +308,14 @@ def __get__(self, obj, cls=None): if obj is None: return self - return getattr(self.tl_get(obj), self.tl_name, *self.tl_default) + try: + return getattr(self.tl_get(obj), self.tl_name) + except AttributeError: + if self.tl_default_factory is None: + raise + result = self.tl_default_factory() + setattr(self.tl_get(obj), self.tl_name, result) + return result def __set__(self, obj, value): setattr(self.tl_get(obj), self.tl_name, value) diff --git a/pypy/module/pypystm/test_pypy_c/test_conflict.py b/pypy/module/pypystm/test_pypy_c/test_with_conflict.py rename from pypy/module/pypystm/test_pypy_c/test_conflict.py rename to pypy/module/pypystm/test_pypy_c/test_with_conflict.py diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py --- a/rpython/rlib/rstm.py +++ b/rpython/rlib/rstm.py @@ -248,6 +248,7 @@ _HASHTABLE_OBJ = lltype.GcStruct('HASHTABLE_OBJ', ('ll_raw_hashtable', _STM_HASHTABLE_P), + hints={'immutable': True}, rtti=True, adtmeths={'get': _ll_hashtable_get, 'set': _ll_hashtable_set, @@ -286,8 +287,7 @@ p = lltype.malloc(_STM_HASHTABLE_ENTRY) else: p = lltype.nullptr(_STM_HASHTABLE_ENTRY) - h = lltype.malloc(_HASHTABLE_OBJ) - h.ll_raw_hashtable = lltype.nullptr(_STM_HASHTABLE_P.TO) + h = lltype.malloc(_HASHTABLE_OBJ, zero=True) h.ll_raw_hashtable = llop.stm_hashtable_create(_STM_HASHTABLE_P, p) return h From noreply at buildbot.pypy.org Mon Mar 16 16:45:36 2015 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 16 Mar 2015 16:45:36 +0100 (CET) Subject: [pypy-commit] pypy dtrace-support: progress Message-ID: <20150316154536.7FA7C1C0455@cobra.cs.uni-duesseldorf.de> Author: fijal Branch: dtrace-support Changeset: r76408:57942acfb5de Date: 2015-03-16 17:45 +0200 http://bitbucket.org/pypy/pypy/changeset/57942acfb5de/ Log: progress diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py --- a/rpython/translator/c/funcgen.py +++ b/rpython/translator/c/funcgen.py @@ -784,8 +784,18 @@ def _op_debug(self, opname, arg): if isinstance(arg, Constant): - string_literal = c_string_constant(''.join(arg.value.chars)) - return "%s(%s);" % (opname, string_literal) + val = ''.join(arg.value.chars) + if self.db.translator.config.translation.dtrace: + if opname == 'PYPY_DEBUG_START': + name_for_op = 'START' + else: + name_for_op = 'END' + prefix = 'PYPY_PROBES_%s_%s();' % ( + val.replace('-', '_').upper(), name_for_op) + else: + prefix = '' + string_literal = c_string_constant(val) + return prefix + "%s(%s);" % (opname, string_literal) else: x = "%s(RPyString_AsCharP(%s));\n" % (opname, self.expr(arg)) x += "RPyString_FreeCache();" diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py --- a/rpython/translator/c/genc.py +++ b/rpython/translator/c/genc.py @@ -15,6 +15,7 @@ from rpython.translator.gensupp import uniquemodulename, NameManager from rpython.translator.tool.cbuild import ExternalCompilationInfo + _CYGWIN = sys.platform == 'cygwin' _CPYTHON_RE = py.std.re.compile('^Python 2.[567]') @@ -241,7 +242,8 @@ defines['PYPY_MAIN_FUNCTION'] = "pypy_main_startup" self.eci, cfile, extra, headers_to_precompile = \ gen_source(db, modulename, targetdir, - self.eci, defines=defines, split=self.split) + self.eci, defines=defines, split=self.split, + dtrace=self.config.translation.dtrace) self.c_source_filename = py.path.local(cfile) self.extrafiles = self.eventually_copy(extra) self.gen_makefile(targetdir, exe_name=exe_name, @@ -251,16 +253,20 @@ return cfile def _generate_dtrace_probe_file(self, debug_nodes): - name = self.targetdir.join('pypy.p') + name = self.targetdir.join('pypy.d') f = name.open('w') f.write('provider pypy_probes {\n') for debug_node in debug_nodes: debug_node = debug_node.replace('-', '_') - f.write(' probe %s__start(void);' % debug_node) - f.write(' probe %s__done(void);' % debug_node) - f.write('};') + f.write(' probe %s__start();\n' % debug_node) + f.write(' probe %s__end();\n' % debug_node) + f.write('};\n') f.close() - # XXX run dtrace + returncode, stdout, stderr = runsubprocess.run_subprocess( + 'dtrace', ['-o', str(self.targetdir.join('pypy_probes.h')), + '-h', '-s', str(name)]) + if returncode: + raise Exception("Dtrace exploded: %s" % stderr) def eventually_copy(self, cfiles): extrafiles = [] @@ -818,7 +824,7 @@ def gen_source(database, modulename, targetdir, - eci, defines={}, split=False): + eci, defines={}, split=False, dtrace=False): if isinstance(targetdir, str): targetdir = py.path.local(targetdir) @@ -839,6 +845,8 @@ eci.write_c_header(fi) print >> fi, '#include "src/g_prerequisite.h"' + if dtrace: + print >> fi, '#include "pypy_probes.h"' fi.write('#endif /* _PY_COMMON_HEADER_H*/\n') fi.close() From noreply at buildbot.pypy.org Mon Mar 16 17:12:24 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 16 Mar 2015 17:12:24 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: fix Message-ID: <20150316161224.7AA8D1C046C@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76409:88511e0d0557 Date: 2015-03-02 17:50 +0100 http://bitbucket.org/pypy/pypy/changeset/88511e0d0557/ Log: fix (grafted from 2157c7589dd68e40922774d01db5ee4766bd70ce) diff --git a/rpython/translator/c/src/debug_print.h b/rpython/translator/c/src/debug_print.h --- a/rpython/translator/c/src/debug_print.h +++ b/rpython/translator/c/src/debug_print.h @@ -34,7 +34,7 @@ #define __thread_if_stm /* nothing */ #endif -RPY_EXTERN __thread struct pypy_ExcData0 pypy_g_ExcData; +RPY_EXTERN __thread_if_stm struct pypy_ExcData0 pypy_g_ExcData; #define pypy_have_debug_prints pypy_g_ExcData.ed_have_debug_prints /* macros used by the generated code */ From noreply at buildbot.pypy.org Mon Mar 16 17:12:25 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 16 Mar 2015 17:12:25 +0100 (CET) Subject: [pypy-commit] pypy default: Use RLock instead of Lock. The goal is to ensure that only one thread Message-ID: <20150316161225.B4D111C046C@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76410:ca92ff804835 Date: 2015-03-16 17:11 +0100 http://bitbucket.org/pypy/pypy/changeset/ca92ff804835/ Log: Use RLock instead of Lock. The goal is to ensure that only one thread at a time uses tkinter, if the C library is "non-threaded". But we're getting rare cases of deadlocks even with only one thread, which is likely a rare path ending up in a recursive locking. diff --git a/lib_pypy/_tkinter/app.py b/lib_pypy/_tkinter/app.py --- a/lib_pypy/_tkinter/app.py +++ b/lib_pypy/_tkinter/app.py @@ -96,7 +96,7 @@ if not self.threaded: # TCL is not thread-safe, calls needs to be serialized. - self._tcl_lock = threading.Lock() + self._tcl_lock = threading.RLock() else: self._tcl_lock = _DummyLock() From noreply at buildbot.pypy.org Mon Mar 16 17:25:32 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 16 Mar 2015 17:25:32 +0100 (CET) Subject: [pypy-commit] pypy default: Fix test to also work in lldebug mode (it used to fail because Message-ID: <20150316162532.4FB221C0579@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76411:7926ef4fb172 Date: 2015-03-16 17:21 +0100 http://bitbucket.org/pypy/pypy/changeset/7926ef4fb172/ Log: Fix test to also work in lldebug mode (it used to fail because the C hack to allocate does not track the allocation) diff --git a/rpython/rtyper/lltypesystem/test/test_rffi.py b/rpython/rtyper/lltypesystem/test/test_rffi.py --- a/rpython/rtyper/lltypesystem/test/test_rffi.py +++ b/rpython/rtyper/lltypesystem/test/test_rffi.py @@ -102,24 +102,19 @@ #include #include - char *f(char* arg) + void f(char *target, char* arg) { - char *ret; - /* lltype.free uses OP_RAW_FREE, we must allocate - * with the matching function - */ - OP_RAW_MALLOC(strlen(arg) + 1, ret, char*) - strcpy(ret, arg); - return ret; + strcpy(target, arg); } """) eci = ExternalCompilationInfo(separate_module_sources=[c_source], - post_include_bits=['char *f(char*);']) - z = llexternal('f', [CCHARP], CCHARP, compilation_info=eci) + post_include_bits=['void f(char*,char*);']) + z = llexternal('f', [CCHARP, CCHARP], lltype.Void, compilation_info=eci) def f(): s = str2charp("xxx") - l_res = z(s) + l_res = lltype.malloc(CCHARP.TO, 10, flavor='raw') + z(l_res, s) res = charp2str(l_res) lltype.free(l_res, flavor='raw') free_charp(s) From noreply at buildbot.pypy.org Mon Mar 16 17:25:33 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 16 Mar 2015 17:25:33 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Fix test to also work in lldebug mode (it used to fail because Message-ID: <20150316162533.6DD5E1C0579@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76412:bb48a905aeb5 Date: 2015-03-16 17:21 +0100 http://bitbucket.org/pypy/pypy/changeset/bb48a905aeb5/ Log: Fix test to also work in lldebug mode (it used to fail because the C hack to allocate does not track the allocation) (grafted from 7926ef4fb1726f88a84ff26324c3a2cff023210a) diff --git a/rpython/rtyper/lltypesystem/test/test_rffi.py b/rpython/rtyper/lltypesystem/test/test_rffi.py --- a/rpython/rtyper/lltypesystem/test/test_rffi.py +++ b/rpython/rtyper/lltypesystem/test/test_rffi.py @@ -102,24 +102,19 @@ #include #include - char *f(char* arg) + void f(char *target, char* arg) { - char *ret; - /* lltype.free uses OP_RAW_FREE, we must allocate - * with the matching function - */ - OP_RAW_MALLOC(strlen(arg) + 1, ret, char*) - strcpy(ret, arg); - return ret; + strcpy(target, arg); } """) eci = ExternalCompilationInfo(separate_module_sources=[c_source], - post_include_bits=['char *f(char*);']) - z = llexternal('f', [CCHARP], CCHARP, compilation_info=eci) + post_include_bits=['void f(char*,char*);']) + z = llexternal('f', [CCHARP, CCHARP], lltype.Void, compilation_info=eci) def f(): s = str2charp("xxx") - l_res = z(s) + l_res = lltype.malloc(CCHARP.TO, 10, flavor='raw') + z(l_res, s) res = charp2str(l_res) lltype.free(l_res, flavor='raw') free_charp(s) From noreply at buildbot.pypy.org Mon Mar 16 17:43:00 2015 From: noreply at buildbot.pypy.org (mjacob) Date: Mon, 16 Mar 2015 17:43:00 +0100 (CET) Subject: [pypy-commit] pypy default: Fix typo. Message-ID: <20150316164300.77BB41C0455@cobra.cs.uni-duesseldorf.de> Author: Manuel Jacob Branch: Changeset: r76413:402d41538ce3 Date: 2015-03-16 17:42 +0100 http://bitbucket.org/pypy/pypy/changeset/402d41538ce3/ Log: Fix typo. diff --git a/rpython/doc/jit/optimizer.rst b/rpython/doc/jit/optimizer.rst --- a/rpython/doc/jit/optimizer.rst +++ b/rpython/doc/jit/optimizer.rst @@ -1,7 +1,7 @@ .. _trace_optimizer: -Trace Optimizier -================ +Trace Optimizer +=============== Traces of user programs are not directly translated into machine code. The optimizer module implements several different semantic preserving @@ -14,10 +14,10 @@ Before some optimizations are explained in more detail, it is essential to understand how traces look like. The optimizer comes with a test suit. It contains many trace -examples and you might want to take a look at it +examples and you might want to take a look at it (in `rpython/jit/metainterp/optimizeopt/test/*.py`). The allowed operations can be found in `rpython/jit/metainterp/resoperation.py`. -Here is an example of a trace:: +Here is an example of a trace:: [p0,i0,i1] label(p0, i0, i1) From noreply at buildbot.pypy.org Mon Mar 16 17:44:34 2015 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 16 Mar 2015 17:44:34 +0100 (CET) Subject: [pypy-commit] buildbot default: fix - seperate commands (arigato) Message-ID: <20150316164434.354481C009D@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r941:6dab224b6ce9 Date: 2015-03-16 18:46 +0200 http://bitbucket.org/pypy/buildbot/changeset/6dab224b6ce9/ Log: fix - seperate commands (arigato) diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -405,7 +405,7 @@ '/D', '-' + nDays, '/c', "cmd /c rmdir /q /s @path"] else: command = ['find', Interpolate(tmp_or_crazy + pytest), '-mtime', - '+' + nDays, '-exec', 'rm -r', '{}', ';'] + '+' + nDays, '-exec', 'rm', '-r', '{}', ';'] factory.addStep(SuccessAlways( description="cleanout old test files", command = command, @@ -504,7 +504,7 @@ '/D', '-' + nDays, '/c', "cmd /c rmdir /q /s @path"] else: command = ['find', Interpolate(tmp_or_crazy + pytest), '-mtime', - '+' + nDays, '-exec', 'rm -r', '{}', ';'] + '+' + nDays, '-exec', 'rm', '-r', '{}', ';'] self.addStep(SuccessAlways( description="cleanout old test files", command = command, From noreply at buildbot.pypy.org Mon Mar 16 18:02:25 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 16 Mar 2015 18:02:25 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: hg merge release-2.5.x Message-ID: <20150316170225.5EAF91C0455@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76414:ddbd63ba0da8 Date: 2015-03-16 18:01 +0100 http://bitbucket.org/pypy/pypy/changeset/ddbd63ba0da8/ Log: hg merge release-2.5.x diff too long, truncating to 2000 out of 41145 lines diff --git a/.gitignore b/.gitignore --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,10 @@ bin/pypy-c include/*.h +include/numpy/ lib_pypy/ctypes_config_cache/_[^_]*_*.py +libpypy-c.* +pypy-c pypy/_cache pypy/doc/*.html pypy/doc/config/*.html @@ -18,4 +21,5 @@ pypy/translator/c/src/dtoa.o pypy/translator/goal/pypy-c pypy/translator/goal/target*-c -release/ \ No newline at end of file +release/ +rpython/_cache/ diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -7,10 +7,7 @@ 9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm ab0dd631c22015ed88e583d9fdd4c43eebf0be21 pypy-2.1-beta1-arm 20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0 -20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0 -0000000000000000000000000000000000000000 release-2.3.0 394146e9bb673514c61f0150ab2013ccf78e8de7 release-2.3 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1 -32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1 -0000000000000000000000000000000000000000 release-2.2=3.1 +10f1b29a2bd21f837090286174a9ca030b8680b2 release-2.5.0 diff --git a/lib-python/2.7/CGIHTTPServer.py b/lib-python/2.7/CGIHTTPServer.py --- a/lib-python/2.7/CGIHTTPServer.py +++ b/lib-python/2.7/CGIHTTPServer.py @@ -106,16 +106,16 @@ def run_cgi(self): """Execute a CGI script.""" dir, rest = self.cgi_info - - i = rest.find('/') + path = dir + '/' + rest + i = path.find('/', len(dir)+1) while i >= 0: - nextdir = rest[:i] - nextrest = rest[i+1:] + nextdir = path[:i] + nextrest = path[i+1:] scriptdir = self.translate_path(nextdir) if os.path.isdir(scriptdir): dir, rest = nextdir, nextrest - i = rest.find('/') + i = path.find('/', len(dir)+1) else: break diff --git a/lib-python/2.7/Cookie.py b/lib-python/2.7/Cookie.py --- a/lib-python/2.7/Cookie.py +++ b/lib-python/2.7/Cookie.py @@ -56,7 +56,7 @@ >>> C = Cookie.SmartCookie() [Note: Long-time users of Cookie.py will remember using -Cookie.Cookie() to create an Cookie object. Although deprecated, it +Cookie.Cookie() to create a Cookie object. Although deprecated, it is still supported by the code. See the Backward Compatibility notes for more information.] @@ -426,6 +426,8 @@ "version" : "Version", } + _flags = {'secure', 'httponly'} + def __init__(self): # Set defaults self.key = self.value = self.coded_value = None @@ -529,9 +531,11 @@ _LegalCharsPatt = r"[\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=]" _CookiePattern = re.compile( r"(?x)" # This is a Verbose pattern + r"\s*" # Optional whitespace at start of cookie r"(?P" # Start of group 'key' ""+ _LegalCharsPatt +"+?" # Any word of at least one letter, nongreedy r")" # End of group 'key' + r"(" # Optional group: there may not be a value. r"\s*=\s*" # Equal Sign r"(?P" # Start of group 'val' r'"(?:[^\\"]|\\.)*"' # Any doublequoted string @@ -540,7 +544,9 @@ r"|" # or ""+ _LegalCharsPatt +"*" # Any word or empty string r")" # End of group 'val' - r"\s*;?" # Probably ending in a semi-colon + r")?" # End of optional value group + r"\s*" # Any number of spaces. + r"(\s+|;|$)" # Ending either at space, semicolon, or EOS. ) @@ -585,8 +591,12 @@ def __setitem__(self, key, value): """Dictionary style assignment.""" - rval, cval = self.value_encode(value) - self.__set(key, rval, cval) + if isinstance(value, Morsel): + # allow assignment of constructed Morsels (e.g. for pickling) + dict.__setitem__(self, key, value) + else: + rval, cval = self.value_encode(value) + self.__set(key, rval, cval) # end __setitem__ def output(self, attrs=None, header="Set-Cookie:", sep="\015\012"): @@ -641,7 +651,7 @@ while 0 <= i < n: # Start looking for a cookie - match = patt.search(str, i) + match = patt.match(str, i) if not match: break # No more cookies K,V = match.group("key"), match.group("val") @@ -656,8 +666,12 @@ M[ K[1:] ] = V elif K.lower() in Morsel._reserved: if M: - M[ K ] = _unquote(V) - else: + if V is None: + if K.lower() in Morsel._flags: + M[K] = True + else: + M[K] = _unquote(V) + elif V is not None: rval, cval = self.value_decode(V) self.__set(K, rval, cval) M = self[K] diff --git a/lib-python/2.7/SocketServer.py b/lib-python/2.7/SocketServer.py --- a/lib-python/2.7/SocketServer.py +++ b/lib-python/2.7/SocketServer.py @@ -416,8 +416,12 @@ self.socket = socket.socket(self.address_family, self.socket_type) if bind_and_activate: - self.server_bind() - self.server_activate() + try: + self.server_bind() + self.server_activate() + except: + self.server_close() + raise def server_bind(self): """Called by constructor to bind the socket. diff --git a/lib-python/2.7/_abcoll.py b/lib-python/2.7/_abcoll.py --- a/lib-python/2.7/_abcoll.py +++ b/lib-python/2.7/_abcoll.py @@ -143,7 +143,7 @@ methods except for __contains__, __iter__ and __len__. To override the comparisons (presumably for speed, as the - semantics are fixed), all you have to do is redefine __le__ and + semantics are fixed), redefine __le__ and __ge__, then the other operations will automatically follow suit. """ diff --git a/lib-python/2.7/argparse.py b/lib-python/2.7/argparse.py --- a/lib-python/2.7/argparse.py +++ b/lib-python/2.7/argparse.py @@ -1089,7 +1089,14 @@ # parse all the remaining options into the namespace # store any unrecognized options on the object, so that the top # level parser can decide what to do with them - namespace, arg_strings = parser.parse_known_args(arg_strings, namespace) + + # In case this subparser defines new defaults, we parse them + # in a new namespace object and then update the original + # namespace for the relevant parts. + subnamespace, arg_strings = parser.parse_known_args(arg_strings, None) + for key, value in vars(subnamespace).items(): + setattr(namespace, key, value) + if arg_strings: vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, []) getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings) diff --git a/lib-python/2.7/asynchat.py b/lib-python/2.7/asynchat.py --- a/lib-python/2.7/asynchat.py +++ b/lib-python/2.7/asynchat.py @@ -46,12 +46,17 @@ you - by calling your self.found_terminator() method. """ +import asyncore +import errno import socket -import asyncore from collections import deque from sys import py3kwarning from warnings import filterwarnings, catch_warnings +_BLOCKING_IO_ERRORS = (errno.EAGAIN, errno.EALREADY, errno.EINPROGRESS, + errno.EWOULDBLOCK) + + class async_chat (asyncore.dispatcher): """This is an abstract class. You must derive from this class, and add the two methods collect_incoming_data() and found_terminator()""" @@ -109,6 +114,8 @@ try: data = self.recv (self.ac_in_buffer_size) except socket.error, why: + if why.args[0] in _BLOCKING_IO_ERRORS: + return self.handle_error() return diff --git a/lib-python/2.7/bsddb/test/test_queue.py b/lib-python/2.7/bsddb/test/test_queue.py --- a/lib-python/2.7/bsddb/test/test_queue.py +++ b/lib-python/2.7/bsddb/test/test_queue.py @@ -10,6 +10,7 @@ #---------------------------------------------------------------------- + at unittest.skip("fails on Windows; see issue 22943") class SimpleQueueTestCase(unittest.TestCase): def setUp(self): self.filename = get_new_database_path() diff --git a/lib-python/2.7/cookielib.py b/lib-python/2.7/cookielib.py --- a/lib-python/2.7/cookielib.py +++ b/lib-python/2.7/cookielib.py @@ -1719,12 +1719,12 @@ def __repr__(self): r = [] for cookie in self: r.append(repr(cookie)) - return "<%s[%s]>" % (self.__class__, ", ".join(r)) + return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r)) def __str__(self): r = [] for cookie in self: r.append(str(cookie)) - return "<%s[%s]>" % (self.__class__, ", ".join(r)) + return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r)) # derives from IOError for backwards-compatibility with Python 2.4.0 diff --git a/lib-python/2.7/ctypes/test/test_pointers.py b/lib-python/2.7/ctypes/test/test_pointers.py --- a/lib-python/2.7/ctypes/test/test_pointers.py +++ b/lib-python/2.7/ctypes/test/test_pointers.py @@ -7,6 +7,8 @@ c_long, c_ulong, c_longlong, c_ulonglong, c_double, c_float] python_types = [int, int, int, int, int, long, int, long, long, long, float, float] +LargeNamedType = type('T' * 2 ** 25, (Structure,), {}) +large_string = 'T' * 2 ** 25 class PointersTestCase(unittest.TestCase): @@ -188,5 +190,11 @@ mth = WINFUNCTYPE(None)(42, "name", (), None) self.assertEqual(bool(mth), True) + def test_pointer_type_name(self): + self.assertTrue(POINTER(LargeNamedType)) + + def test_pointer_type_str_name(self): + self.assertTrue(POINTER(large_string)) + if __name__ == '__main__': unittest.main() diff --git a/lib-python/2.7/ctypes/test/test_python_api.py b/lib-python/2.7/ctypes/test/test_python_api.py --- a/lib-python/2.7/ctypes/test/test_python_api.py +++ b/lib-python/2.7/ctypes/test/test_python_api.py @@ -46,8 +46,8 @@ # This test is unreliable, because it is possible that code in # unittest changes the refcount of the '42' integer. So, it # is disabled by default. - @requires("refcount") def test_PyInt_Long(self): + requires("refcount") ref42 = grc(42) pythonapi.PyInt_FromLong.restype = py_object self.assertEqual(pythonapi.PyInt_FromLong(42), 42) diff --git a/lib-python/2.7/ctypes/test/test_win32.py b/lib-python/2.7/ctypes/test/test_win32.py --- a/lib-python/2.7/ctypes/test/test_win32.py +++ b/lib-python/2.7/ctypes/test/test_win32.py @@ -38,8 +38,11 @@ @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') class FunctionCallTestCase(unittest.TestCase): - @requires("SEH") + @unittest.skipUnless('MSC' in sys.version, "SEH only supported by MSC") + @unittest.skipIf(sys.executable.endswith('_d.exe'), + "SEH not enabled in debug builds") def test_SEH(self): + requires("SEH") # Call functions with invalid arguments, and make sure # that access violations are trapped and raise an # exception. @@ -87,9 +90,29 @@ dll = CDLL(_ctypes_test.__file__) - pt = POINT(10, 10) - rect = RECT(0, 0, 20, 20) - self.assertEqual(1, dll.PointInRect(byref(rect), pt)) + pt = POINT(15, 25) + left = c_long.in_dll(dll, 'left') + top = c_long.in_dll(dll, 'top') + right = c_long.in_dll(dll, 'right') + bottom = c_long.in_dll(dll, 'bottom') + rect = RECT(left, top, right, bottom) + PointInRect = dll.PointInRect + PointInRect.argtypes = [POINTER(RECT), POINT] + self.assertEqual(1, PointInRect(byref(rect), pt)) + + ReturnRect = dll.ReturnRect + ReturnRect.argtypes = [c_int, RECT, POINTER(RECT), POINT, RECT, + POINTER(RECT), POINT, RECT] + ReturnRect.restype = RECT + for i in range(4): + ret = ReturnRect(i, rect, pointer(rect), pt, rect, + byref(rect), pt, rect) + # the c function will check and modify ret if something is + # passed in improperly + self.assertEqual(ret.left, left.value) + self.assertEqual(ret.right, right.value) + self.assertEqual(ret.top, top.value) + self.assertEqual(ret.bottom, bottom.value) if __name__ == '__main__': unittest.main() diff --git a/lib-python/2.7/decimal.py b/lib-python/2.7/decimal.py --- a/lib-python/2.7/decimal.py +++ b/lib-python/2.7/decimal.py @@ -136,7 +136,6 @@ __version__ = '1.70' # Highest version of the spec this complies with -import copy as _copy import math as _math import numbers as _numbers @@ -3665,6 +3664,8 @@ if self._is_special: sign = _format_sign(self._sign, spec) body = str(self.copy_abs()) + if spec['type'] == '%': + body += '%' return _format_align(sign, body, spec) # a type of None defaults to 'g' or 'G', depending on context @@ -6033,7 +6034,10 @@ format_dict['decimal_point'] = '.' # record whether return type should be str or unicode - format_dict['unicode'] = isinstance(format_spec, unicode) + try: + format_dict['unicode'] = isinstance(format_spec, unicode) + except NameError: + format_dict['unicode'] = False return format_dict diff --git a/lib-python/2.7/distutils/__init__.py b/lib-python/2.7/distutils/__init__.py --- a/lib-python/2.7/distutils/__init__.py +++ b/lib-python/2.7/distutils/__init__.py @@ -15,5 +15,5 @@ # Updated automatically by the Python release process. # #--start constants-- -__version__ = "2.7.8" +__version__ = "2.7.9" #--end constants-- diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py --- a/lib-python/2.7/distutils/command/build_ext.py +++ b/lib-python/2.7/distutils/command/build_ext.py @@ -245,7 +245,7 @@ # Python's library directory must be appended to library_dirs # See Issues: #1600860, #4366 if (sysconfig.get_config_var('Py_ENABLE_SHARED')): - if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")): + if not sysconfig.python_build: # building third party extensions self.library_dirs.append(sysconfig.get_config_var('LIBDIR')) else: diff --git a/lib-python/2.7/distutils/command/upload.py b/lib-python/2.7/distutils/command/upload.py --- a/lib-python/2.7/distutils/command/upload.py +++ b/lib-python/2.7/distutils/command/upload.py @@ -136,8 +136,8 @@ # Build up the MIME payload for the POST data boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254' - sep_boundary = '\n--' + boundary - end_boundary = sep_boundary + '--' + sep_boundary = '\r\n--' + boundary + end_boundary = sep_boundary + '--\r\n' body = StringIO.StringIO() for key, value in data.items(): # handle multiple entries for the same name @@ -151,14 +151,13 @@ fn = "" body.write(sep_boundary) - body.write('\nContent-Disposition: form-data; name="%s"'%key) + body.write('\r\nContent-Disposition: form-data; name="%s"' % key) body.write(fn) - body.write("\n\n") + body.write("\r\n\r\n") body.write(value) if value and value[-1] == '\r': body.write('\n') # write an extra newline (lurve Macs) body.write(end_boundary) - body.write("\n") body = body.getvalue() self.announce("Submitting %s to %s" % (filename, self.repository), log.INFO) diff --git a/lib-python/2.7/distutils/file_util.py b/lib-python/2.7/distutils/file_util.py --- a/lib-python/2.7/distutils/file_util.py +++ b/lib-python/2.7/distutils/file_util.py @@ -85,7 +85,8 @@ (os.symlink) instead of copying: set it to "hard" or "sym"; if it is None (the default), files are copied. Don't set 'link' on systems that don't support it: 'copy_file()' doesn't check if hard or symbolic - linking is available. + linking is available. If hardlink fails, falls back to + _copy_file_contents(). Under Mac OS, uses the native file copy function in macostools; on other systems, uses '_copy_file_contents()' to copy file contents. @@ -137,24 +138,31 @@ # (Unix only, of course, but that's the caller's responsibility) if link == 'hard': if not (os.path.exists(dst) and os.path.samefile(src, dst)): - os.link(src, dst) + try: + os.link(src, dst) + return (dst, 1) + except OSError: + # If hard linking fails, fall back on copying file + # (some special filesystems don't support hard linking + # even under Unix, see issue #8876). + pass elif link == 'sym': if not (os.path.exists(dst) and os.path.samefile(src, dst)): os.symlink(src, dst) + return (dst, 1) # Otherwise (non-Mac, not linking), copy the file contents and # (optionally) copy the times and mode. - else: - _copy_file_contents(src, dst) - if preserve_mode or preserve_times: - st = os.stat(src) + _copy_file_contents(src, dst) + if preserve_mode or preserve_times: + st = os.stat(src) - # According to David Ascher , utime() should be done - # before chmod() (at least under NT). - if preserve_times: - os.utime(dst, (st[ST_ATIME], st[ST_MTIME])) - if preserve_mode: - os.chmod(dst, S_IMODE(st[ST_MODE])) + # According to David Ascher , utime() should be done + # before chmod() (at least under NT). + if preserve_times: + os.utime(dst, (st[ST_ATIME], st[ST_MTIME])) + if preserve_mode: + os.chmod(dst, S_IMODE(st[ST_MODE])) return (dst, 1) diff --git a/lib-python/2.7/distutils/sysconfig_cpython.py b/lib-python/2.7/distutils/sysconfig_cpython.py --- a/lib-python/2.7/distutils/sysconfig_cpython.py +++ b/lib-python/2.7/distutils/sysconfig_cpython.py @@ -165,7 +165,8 @@ # version and build tools may not support the same set # of CPU architectures for universal builds. global _config_vars - if not _config_vars.get('CUSTOMIZED_OSX_COMPILER', ''): + # Use get_config_var() to ensure _config_vars is initialized. + if not get_config_var('CUSTOMIZED_OSX_COMPILER'): import _osx_support _osx_support.customize_compiler(_config_vars) _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True' diff --git a/lib-python/2.7/distutils/tests/test_bdist_rpm.py b/lib-python/2.7/distutils/tests/test_bdist_rpm.py --- a/lib-python/2.7/distutils/tests/test_bdist_rpm.py +++ b/lib-python/2.7/distutils/tests/test_bdist_rpm.py @@ -25,6 +25,7 @@ """ class BuildRpmTestCase(support.TempdirManager, + support.EnvironGuard, support.LoggingSilencer, unittest.TestCase): @@ -50,6 +51,7 @@ def test_quiet(self): # let's create a package tmp_dir = self.mkdtemp() + os.environ['HOME'] = tmp_dir # to confine dir '.rpmdb' creation pkg_dir = os.path.join(tmp_dir, 'foo') os.mkdir(pkg_dir) self.write_file((pkg_dir, 'setup.py'), SETUP_PY) @@ -92,6 +94,7 @@ def test_no_optimize_flag(self): # let's create a package that brakes bdist_rpm tmp_dir = self.mkdtemp() + os.environ['HOME'] = tmp_dir # to confine dir '.rpmdb' creation pkg_dir = os.path.join(tmp_dir, 'foo') os.mkdir(pkg_dir) self.write_file((pkg_dir, 'setup.py'), SETUP_PY) diff --git a/lib-python/2.7/distutils/tests/test_dist.py b/lib-python/2.7/distutils/tests/test_dist.py --- a/lib-python/2.7/distutils/tests/test_dist.py +++ b/lib-python/2.7/distutils/tests/test_dist.py @@ -11,7 +11,7 @@ from distutils.dist import Distribution, fix_help_options from distutils.cmd import Command import distutils.dist -from test.test_support import TESTFN, captured_stdout, run_unittest +from test.test_support import TESTFN, captured_stdout, run_unittest, unlink from distutils.tests import support @@ -64,6 +64,7 @@ with open(TESTFN, "w") as f: f.write("[global]\n") f.write("command_packages = foo.bar, splat") + self.addCleanup(unlink, TESTFN) files = [TESTFN] sys.argv.append("build") diff --git a/lib-python/2.7/distutils/tests/test_file_util.py b/lib-python/2.7/distutils/tests/test_file_util.py --- a/lib-python/2.7/distutils/tests/test_file_util.py +++ b/lib-python/2.7/distutils/tests/test_file_util.py @@ -8,6 +8,11 @@ from distutils.tests import support from test.test_support import run_unittest + +requires_os_link = unittest.skipUnless(hasattr(os, "link"), + "test requires os.link()") + + class FileUtilTestCase(support.TempdirManager, unittest.TestCase): def _log(self, msg, *args): @@ -74,6 +79,44 @@ copy_file(foo, dst_dir) self.assertTrue(os.path.exists(os.path.join(dst_dir, 'foo'))) + @requires_os_link + def test_copy_file_hard_link(self): + with open(self.source, 'w') as f: + f.write('some content') + st = os.stat(self.source) + copy_file(self.source, self.target, link='hard') + st2 = os.stat(self.source) + st3 = os.stat(self.target) + self.assertTrue(os.path.samestat(st, st2), (st, st2)) + self.assertTrue(os.path.samestat(st2, st3), (st2, st3)) + with open(self.source, 'r') as f: + self.assertEqual(f.read(), 'some content') + + @requires_os_link + def test_copy_file_hard_link_failure(self): + # If hard linking fails, copy_file() falls back on copying file + # (some special filesystems don't support hard linking even under + # Unix, see issue #8876). + with open(self.source, 'w') as f: + f.write('some content') + st = os.stat(self.source) + def _os_link(*args): + raise OSError(0, "linking unsupported") + old_link = os.link + os.link = _os_link + try: + copy_file(self.source, self.target, link='hard') + finally: + os.link = old_link + st2 = os.stat(self.source) + st3 = os.stat(self.target) + self.assertTrue(os.path.samestat(st, st2), (st, st2)) + self.assertFalse(os.path.samestat(st2, st3), (st2, st3)) + for fn in (self.source, self.target): + with open(fn, 'r') as f: + self.assertEqual(f.read(), 'some content') + + def test_suite(): return unittest.makeSuite(FileUtilTestCase) diff --git a/lib-python/2.7/distutils/tests/test_sysconfig.py b/lib-python/2.7/distutils/tests/test_sysconfig.py --- a/lib-python/2.7/distutils/tests/test_sysconfig.py +++ b/lib-python/2.7/distutils/tests/test_sysconfig.py @@ -3,6 +3,9 @@ import test import unittest import shutil +import subprocess +import sys +import textwrap from distutils import sysconfig from distutils.tests import support @@ -99,6 +102,24 @@ self.assertEqual(global_sysconfig.get_config_var('LDSHARED'), sysconfig.get_config_var('LDSHARED')) self.assertEqual(global_sysconfig.get_config_var('CC'), sysconfig.get_config_var('CC')) + def test_customize_compiler_before_get_config_vars(self): + # Issue #21923: test that a Distribution compiler + # instance can be called without an explicit call to + # get_config_vars(). + with open(TESTFN, 'w') as f: + f.writelines(textwrap.dedent('''\ + from distutils.core import Distribution + config = Distribution().get_command_obj('config') + # try_compile may pass or it may fail if no compiler + # is found but it should not raise an exception. + rc = config.try_compile('int x;') + ''')) + p = subprocess.Popen([str(sys.executable), TESTFN], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True) + outs, errs = p.communicate() + self.assertEqual(0, p.returncode, "Subprocess failed: " + outs) def test_suite(): diff --git a/lib-python/2.7/distutils/tests/test_upload.py b/lib-python/2.7/distutils/tests/test_upload.py --- a/lib-python/2.7/distutils/tests/test_upload.py +++ b/lib-python/2.7/distutils/tests/test_upload.py @@ -119,7 +119,7 @@ # what did we send ? self.assertIn('dédé', self.last_open.req.data) headers = dict(self.last_open.req.headers) - self.assertEqual(headers['Content-length'], '2085') + self.assertEqual(headers['Content-length'], '2159') self.assertTrue(headers['Content-type'].startswith('multipart/form-data')) self.assertEqual(self.last_open.req.get_method(), 'POST') self.assertEqual(self.last_open.req.get_full_url(), diff --git a/lib-python/2.7/doctest.py b/lib-python/2.7/doctest.py --- a/lib-python/2.7/doctest.py +++ b/lib-python/2.7/doctest.py @@ -216,7 +216,7 @@ # get_data() opens files as 'rb', so one must do the equivalent # conversion as universal newlines would do. return file_contents.replace(os.linesep, '\n'), filename - with open(filename) as f: + with open(filename, 'U') as f: return f.read(), filename # Use sys.stdout encoding for ouput. diff --git a/lib-python/2.7/email/feedparser.py b/lib-python/2.7/email/feedparser.py --- a/lib-python/2.7/email/feedparser.py +++ b/lib-python/2.7/email/feedparser.py @@ -49,8 +49,8 @@ simple abstraction -- it parses until EOF closes the current message. """ def __init__(self): - # The last partial line pushed into this object. - self._partial = '' + # Chunks of the last partial line pushed into this object. + self._partial = [] # The list of full, pushed lines, in reverse order self._lines = [] # The stack of false-EOF checking predicates. @@ -66,8 +66,8 @@ def close(self): # Don't forget any trailing partial line. - self._lines.append(self._partial) - self._partial = '' + self.pushlines(''.join(self._partial).splitlines(True)) + self._partial = [] self._closed = True def readline(self): @@ -95,8 +95,29 @@ def push(self, data): """Push some new data into this object.""" - # Handle any previous leftovers - data, self._partial = self._partial + data, '' + # Crack into lines, but preserve the linesep characters on the end of each + parts = data.splitlines(True) + + if not parts or not parts[0].endswith(('\n', '\r')): + # No new complete lines, so just accumulate partials + self._partial += parts + return + + if self._partial: + # If there are previous leftovers, complete them now + self._partial.append(parts[0]) + parts[0:1] = ''.join(self._partial).splitlines(True) + del self._partial[:] + + # If the last element of the list does not end in a newline, then treat + # it as a partial line. We only check for '\n' here because a line + # ending with '\r' might be a line that was split in the middle of a + # '\r\n' sequence (see bugs 1555570 and 1721862). + if not parts[-1].endswith('\n'): + self._partial = [parts.pop()] + self.pushlines(parts) + + def pushlines(self, lines): # Crack into lines, but preserve the newlines on the end of each parts = NLCRE_crack.split(data) # The *ahem* interesting behaviour of re.split when supplied grouping diff --git a/lib-python/2.7/email/mime/nonmultipart.py b/lib-python/2.7/email/mime/nonmultipart.py --- a/lib-python/2.7/email/mime/nonmultipart.py +++ b/lib-python/2.7/email/mime/nonmultipart.py @@ -12,7 +12,7 @@ class MIMENonMultipart(MIMEBase): - """Base class for MIME multipart/* type messages.""" + """Base class for MIME non-multipart type messages.""" def attach(self, payload): # The public API prohibits attaching multiple subparts to MIMEBase diff --git a/lib-python/2.7/email/test/test_email.py b/lib-python/2.7/email/test/test_email.py --- a/lib-python/2.7/email/test/test_email.py +++ b/lib-python/2.7/email/test/test_email.py @@ -11,6 +11,7 @@ import warnings import textwrap from cStringIO import StringIO +from random import choice import email @@ -2578,16 +2579,64 @@ bsf.push(il) nt += n n1 = 0 - while True: - ol = bsf.readline() - if ol == NeedMoreData: - break + for ol in iter(bsf.readline, NeedMoreData): om.append(ol) n1 += 1 self.assertEqual(n, n1) self.assertEqual(len(om), nt) self.assertEqual(''.join([il for il, n in imt]), ''.join(om)) + def test_push_random(self): + from email.feedparser import BufferedSubFile, NeedMoreData + + n = 10000 + chunksize = 5 + chars = 'abcd \t\r\n' + + s = ''.join(choice(chars) for i in range(n)) + '\n' + target = s.splitlines(True) + + bsf = BufferedSubFile() + lines = [] + for i in range(0, len(s), chunksize): + chunk = s[i:i+chunksize] + bsf.push(chunk) + lines.extend(iter(bsf.readline, NeedMoreData)) + self.assertEqual(lines, target) + + +class TestFeedParsers(TestEmailBase): + + def parse(self, chunks): + from email.feedparser import FeedParser + feedparser = FeedParser() + for chunk in chunks: + feedparser.feed(chunk) + return feedparser.close() + + def test_newlines(self): + m = self.parse(['a:\nb:\rc:\r\nd:\n']) + self.assertEqual(m.keys(), ['a', 'b', 'c', 'd']) + m = self.parse(['a:\nb:\rc:\r\nd:']) + self.assertEqual(m.keys(), ['a', 'b', 'c', 'd']) + m = self.parse(['a:\rb', 'c:\n']) + self.assertEqual(m.keys(), ['a', 'bc']) + m = self.parse(['a:\r', 'b:\n']) + self.assertEqual(m.keys(), ['a', 'b']) + m = self.parse(['a:\r', '\nb:\n']) + self.assertEqual(m.keys(), ['a', 'b']) + + def test_long_lines(self): + # Expected peak memory use on 32-bit platform: 4*N*M bytes. + M, N = 1000, 20000 + m = self.parse(['a:b\n\n'] + ['x'*M] * N) + self.assertEqual(m.items(), [('a', 'b')]) + self.assertEqual(m.get_payload(), 'x'*M*N) + m = self.parse(['a:b\r\r'] + ['x'*M] * N) + self.assertEqual(m.items(), [('a', 'b')]) + self.assertEqual(m.get_payload(), 'x'*M*N) + m = self.parse(['a:\r', 'b: '] + ['x'*M] * N) + self.assertEqual(m.items(), [('a', ''), ('b', 'x'*M*N)]) class TestParsers(TestEmailBase): @@ -3180,7 +3229,6 @@ self.assertEqual(res, '=?iso-8859-2?q?abc?=') self.assertIsInstance(res, str) - # Test RFC 2231 header parameters (en/de)coding class TestRFC2231(TestEmailBase): def test_get_param(self): diff --git a/lib-python/2.7/ensurepip/__init__.py b/lib-python/2.7/ensurepip/__init__.py new file mode 100644 --- /dev/null +++ b/lib-python/2.7/ensurepip/__init__.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python2 +from __future__ import print_function + +import os +import os.path +import pkgutil +import shutil +import sys +import tempfile + + +__all__ = ["version", "bootstrap"] + + +_SETUPTOOLS_VERSION = "7.0" + +_PIP_VERSION = "1.5.6" + +# pip currently requires ssl support, so we try to provide a nicer +# error message when that is missing (http://bugs.python.org/issue19744) +_MISSING_SSL_MESSAGE = ("pip {} requires SSL/TLS".format(_PIP_VERSION)) +try: + import ssl +except ImportError: + ssl = None + + def _require_ssl_for_pip(): + raise RuntimeError(_MISSING_SSL_MESSAGE) +else: + def _require_ssl_for_pip(): + pass + +_PROJECTS = [ + ("setuptools", _SETUPTOOLS_VERSION), + ("pip", _PIP_VERSION), +] + + +def _run_pip(args, additional_paths=None): + # Add our bundled software to the sys.path so we can import it + if additional_paths is not None: + sys.path = additional_paths + sys.path + + # Install the bundled software + import pip + pip.main(args) + + +def version(): + """ + Returns a string specifying the bundled version of pip. + """ + return _PIP_VERSION + + +def _disable_pip_configuration_settings(): + # We deliberately ignore all pip environment variables + # when invoking pip + # See http://bugs.python.org/issue19734 for details + keys_to_remove = [k for k in os.environ if k.startswith("PIP_")] + for k in keys_to_remove: + del os.environ[k] + # We also ignore the settings in the default pip configuration file + # See http://bugs.python.org/issue20053 for details + os.environ['PIP_CONFIG_FILE'] = os.devnull + + +def bootstrap(root=None, upgrade=False, user=False, + altinstall=False, default_pip=True, + verbosity=0): + """ + Bootstrap pip into the current Python installation (or the given root + directory). + + Note that calling this function will alter both sys.path and os.environ. + """ + if altinstall and default_pip: + raise ValueError("Cannot use altinstall and default_pip together") + + _require_ssl_for_pip() + _disable_pip_configuration_settings() + + # By default, installing pip and setuptools installs all of the + # following scripts (X.Y == running Python version): + # + # pip, pipX, pipX.Y, easy_install, easy_install-X.Y + # + # pip 1.5+ allows ensurepip to request that some of those be left out + if altinstall: + # omit pip, pipX and easy_install + os.environ["ENSUREPIP_OPTIONS"] = "altinstall" + elif not default_pip: + # omit pip and easy_install + os.environ["ENSUREPIP_OPTIONS"] = "install" + + tmpdir = tempfile.mkdtemp() + try: + # Put our bundled wheels into a temporary directory and construct the + # additional paths that need added to sys.path + additional_paths = [] + for project, version in _PROJECTS: + wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version) + whl = pkgutil.get_data( + "ensurepip", + "_bundled/{}".format(wheel_name), + ) + with open(os.path.join(tmpdir, wheel_name), "wb") as fp: + fp.write(whl) + + additional_paths.append(os.path.join(tmpdir, wheel_name)) + + # Construct the arguments to be passed to the pip command + args = ["install", "--no-index", "--find-links", tmpdir] + if root: + args += ["--root", root] + if upgrade: + args += ["--upgrade"] + if user: + args += ["--user"] + if verbosity: + args += ["-" + "v" * verbosity] + + _run_pip(args + [p[0] for p in _PROJECTS], additional_paths) + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + +def _uninstall_helper(verbosity=0): + """Helper to support a clean default uninstall process on Windows + + Note that calling this function may alter os.environ. + """ + # Nothing to do if pip was never installed, or has been removed + try: + import pip + except ImportError: + return + + # If the pip version doesn't match the bundled one, leave it alone + if pip.__version__ != _PIP_VERSION: + msg = ("ensurepip will only uninstall a matching version " + "({!r} installed, {!r} bundled)") + print(msg.format(pip.__version__, _PIP_VERSION), file=sys.stderr) + return + + _require_ssl_for_pip() + _disable_pip_configuration_settings() + + # Construct the arguments to be passed to the pip command + args = ["uninstall", "-y"] + if verbosity: + args += ["-" + "v" * verbosity] + + _run_pip(args + [p[0] for p in reversed(_PROJECTS)]) + + +def _main(argv=None): + if ssl is None: + print("Ignoring ensurepip failure: {}".format(_MISSING_SSL_MESSAGE), + file=sys.stderr) + return + + import argparse + parser = argparse.ArgumentParser(prog="python -m ensurepip") + parser.add_argument( + "--version", + action="version", + version="pip {}".format(version()), + help="Show the version of pip that is bundled with this Python.", + ) + parser.add_argument( + "-v", "--verbose", + action="count", + default=0, + dest="verbosity", + help=("Give more output. Option is additive, and can be used up to 3 " + "times."), + ) + parser.add_argument( + "-U", "--upgrade", + action="store_true", + default=False, + help="Upgrade pip and dependencies, even if already installed.", + ) + parser.add_argument( + "--user", + action="store_true", + default=False, + help="Install using the user scheme.", + ) + parser.add_argument( + "--root", + default=None, + help="Install everything relative to this alternate root directory.", + ) + parser.add_argument( + "--altinstall", + action="store_true", + default=False, + help=("Make an alternate install, installing only the X.Y versioned" + "scripts (Default: pipX, pipX.Y, easy_install-X.Y)"), + ) + parser.add_argument( + "--default-pip", + action="store_true", + default=True, + dest="default_pip", + help=argparse.SUPPRESS, + ) + parser.add_argument( + "--no-default-pip", + action="store_false", + dest="default_pip", + help=("Make a non default install, installing only the X and X.Y " + "versioned scripts."), + ) + + args = parser.parse_args(argv) + + bootstrap( + root=args.root, + upgrade=args.upgrade, + user=args.user, + verbosity=args.verbosity, + altinstall=args.altinstall, + default_pip=args.default_pip, + ) diff --git a/lib-python/2.7/ensurepip/__main__.py b/lib-python/2.7/ensurepip/__main__.py new file mode 100644 --- /dev/null +++ b/lib-python/2.7/ensurepip/__main__.py @@ -0,0 +1,4 @@ +import ensurepip + +if __name__ == "__main__": + ensurepip._main() diff --git a/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..097ab43430d4c1302b0be353a8c16407c370693b GIT binary patch [cut] diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..fa1d1054da1dab98f8906555d31a9fda271b3a85 GIT binary patch [cut] diff --git a/lib-python/2.7/ensurepip/_uninstall.py b/lib-python/2.7/ensurepip/_uninstall.py new file mode 100644 --- /dev/null +++ b/lib-python/2.7/ensurepip/_uninstall.py @@ -0,0 +1,30 @@ +"""Basic pip uninstallation support, helper for the Windows uninstaller""" + +import argparse +import ensurepip + + +def _main(argv=None): + parser = argparse.ArgumentParser(prog="python -m ensurepip._uninstall") + parser.add_argument( + "--version", + action="version", + version="pip {}".format(ensurepip.version()), + help="Show the version of pip this will attempt to uninstall.", + ) + parser.add_argument( + "-v", "--verbose", + action="count", + default=0, + dest="verbosity", + help=("Give more output. Option is additive, and can be used up to 3 " + "times."), + ) + + args = parser.parse_args(argv) + + ensurepip._uninstall_helper(verbosity=args.verbosity) + + +if __name__ == "__main__": + _main() diff --git a/lib-python/2.7/glob.py b/lib-python/2.7/glob.py --- a/lib-python/2.7/glob.py +++ b/lib-python/2.7/glob.py @@ -35,11 +35,16 @@ patterns. """ + dirname, basename = os.path.split(pathname) if not has_magic(pathname): - if os.path.lexists(pathname): - yield pathname + if basename: + if os.path.lexists(pathname): + yield pathname + else: + # Patterns ending with a slash should match only directories + if os.path.isdir(dirname): + yield pathname return - dirname, basename = os.path.split(pathname) if not dirname: for name in glob1(os.curdir, basename): yield name diff --git a/lib-python/2.7/gzip.py b/lib-python/2.7/gzip.py --- a/lib-python/2.7/gzip.py +++ b/lib-python/2.7/gzip.py @@ -164,9 +164,16 @@ def _write_gzip_header(self): self.fileobj.write('\037\213') # magic header self.fileobj.write('\010') # compression method - fname = os.path.basename(self.name) - if fname.endswith(".gz"): - fname = fname[:-3] + try: + # RFC 1952 requires the FNAME field to be Latin-1. Do not + # include filenames that cannot be represented that way. + fname = os.path.basename(self.name) + if not isinstance(fname, str): + fname = fname.encode('latin-1') + if fname.endswith('.gz'): + fname = fname[:-3] + except UnicodeEncodeError: + fname = '' flags = 0 if fname: flags = FNAME diff --git a/lib-python/2.7/hashlib.py b/lib-python/2.7/hashlib.py --- a/lib-python/2.7/hashlib.py +++ b/lib-python/2.7/hashlib.py @@ -15,8 +15,9 @@ md5(), sha1(), sha224(), sha256(), sha384(), and sha512() -More algorithms may be available on your platform but the above are -guaranteed to exist. +More algorithms may be available on your platform but the above are guaranteed +to exist. See the algorithms_guaranteed and algorithms_available attributes +to find out what algorithm names can be passed to new(). NOTE: If you want the adler32 or crc32 hash functions they are available in the zlib module. @@ -58,9 +59,14 @@ # always available algorithm is added. __always_supported = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512') +algorithms_guaranteed = set(__always_supported) +algorithms_available = set(__always_supported) + algorithms = __always_supported -__all__ = __always_supported + ('new', 'algorithms', 'pbkdf2_hmac') +__all__ = __always_supported + ('new', 'algorithms_guaranteed', + 'algorithms_available', 'algorithms', + 'pbkdf2_hmac') def __get_builtin_constructor(name): @@ -128,6 +134,8 @@ import _hashlib new = __hash_new __get_hash = __get_openssl_constructor + algorithms_available = algorithms_available.union( + _hashlib.openssl_md_meth_names) except ImportError: new = __py_new __get_hash = __get_builtin_constructor diff --git a/lib-python/2.7/httplib.py b/lib-python/2.7/httplib.py --- a/lib-python/2.7/httplib.py +++ b/lib-python/2.7/httplib.py @@ -215,6 +215,10 @@ # maximal line length when calling readline(). _MAXLINE = 65536 +# maximum amount of headers accepted +_MAXHEADERS = 100 + + class HTTPMessage(mimetools.Message): def addheader(self, key, value): @@ -271,6 +275,8 @@ elif self.seekable: tell = self.fp.tell while True: + if len(hlist) > _MAXHEADERS: + raise HTTPException("got more than %d headers" % _MAXHEADERS) if tell: try: startofline = tell() @@ -1185,21 +1191,29 @@ def __init__(self, host, port=None, key_file=None, cert_file=None, strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - source_address=None): + source_address=None, context=None): HTTPConnection.__init__(self, host, port, strict, timeout, source_address) self.key_file = key_file self.cert_file = cert_file + if context is None: + context = ssl._create_default_https_context() + if key_file or cert_file: + context.load_cert_chain(cert_file, key_file) + self._context = context def connect(self): "Connect to a host on a given (SSL) port." - sock = self._create_connection((self.host, self.port), - self.timeout, self.source_address) + HTTPConnection.connect(self) + if self._tunnel_host: - self.sock = sock - self._tunnel() - self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file) + server_hostname = self._tunnel_host + else: + server_hostname = self.host + + self.sock = self._context.wrap_socket(self.sock, + server_hostname=server_hostname) __all__.append("HTTPSConnection") @@ -1214,14 +1228,15 @@ _connection_class = HTTPSConnection def __init__(self, host='', port=None, key_file=None, cert_file=None, - strict=None): + strict=None, context=None): # provide a default host, pass the X509 cert info # urf. compensate for bad input. if port == 0: port = None self._setup(self._connection_class(host, port, key_file, - cert_file, strict)) + cert_file, strict, + context=context)) # we never actually use these for anything, but we keep them # here for compatibility with post-1.5.2 CVS. diff --git a/lib-python/2.7/idlelib/Bindings.py b/lib-python/2.7/idlelib/Bindings.py --- a/lib-python/2.7/idlelib/Bindings.py +++ b/lib-python/2.7/idlelib/Bindings.py @@ -75,7 +75,8 @@ ('!_Auto-open Stack Viewer', '<>'), ]), ('options', [ - ('_Configure IDLE...', '<>'), + ('Configure _IDLE', '<>'), + ('Configure _Extensions', '<>'), None, ]), ('help', [ diff --git a/lib-python/2.7/idlelib/CallTipWindow.py b/lib-python/2.7/idlelib/CallTipWindow.py --- a/lib-python/2.7/idlelib/CallTipWindow.py +++ b/lib-python/2.7/idlelib/CallTipWindow.py @@ -2,9 +2,8 @@ After ToolTip.py, which uses ideas gleaned from PySol Used by the CallTips IDLE extension. - """ -from Tkinter import * +from Tkinter import Toplevel, Label, LEFT, SOLID, TclError HIDE_VIRTUAL_EVENT_NAME = "<>" HIDE_SEQUENCES = ("", "") @@ -133,35 +132,28 @@ return bool(self.tipwindow) -def _calltip_window(parent): - root = Tk() - root.title("Test calltips") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) +def _calltip_window(parent): # htest # + from Tkinter import Toplevel, Text, LEFT, BOTH - class MyEditWin: # comparenceptually an editor_window - def __init__(self): - text = self.text = Text(root) - text.pack(side=LEFT, fill=BOTH, expand=1) - text.insert("insert", "string.split") - root.update() - self.calltip = CallTip(text) + top = Toplevel(parent) + top.title("Test calltips") + top.geometry("200x100+%d+%d" % (parent.winfo_rootx() + 200, + parent.winfo_rooty() + 150)) + text = Text(top) + text.pack(side=LEFT, fill=BOTH, expand=1) + text.insert("insert", "string.split") + top.update() + calltip = CallTip(text) - text.event_add("<>", "(") - text.event_add("<>", ")") - text.bind("<>", self.calltip_show) - text.bind("<>", self.calltip_hide) - - text.focus_set() - root.mainloop() - - def calltip_show(self, event): - self.calltip.showtip("Hello world", "insert", "end") - - def calltip_hide(self, event): - self.calltip.hidetip() - - editwin = MyEditWin() + def calltip_show(event): + calltip.showtip("(s=Hello world)", "insert", "end") + def calltip_hide(event): + calltip.hidetip() + text.event_add("<>", "(") + text.event_add("<>", ")") + text.bind("<>", calltip_show) + text.bind("<>", calltip_hide) + text.focus_set() if __name__=='__main__': from idlelib.idle_test.htest import run diff --git a/lib-python/2.7/idlelib/ClassBrowser.py b/lib-python/2.7/idlelib/ClassBrowser.py --- a/lib-python/2.7/idlelib/ClassBrowser.py +++ b/lib-python/2.7/idlelib/ClassBrowser.py @@ -19,6 +19,9 @@ from idlelib.TreeWidget import TreeNode, TreeItem, ScrolledCanvas from idlelib.configHandler import idleConf +file_open = None # Method...Item and Class...Item use this. +# Normally PyShell.flist.open, but there is no PyShell.flist for htest. + class ClassBrowser: def __init__(self, flist, name, path, _htest=False): @@ -27,6 +30,9 @@ """ _htest - bool, change box when location running htest. """ + global file_open + if not _htest: + file_open = PyShell.flist.open self.name = name self.file = os.path.join(path[0], self.name + ".py") self._htest = _htest @@ -101,7 +107,7 @@ return [] try: dict = pyclbr.readmodule_ex(name, [dir] + sys.path) - except ImportError, msg: + except ImportError: return [] items = [] self.classes = {} @@ -170,7 +176,7 @@ def OnDoubleClick(self): if not os.path.exists(self.file): return - edit = PyShell.flist.open(self.file) + edit = file_open(self.file) if hasattr(self.cl, 'lineno'): lineno = self.cl.lineno edit.gotoline(lineno) @@ -206,7 +212,7 @@ def OnDoubleClick(self): if not os.path.exists(self.file): return - edit = PyShell.flist.open(self.file) + edit = file_open(self.file) edit.gotoline(self.cl.methods[self.name]) def _class_browser(parent): #Wrapper for htest @@ -221,8 +227,9 @@ dir, file = os.path.split(file) name = os.path.splitext(file)[0] flist = PyShell.PyShellFileList(parent) + global file_open + file_open = flist.open ClassBrowser(flist, name, [dir], _htest=True) - parent.mainloop() if __name__ == "__main__": from idlelib.idle_test.htest import run diff --git a/lib-python/2.7/idlelib/ColorDelegator.py b/lib-python/2.7/idlelib/ColorDelegator.py --- a/lib-python/2.7/idlelib/ColorDelegator.py +++ b/lib-python/2.7/idlelib/ColorDelegator.py @@ -2,7 +2,6 @@ import re import keyword import __builtin__ -from Tkinter import * from idlelib.Delegator import Delegator from idlelib.configHandler import idleConf @@ -34,7 +33,6 @@ prog = re.compile(make_pat(), re.S) idprog = re.compile(r"\s+(\w+)", re.S) -asprog = re.compile(r".*?\b(as)\b") class ColorDelegator(Delegator): @@ -42,7 +40,6 @@ Delegator.__init__(self) self.prog = prog self.idprog = idprog - self.asprog = asprog self.LoadTagDefs() def setdelegate(self, delegate): @@ -74,7 +71,6 @@ "DEFINITION": idleConf.GetHighlight(theme, "definition"), "SYNC": {'background':None,'foreground':None}, "TODO": {'background':None,'foreground':None}, - "BREAK": idleConf.GetHighlight(theme, "break"), "ERROR": idleConf.GetHighlight(theme, "error"), # The following is used by ReplaceDialog: "hit": idleConf.GetHighlight(theme, "hit"), @@ -216,22 +212,6 @@ self.tag_add("DEFINITION", head + "+%dc" % a, head + "+%dc" % b) - elif value == "import": - # color all the "as" words on same line, except - # if in a comment; cheap approximation to the - # truth - if '#' in chars: - endpos = chars.index('#') - else: - endpos = len(chars) - while True: - m1 = self.asprog.match(chars, b, endpos) - if not m1: - break - a, b = m1.span(1) - self.tag_add("KEYWORD", - head + "+%dc" % a, - head + "+%dc" % b) m = self.prog.search(chars, m.end()) if "SYNC" in self.tag_names(next + "-1c"): head = next @@ -255,20 +235,23 @@ for tag in self.tagdefs.keys(): self.tag_remove(tag, "1.0", "end") -def _color_delegator(parent): +def _color_delegator(parent): # htest # + from Tkinter import Toplevel, Text from idlelib.Percolator import Percolator - root = Tk() - root.title("Test ColorDelegator") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) - source = "if somename: x = 'abc' # comment\nprint" - text = Text(root, background="white") + + top = Toplevel(parent) + top.title("Test ColorDelegator") + top.geometry("200x100+%d+%d" % (parent.winfo_rootx() + 200, + parent.winfo_rooty() + 150)) + source = "if somename: x = 'abc' # comment\nprint\n" + text = Text(top, background="white") + text.pack(expand=1, fill="both") text.insert("insert", source) - text.pack(expand=1, fill="both") + text.focus_set() + p = Percolator(text) d = ColorDelegator() p.insertfilter(d) - root.mainloop() if __name__ == "__main__": from idlelib.idle_test.htest import run diff --git a/lib-python/2.7/idlelib/Debugger.py b/lib-python/2.7/idlelib/Debugger.py --- a/lib-python/2.7/idlelib/Debugger.py +++ b/lib-python/2.7/idlelib/Debugger.py @@ -1,6 +1,5 @@ import os import bdb -import types from Tkinter import * from idlelib.WindowList import ListedToplevel from idlelib.ScrolledList import ScrolledList diff --git a/lib-python/2.7/idlelib/EditorWindow.py b/lib-python/2.7/idlelib/EditorWindow.py --- a/lib-python/2.7/idlelib/EditorWindow.py +++ b/lib-python/2.7/idlelib/EditorWindow.py @@ -1,6 +1,6 @@ import sys import os -from platform import python_version +import platform import re import imp from Tkinter import * @@ -22,6 +22,8 @@ # The default tab setting for a Text widget, in average-width characters. TK_TABWIDTH_DEFAULT = 8 +_py_version = ' (%s)' % platform.python_version() + def _sphinx_version(): "Format sys.version_info to produce the Sphinx version string used to install the chm docs" major, minor, micro, level, serial = sys.version_info @@ -151,7 +153,7 @@ # Safari requires real file:-URLs EditorWindow.help_url = 'file://' + EditorWindow.help_url else: - EditorWindow.help_url = "http://docs.python.org/%d.%d" % sys.version_info[:2] + EditorWindow.help_url = "https://docs.python.org/%d.%d/" % sys.version_info[:2] currentTheme=idleConf.CurrentTheme() self.flist = flist root = root or flist.root @@ -214,6 +216,8 @@ text.bind("<>", self.python_docs) text.bind("<>", self.about_dialog) text.bind("<>", self.config_dialog) + text.bind("<>", + self.config_extensions_dialog) text.bind("<>", self.open_module) text.bind("<>", lambda event: "break") text.bind("<>", self.select_all) @@ -568,6 +572,8 @@ def config_dialog(self, event=None): configDialog.ConfigDialog(self.top,'Settings') + def config_extensions_dialog(self, event=None): + configDialog.ConfigExtensionsDialog(self.top) def help_dialog(self, event=None): if self.root: @@ -691,30 +697,29 @@ return # XXX Ought to insert current file's directory in front of path try: - (f, file, (suffix, mode, type)) = _find_module(name) + (f, file_path, (suffix, mode, mtype)) = _find_module(name) except (NameError, ImportError) as msg: tkMessageBox.showerror("Import error", str(msg), parent=self.text) return - if type != imp.PY_SOURCE: + if mtype != imp.PY_SOURCE: tkMessageBox.showerror("Unsupported type", "%s is not a source module" % name, parent=self.text) return if f: f.close() if self.flist: - self.flist.open(file) + self.flist.open(file_path) else: - self.io.loadfile(file) + self.io.loadfile(file_path) + return file_path def open_class_browser(self, event=None): filename = self.io.filename - if not filename: - tkMessageBox.showerror( - "No filename", - "This buffer has no associated filename", - master=self.text) - self.text.focus_set() - return None + if not (self.__class__.__name__ == 'PyShellEditorWindow' + and filename): + filename = self.open_module() + if filename is None: + return head, tail = os.path.split(filename) base, ext = os.path.splitext(tail) from idlelib import ClassBrowser @@ -779,7 +784,7 @@ self.color = None def ResetColorizer(self): - "Update the colour theme" + "Update the color theme" # Called from self.filename_change_hook and from configDialog.py self._rmcolorizer() self._addcolorizer() @@ -944,7 +949,7 @@ short = self.short_title() long = self.long_title() if short and long: - title = short + " - " + long + title = short + " - " + long + _py_version elif short: title = short elif long: @@ -968,14 +973,13 @@ self.undo.reset_undo() def short_title(self): - pyversion = "Python " + python_version() + ": " filename = self.io.filename if filename: filename = os.path.basename(filename) else: filename = "Untitled" # return unicode string to display non-ASCII chars correctly - return pyversion + self._filename_to_unicode(filename) + return self._filename_to_unicode(filename) def long_title(self): # return unicode string to display non-ASCII chars correctly @@ -1711,7 +1715,8 @@ tk.call('set', 'tcl_nonwordchars', '[^a-zA-Z0-9_]') -def _editor_window(parent): +def _editor_window(parent): # htest # + # error if close master window first - timer event, after script root = parent fixwordbreaks(root) if sys.argv[1:]: @@ -1721,7 +1726,8 @@ macosxSupport.setupApp(root, None) edit = EditorWindow(root=root, filename=filename) edit.text.bind("<>", edit.close_event) - parent.mainloop() + # Does not stop error, neither does following + # edit.text.bind("<>", edit.close_event) if __name__ == '__main__': diff --git a/lib-python/2.7/idlelib/GrepDialog.py b/lib-python/2.7/idlelib/GrepDialog.py --- a/lib-python/2.7/idlelib/GrepDialog.py +++ b/lib-python/2.7/idlelib/GrepDialog.py @@ -45,10 +45,10 @@ def create_entries(self): SearchDialogBase.create_entries(self) - self.globent = self.make_entry("In files:", self.globvar) + self.globent = self.make_entry("In files:", self.globvar)[0] def create_other_buttons(self): - f = self.make_frame() + f = self.make_frame()[0] btn = Checkbutton(f, anchor="w", variable=self.recvar, @@ -131,7 +131,7 @@ self.top.withdraw() -def _grep_dialog(parent): # for htest +def _grep_dialog(parent): # htest # from idlelib.PyShell import PyShellFileList root = Tk() root.title("Test GrepDialog") diff --git a/lib-python/2.7/idlelib/IOBinding.py b/lib-python/2.7/idlelib/IOBinding.py --- a/lib-python/2.7/idlelib/IOBinding.py +++ b/lib-python/2.7/idlelib/IOBinding.py @@ -19,11 +19,7 @@ from idlelib.configHandler import idleConf -try: - from codecs import BOM_UTF8 -except ImportError: - # only available since Python 2.3 - BOM_UTF8 = '\xef\xbb\xbf' +from codecs import BOM_UTF8 # Try setting the locale, so that we can find out # what encoding to use @@ -72,6 +68,7 @@ encoding = encoding.lower() coding_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)') +blank_re = re.compile(r'^[ \t\f]*(?:[#\r\n]|$)') class EncodingMessage(SimpleDialog): "Inform user that an encoding declaration is needed." @@ -130,6 +127,8 @@ match = coding_re.match(line) if match is not None: break + if not blank_re.match(line): + return None else: return None name = match.group(1) @@ -529,6 +528,8 @@ ("All files", "*"), ] + defaultextension = '.py' if sys.platform == 'darwin' else '' + def askopenfile(self): dir, base = self.defaultfilename("open") if not self.opendialog: @@ -554,8 +555,10 @@ def asksavefile(self): dir, base = self.defaultfilename("save") if not self.savedialog: - self.savedialog = tkFileDialog.SaveAs(master=self.text, - filetypes=self.filetypes) + self.savedialog = tkFileDialog.SaveAs( + master=self.text, + filetypes=self.filetypes, + defaultextension=self.defaultextension) filename = self.savedialog.show(initialdir=dir, initialfile=base) if isinstance(filename, unicode): filename = filename.encode(filesystemencoding) diff --git a/lib-python/2.7/idlelib/NEWS.txt b/lib-python/2.7/idlelib/NEWS.txt --- a/lib-python/2.7/idlelib/NEWS.txt +++ b/lib-python/2.7/idlelib/NEWS.txt @@ -1,6 +1,183 @@ +What's New in IDLE 2.7.9? +========================= + +*Release data: 2014-12-07* (projected) + +- Issue #16893: Update Idle doc chapter to match current Idle and add new + information. + +- Issue #3068: Add Idle extension configuration dialog to Options menu. + Changes are written to HOME/.idlerc/config-extensions.cfg. + Original patch by Tal Einat. + +- Issue #16233: A module browser (File : Class Browser, Alt+C) requires a + editor window with a filename. When Class Browser is requested otherwise, + from a shell, output window, or 'Untitled' editor, Idle no longer displays + an error box. It now pops up an Open Module box (Alt+M). If a valid name + is entered and a module is opened, a corresponding browser is also opened. + +- Issue #4832: Save As to type Python files automatically adds .py to the + name you enter (even if your system does not display it). Some systems + automatically add .txt when type is Text files. + +- Issue #21986: Code objects are not normally pickled by the pickle module. + To match this, they are no longer pickled when running under Idle. + +- Issue #22221: IDLE now ignores the source encoding declaration on the second + line if the first line contains anything except a comment. + +- Issue #17390: Adjust Editor window title; remove 'Python', + move version to end. + +- Issue #14105: Idle debugger breakpoints no longer disappear + when inseting or deleting lines. + + +What's New in IDLE 2.7.8? +========================= + +*Release date: 2014-06-29* + +- Issue #21940: Add unittest for WidgetRedirector. Initial patch by Saimadhav + Heblikar. + +- Issue #18592: Add unittest for SearchDialogBase. Patch by Phil Webster. + +- Issue #21694: Add unittest for ParenMatch. Patch by Saimadhav Heblikar. + +- Issue #21686: add unittest for HyperParser. Original patch by Saimadhav + Heblikar. + +- Issue #12387: Add missing upper(lower)case versions of default Windows key + bindings for Idle so Caps Lock does not disable them. Patch by Roger Serwy. + +- Issue #21695: Closing a Find-in-files output window while the search is + still in progress no longer closes Idle. + +- Issue #18910: Add unittest for textView. Patch by Phil Webster. + +- Issue #18292: Add unittest for AutoExpand. Patch by Saihadhav Heblikar. + +- Issue #18409: Add unittest for AutoComplete. Patch by Phil Webster. + + +What's New in IDLE 2.7.7? +========================= + +*Release date: 2014-05-31* + +- Issue #18104: Add idlelib/idle_test/htest.py with a few sample tests to begin + consolidating and improving human-validated tests of Idle. Change other files + as needed to work with htest. Running the module as __main__ runs all tests. + +- Issue #21139: Change default paragraph width to 72, the PEP 8 recommendation. + +- Issue #21284: Paragraph reformat test passes after user changes reformat width. + +- Issue #20406: Use Python application icons for Idle window title bars. + Patch mostly by Serhiy Storchaka. + +- Issue #21029: Occurrences of "print" are now consistently colored as + being a keyword (the colorizer doesn't know if print functions are + enabled in the source). + +- Issue #17721: Remove non-functional configuration dialog help button until we + make it actually gives some help when clicked. Patch by Guilherme Sim�es. + +- Issue #17390: Add Python version to Idle editor window title bar. + Original patches by Edmond Burnett and Kent Johnson. + +- Issue #20058: sys.stdin.readline() in IDLE now always returns only one line. + +- Issue #19481: print() of unicode, str or bytearray subclass instance in IDLE + no more hangs. + +- Issue #18270: Prevent possible IDLE AttributeError on OS X when no initial + shell window is present. + +- Issue #17654: Ensure IDLE menus are customized properly on OS X for + non-framework builds and for all variants of Tk. + + +What's New in IDLE 2.7.6? +========================= + +*Release date: 2013-11-10* + +- Issue #19426: Fixed the opening of Python source file with specified encoding. + +- Issue #18873: IDLE now detects Python source code encoding only in comment + lines. + +- Issue #18988: The "Tab" key now works when a word is already autocompleted. + +- Issue #18489: Add tests for SearchEngine. Original patch by Phil Webster. + +- Issue #18429: Format / Format Paragraph, now works when comment blocks + are selected. As with text blocks, this works best when the selection + only includes complete lines. + +- Issue #18226: Add docstrings and unittests for FormatParagraph.py. + Original patches by Todd Rovito and Phil Webster. + +- Issue #18279: Format - Strip trailing whitespace no longer marks a file as + changed when it has not been changed. This fix followed the addition of a + test file originally written by Phil Webster (the issue's main goal). + +- Issue #18539: Calltips now work for float default arguments. + +- Issue #7136: In the Idle File menu, "New Window" is renamed "New File". + Patch by Tal Einat, Roget Serwy, and Todd Rovito. + +- Issue #8515: Set __file__ when run file in IDLE. + Initial patch by Bruce Frederiksen. + +- Issue #5492: Avoid traceback when exiting IDLE caused by a race condition. + +- Issue #17511: Keep IDLE find dialog open after clicking "Find Next". + Original patch by Sarah K. + +- Issue #15392: Create a unittest framework for IDLE. + Preliminary patch by Rajagopalasarma Jayakrishnan + See Lib/idlelib/idle_test/README.txt for how to run Idle tests. + +- Issue #14146: Highlight source line while debugging on Windows. + +- Issue #17532: Always include Options menu for IDLE on OS X. + Patch by Guilherme Sim�es. + + What's New in IDLE 2.7.5? ========================= +*Release date: 2013-05-12* + +- Issue #17838: Allow sys.stdin to be reassigned. + +- Issue #14735: Update IDLE docs to omit "Control-z on Windows". + +- Issue #17585: Fixed IDLE regression. Now closes when using exit() or quit(). + +- Issue #17657: Show full Tk version in IDLE's about dialog. + Patch by Todd Rovito. + +- Issue #17613: Prevent traceback when removing syntax colorizer in IDLE. + +- Issue #1207589: Backwards-compatibility patch for right-click menu in IDLE. + +- Issue #16887: IDLE now accepts Cancel in tabify/untabify dialog box. + +- Issue #14254: IDLE now handles readline correctly across shell restarts. + +- Issue #17614: IDLE no longer raises exception when quickly closing a file. + +- Issue #6698: IDLE now opens just an editor window when configured to do so. + +- Issue #8900: Using keyboard shortcuts in IDLE to open a file no longer + raises an exception. + +- Issue #6649: Fixed missing exit status in IDLE. Patch by Guilherme Polo. + - Issue #17390: Display Python version on Idle title bar. Initial patch by Edmond Burnett. @@ -8,17 +185,67 @@ What's New in IDLE 2.7.4? ========================= +*Release date: 2013-04-06* + +- Issue #17625: In IDLE, close the replace dialog after it is used. + +- IDLE was displaying spurious SystemExit tracebacks when running scripts + that terminated by raising SystemExit (i.e. unittest and turtledemo). + +- Issue #9290: In IDLE the sys.std* streams now implement io.TextIOBase + interface and support all mandatory methods and properties. + +- Issue #16829: IDLE printing no longer fails if there are spaces or other + special characters in the file path. + +- Issue #16819: IDLE method completion now correctly works for unicode literals. + +- Issue #16504: IDLE now catches SyntaxErrors raised by tokenizer. Patch by + Roger Serwy. + +- Issue #1207589: Add Cut/Copy/Paste items to IDLE right click Context Menu + Patch by Todd Rovito. + +- Issue #13052: Fix IDLE crashing when replace string in Search/Replace dialog + ended with '\'. Patch by Roger Serwy. + +- Issue #9803: Don't close IDLE on saving if breakpoint is open. + Patch by Roger Serwy. + +- Issue #14958: Change IDLE systax highlighting to recognize all string and byte + literals currently supported in Python 2.7. + +- Issue #14962: Update text coloring in IDLE shell window after changing + options. Patch by Roger Serwy. + +- Issue #10997: Prevent a duplicate entry in IDLE's "Recent Files" menu. + +- Issue #12510: Attempting to get invalid tooltip no longer closes IDLE. + Original patch by Roger Serwy. + +- Issue #10365: File open dialog now works instead of crashing + even when parent window is closed. Patch by Roger Serwy. + +- Issue #14876: Use user-selected font for highlight configuration. + Patch by Roger Serwy. + +- Issue #14409: IDLE now properly executes commands in the Shell window + when it cannot read the normal config files on startup and + has to use the built-in default key bindings. + There was previously a bug in one of the defaults. + +- Issue #3573: IDLE hangs when passing invalid command line args + (directory(ies) instead of file(s)) (Patch by Guilherme Polo) + +- Issue #5219: Prevent event handler cascade in IDLE. + - Issue #15318: Prevent writing to sys.stdin. - Issue #13532, #15319: Check that arguments to sys.stdout.write are strings. -- Issue # 12510: Attempt to get certain tool tips no longer crashes IDLE. - -- Issue10365: File open dialog now works instead of crashing even when +- Issue #10365: File open dialog now works instead of crashing even when parent window is closed while dialog is open. -- Issue 14876: use user-selected font for highlight configuration. - - Issue #14018: Update checks for unstable system Tcl/Tk versions on OS X to include versions shipped with OS X 10.7 and 10.8 in addition to 10.6. @@ -29,6 +256,27 @@ From noreply at buildbot.pypy.org Mon Mar 16 18:32:14 2015 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 16 Mar 2015 18:32:14 +0100 (CET) Subject: [pypy-commit] pypy object-dtype2: wip - gc customtrace Message-ID: <20150316173214.02AF41C0455@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: object-dtype2 Changeset: r76415:275dc69bef98 Date: 2015-03-16 19:04 +0200 http://bitbucket.org/pypy/pypy/changeset/275dc69bef98/ Log: wip - gc customtrace diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py --- a/pypy/module/micronumpy/compile.py +++ b/pypy/module/micronumpy/compile.py @@ -511,6 +511,8 @@ dtype = get_dtype_cache(interp.space).w_int64dtype elif self.v == 'float': dtype = get_dtype_cache(interp.space).w_float64dtype + elif self.v == 'object': + dtype = get_dtype_cache(interp.space).w_objectdtype else: raise BadToken('unknown v to dtype "%s"' % self.v) return dtype diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py --- a/pypy/module/micronumpy/concrete.py +++ b/pypy/module/micronumpy/concrete.py @@ -1,5 +1,5 @@ from pypy.interpreter.error import OperationError, oefmt -from rpython.rlib import jit +from rpython.rlib import jit, rgc from rpython.rlib.buffer import Buffer from rpython.rlib.debug import make_sure_not_resized from rpython.rlib.rawstorage import alloc_raw_storage, free_raw_storage, \ @@ -17,7 +17,7 @@ class BaseConcreteArray(object): _immutable_fields_ = ['dtype?', 'storage', 'start', 'size', 'shape[*]', - 'strides[*]', 'backstrides[*]', 'order'] + 'strides[*]', 'backstrides[*]', 'order', 'gcstruct'] start = 0 parent = None flags = 0 @@ -347,6 +347,7 @@ self.backstrides = backstrides self.storage = storage self.start = start + self.gcstruct = None def fill(self, space, box): self.dtype.itemtype.fill(self.storage, self.dtype.elsize, @@ -374,20 +375,32 @@ def base(self): return None +OBJECTSTORE = lltype.GcStruct('ObjectStore', + ('storage', llmemory.Address), + rtti=True) +def customtrace(gc, obj, callback, arg): + xxxx +lambda_customtrace = lambda: customtrace + class ConcreteArray(ConcreteArrayNotOwning): def __init__(self, shape, dtype, order, strides, backstrides, storage=lltype.nullptr(RAW_STORAGE), zero=True): + + gcstruct = None if storage == lltype.nullptr(RAW_STORAGE): storage = dtype.itemtype.malloc(support.product(shape) * dtype.elsize, zero=zero) if dtype.num == NPY.OBJECT: - # Register a customtrace function for this storage - pass + rgc.register_custom_trace_hook(OBJECTSTORE, lambda_customtrace) + gcstruct = lltype.malloc(OBJECTSTORE) + gcstruct.storage = storage ConcreteArrayNotOwning.__init__(self, shape, dtype, order, strides, backstrides, storage) + self.gcstruct = gcstruct def __del__(self): + rgc. free_raw_storage(self.storage, track_allocation=False) From noreply at buildbot.pypy.org Mon Mar 16 18:32:15 2015 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 16 Mar 2015 18:32:15 +0100 (CET) Subject: [pypy-commit] pypy release-2.5.x: stick with a Terry Pratchett homage Message-ID: <20150316173215.2572B1C0455@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: release-2.5.x Changeset: r76416:2af69f62ef18 Date: 2015-03-16 19:14 +0200 http://bitbucket.org/pypy/pypy/changeset/2af69f62ef18/ Log: stick with a Terry Pratchett homage diff --git a/pypy/doc/release-2.5.1.rst b/pypy/doc/release-2.5.1.rst --- a/pypy/doc/release-2.5.1.rst +++ b/pypy/doc/release-2.5.1.rst @@ -1,8 +1,8 @@ -============================== -PyPy 2.5.1 - Hakea Hookeriana -============================== +================================ +PyPy 2.5.1 - Pineapple Bromeliad +================================ -We're pleased to announce PyPy 2.5.1, following on the heels of 2.5.0 +We're pleased to announce PyPy 2.5.1, Pineapple `Bromeliad`_ following on the heels of 2.5.0 You can download the PyPy 2.5.1 release here: @@ -24,6 +24,7 @@ * `NumPy`_ which requires installation of our fork of upstream numpy, available `on bitbucket`_ +.. _`Bromeliad`: http://xkcd.com/1498 .. _`Py3k`: http://pypy.org/py3donate.html .. _`STM`: http://pypy.org/tmdonate2.html .. _`NumPy`: http://pypy.org/numpydonate.html From noreply at buildbot.pypy.org Mon Mar 16 18:32:16 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 16 Mar 2015 18:32:16 +0100 (CET) Subject: [pypy-commit] pypy release-2.5.x: Use RLock instead of Lock. The goal is to ensure that only one thread Message-ID: <20150316173216.4AA7E1C0455@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: release-2.5.x Changeset: r76417:418c58fc67fb Date: 2015-03-16 17:11 +0100 http://bitbucket.org/pypy/pypy/changeset/418c58fc67fb/ Log: Use RLock instead of Lock. The goal is to ensure that only one thread at a time uses tkinter, if the C library is "non-threaded". But we're getting rare cases of deadlocks even with only one thread, which is likely a rare path ending up in a recursive locking. diff --git a/lib_pypy/_tkinter/app.py b/lib_pypy/_tkinter/app.py --- a/lib_pypy/_tkinter/app.py +++ b/lib_pypy/_tkinter/app.py @@ -96,7 +96,7 @@ if not self.threaded: # TCL is not thread-safe, calls needs to be serialized. - self._tcl_lock = threading.Lock() + self._tcl_lock = threading.RLock() else: self._tcl_lock = _DummyLock() From noreply at buildbot.pypy.org Mon Mar 16 18:34:18 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 16 Mar 2015 18:34:18 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: hg merge release-2.5.x (from 2.5.1) Message-ID: <20150316173418.D7F131C0579@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76418:746681435f00 Date: 2015-03-16 18:34 +0100 http://bitbucket.org/pypy/pypy/changeset/746681435f00/ Log: hg merge release-2.5.x (from 2.5.1) diff --git a/lib_pypy/_tkinter/app.py b/lib_pypy/_tkinter/app.py --- a/lib_pypy/_tkinter/app.py +++ b/lib_pypy/_tkinter/app.py @@ -96,7 +96,7 @@ if not self.threaded: # TCL is not thread-safe, calls needs to be serialized. - self._tcl_lock = threading.Lock() + self._tcl_lock = threading.RLock() else: self._tcl_lock = _DummyLock() diff --git a/pypy/doc/release-2.5.1.rst b/pypy/doc/release-2.5.1.rst --- a/pypy/doc/release-2.5.1.rst +++ b/pypy/doc/release-2.5.1.rst @@ -1,8 +1,8 @@ -============================== -PyPy 2.5.1 - Hakea Hookeriana -============================== +================================ +PyPy 2.5.1 - Pineapple Bromeliad +================================ -We're pleased to announce PyPy 2.5.1, following on the heels of 2.5.0 +We're pleased to announce PyPy 2.5.1, Pineapple `Bromeliad`_ following on the heels of 2.5.0 You can download the PyPy 2.5.1 release here: @@ -24,6 +24,7 @@ * `NumPy`_ which requires installation of our fork of upstream numpy, available `on bitbucket`_ +.. _`Bromeliad`: http://xkcd.com/1498 .. _`Py3k`: http://pypy.org/py3donate.html .. _`STM`: http://pypy.org/tmdonate2.html .. _`NumPy`: http://pypy.org/numpydonate.html From noreply at buildbot.pypy.org Mon Mar 16 19:17:52 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 16 Mar 2015 19:17:52 +0100 (CET) Subject: [pypy-commit] pypy default: Test and fix: a type's __eq__ or __ne__ methods must return Message-ID: <20150316181752.4436B1C009D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76419:c5e489d4b8a7 Date: 2015-03-16 19:17 +0100 http://bitbucket.org/pypy/pypy/changeset/c5e489d4b8a7/ Log: Test and fix: a type's __eq__ or __ne__ methods must return NotImplemented instead of False when given a non-type diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py --- a/pypy/objspace/std/test/test_typeobject.py +++ b/pypy/objspace/std/test/test_typeobject.py @@ -1165,3 +1165,17 @@ return x + 1 a = A() assert a.f(1) == 2 + + def test_eq_returns_notimplemented(self): + assert type.__eq__(int, 42) is NotImplemented + assert type.__ne__(dict, 42) is NotImplemented + assert type.__eq__(int, int) is True + assert type.__eq__(int, dict) is False + + def test_cmp_on_types(self): + class X(type): + def __cmp__(self, other): + return -1 + class Y: + __metaclass__ = X + assert (Y < Y) is True diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py --- a/pypy/objspace/std/typeobject.py +++ b/pypy/objspace/std/typeobject.py @@ -645,9 +645,13 @@ "type object '%N' has no attribute %R", self, w_name) def descr_eq(self, space, w_other): + if not isinstance(w_other, W_TypeObject): + return space.w_NotImplemented return space.is_(self, w_other) def descr_ne(self, space, w_other): + if not isinstance(w_other, W_TypeObject): + return space.w_NotImplemented return space.newbool(not space.is_w(self, w_other)) From noreply at buildbot.pypy.org Mon Mar 16 20:55:02 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 16 Mar 2015 20:55:02 +0100 (CET) Subject: [pypy-commit] pypy release-2.5.x: Test and fix: a type's __eq__ or __ne__ methods must return Message-ID: <20150316195502.596311C009D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: release-2.5.x Changeset: r76420:5aed0a41060e Date: 2015-03-16 19:17 +0100 http://bitbucket.org/pypy/pypy/changeset/5aed0a41060e/ Log: Test and fix: a type's __eq__ or __ne__ methods must return NotImplemented instead of False when given a non-type diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py --- a/pypy/objspace/std/test/test_typeobject.py +++ b/pypy/objspace/std/test/test_typeobject.py @@ -1165,3 +1165,17 @@ return x + 1 a = A() assert a.f(1) == 2 + + def test_eq_returns_notimplemented(self): + assert type.__eq__(int, 42) is NotImplemented + assert type.__ne__(dict, 42) is NotImplemented + assert type.__eq__(int, int) is True + assert type.__eq__(int, dict) is False + + def test_cmp_on_types(self): + class X(type): + def __cmp__(self, other): + return -1 + class Y: + __metaclass__ = X + assert (Y < Y) is True diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py --- a/pypy/objspace/std/typeobject.py +++ b/pypy/objspace/std/typeobject.py @@ -645,9 +645,13 @@ "type object '%N' has no attribute %R", self, w_name) def descr_eq(self, space, w_other): + if not isinstance(w_other, W_TypeObject): + return space.w_NotImplemented return space.is_(self, w_other) def descr_ne(self, space, w_other): + if not isinstance(w_other, W_TypeObject): + return space.w_NotImplemented return space.newbool(not space.is_w(self, w_other)) From noreply at buildbot.pypy.org Mon Mar 16 22:41:17 2015 From: noreply at buildbot.pypy.org (mjacob) Date: Mon, 16 Mar 2015 22:41:17 +0100 (CET) Subject: [pypy-commit] pypy default: Don't reraise CompilationError as ImportError in rpython.rlib.rzlib. Message-ID: <20150316214117.C8FFE1C0455@cobra.cs.uni-duesseldorf.de> Author: Manuel Jacob Branch: Changeset: r76421:1e9ff6a329e8 Date: 2015-03-16 22:39 +0100 http://bitbucket.org/pypy/pypy/changeset/1e9ff6a329e8/ Log: Don't reraise CompilationError as ImportError in rpython.rlib.rzlib. This was a bit annoying if rffi_platform.configure_external_library failed for reasons other than missing zlib headers because it masked the actual error. I also changed all places which expected rzlib to raise ImportError. If I missed something or you disagree with this change, feel free to back it out. diff --git a/pypy/module/zipimport/test/test_zipimport_deflated.py b/pypy/module/zipimport/test/test_zipimport_deflated.py --- a/pypy/module/zipimport/test/test_zipimport_deflated.py +++ b/pypy/module/zipimport/test/test_zipimport_deflated.py @@ -14,7 +14,7 @@ def setup_class(cls): try: import rpython.rlib.rzlib - except ImportError: + except CompilationError: py.test.skip("zlib not available, cannot test compressed zipfiles") cls.make_class() cls.w_BAD_ZIP = cls.space.wrap(BAD_ZIP) diff --git a/rpython/rlib/rzipfile.py b/rpython/rlib/rzipfile.py --- a/rpython/rlib/rzipfile.py +++ b/rpython/rlib/rzipfile.py @@ -8,7 +8,7 @@ try: from rpython.rlib import rzlib -except (ImportError, CompilationError): +except CompilationError: rzlib = None crc_32_tab = [ diff --git a/rpython/rlib/rzlib.py b/rpython/rlib/rzlib.py --- a/rpython/rlib/rzlib.py +++ b/rpython/rlib/rzlib.py @@ -22,13 +22,10 @@ includes=['zlib.h'], testonly_libraries = testonly_libraries ) -try: - eci = rffi_platform.configure_external_library( - libname, eci, - [dict(prefix='zlib-'), - ]) -except CompilationError: - raise ImportError("Could not find a zlib library") +eci = rffi_platform.configure_external_library( + libname, eci, + [dict(prefix='zlib-'), + ]) constantnames = ''' diff --git a/rpython/rlib/test/test_rzipfile.py b/rpython/rlib/test/test_rzipfile.py --- a/rpython/rlib/test/test_rzipfile.py +++ b/rpython/rlib/test/test_rzipfile.py @@ -9,7 +9,7 @@ try: from rpython.rlib import rzlib -except ImportError, e: +except CompilationError as e: py.test.skip("zlib not installed: %s " % (e, )) class BaseTestRZipFile(BaseRtypingTest): From noreply at buildbot.pypy.org Mon Mar 16 22:53:21 2015 From: noreply at buildbot.pypy.org (mjacob) Date: Mon, 16 Mar 2015 22:53:21 +0100 (CET) Subject: [pypy-commit] pypy default: Add additional library and include directories needed to cross-compile for AArch64. Message-ID: <20150316215321.53E761C0455@cobra.cs.uni-duesseldorf.de> Author: Manuel Jacob Branch: Changeset: r76422:e0584363e761 Date: 2015-03-16 22:53 +0100 http://bitbucket.org/pypy/pypy/changeset/e0584363e761/ Log: Add additional library and include directories needed to cross- compile for AArch64. diff --git a/rpython/translator/platform/arm.py b/rpython/translator/platform/arm.py --- a/rpython/translator/platform/arm.py +++ b/rpython/translator/platform/arm.py @@ -21,11 +21,14 @@ available_librarydirs = [SB2 + '/lib/arm-linux-gnueabi/', SB2 + '/lib/arm-linux-gnueabihf/', + SB2 + '/lib/aarch64-linux-gnu/', SB2 + '/usr/lib/arm-linux-gnueabi/', - SB2 + '/usr/lib/arm-linux-gnueabihf/'] + SB2 + '/usr/lib/arm-linux-gnueabihf/', + SB2 + '/usr/lib/aarch64-linux-gnu/'] available_includedirs = [SB2 + '/usr/include/arm-linux-gnueabi/', - SB2 + '/usr/include/arm-linux-gnueabihf/'] + SB2 + '/usr/include/arm-linux-gnueabihf/', + SB2 + '/usr/include/aarch64-linux-gnu/'] copied_cache = {} From noreply at buildbot.pypy.org Tue Mar 17 01:54:42 2015 From: noreply at buildbot.pypy.org (mjacob) Date: Tue, 17 Mar 2015 01:54:42 +0100 (CET) Subject: [pypy-commit] pypy default: Pass -rpath-link to the linker for every additional library directory. Message-ID: <20150317005442.8F8AE1C0470@cobra.cs.uni-duesseldorf.de> Author: Manuel Jacob Branch: Changeset: r76423:a5c8f6daa2f8 Date: 2015-03-17 01:51 +0100 http://bitbucket.org/pypy/pypy/changeset/a5c8f6daa2f8/ Log: Pass -rpath-link to the linker for every additional library directory. This is needed for additional library directories to work with --shared. A PyPy translation only passes additional libary directories to the linker if cross-building for ARM. I verified that this is also needed if the library_dirs argument is passed to ExternalCompilationInfo. diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py --- a/rpython/translator/platform/posix.py +++ b/rpython/translator/platform/posix.py @@ -167,7 +167,8 @@ ('CC', self.cc), ('CC_LINK', eci.use_cpp_linker and 'g++' or '$(CC)'), ('LINKFILES', eci.link_files), - ('RPATH_FLAGS', self.rpath_flags), + ('RPATH_FLAGS', self.rpath_flags + ['-Wl,-rpath-link=\'%s\'' % ldir + for ldir in rel_libdirs]), ] for args in definitions: m.definition(*args) From noreply at buildbot.pypy.org Tue Mar 17 10:13:40 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Tue, 17 Mar 2015 10:13:40 +0100 (CET) Subject: [pypy-commit] pypy py3.3: Unicodedata: ensure that PUA codes cannot be used to retrieve aliases. Message-ID: <20150317091340.8EC791C0253@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3.3 Changeset: r76424:751c321375f4 Date: 2015-03-17 10:13 +0100 http://bitbucket.org/pypy/pypy/changeset/751c321375f4/ Log: Unicodedata: ensure that PUA codes cannot be used to retrieve aliases. diff --git a/pypy/module/unicodedata/interp_ucd.py b/pypy/module/unicodedata/interp_ucd.py --- a/pypy/module/unicodedata/interp_ucd.py +++ b/pypy/module/unicodedata/interp_ucd.py @@ -105,7 +105,7 @@ @unwrap_spec(name=str) def lookup(self, space, name): try: - code = self._lookup(name.upper()) + code = self._lookup(name.upper(), with_named_sequence=True) except KeyError: msg = space.mod(space.wrap("undefined character name '%s'"), space.wrap(name)) raise OperationError(space.w_KeyError, msg) diff --git a/pypy/module/unicodedata/test/test_unicodedata.py b/pypy/module/unicodedata/test/test_unicodedata.py --- a/pypy/module/unicodedata/test/test_unicodedata.py +++ b/pypy/module/unicodedata/test/test_unicodedata.py @@ -139,4 +139,11 @@ ] for seqname, codepoints in sequences: assert unicodedata.lookup(seqname) == codepoints + raises(SyntaxError, eval, r'"\N{%s}"' % seqname) + def test_names_in_pua_range(self): + # We are storing named seq in the PUA 15, but their names shouldn't leak + import unicodedata + for cp in range(0xf0000, 0xf0300, 7): + exc = raises(ValueError, unicodedata.name, chr(cp)) + assert str(exc.value) == 'no such name' diff --git a/rpython/rlib/unicodedata/generate_unicodedb.py b/rpython/rlib/unicodedata/generate_unicodedb.py --- a/rpython/rlib/unicodedata/generate_unicodedb.py +++ b/rpython/rlib/unicodedata/generate_unicodedb.py @@ -591,29 +591,32 @@ if not ('0' <= c <= '9' or 'A' <= c <= 'F'): raise KeyError code = int(cjk_code, 16) - if %s: + if %(cjk_interval)s: return code raise KeyError -def lookup(name): +def lookup(name, with_named_sequence=False): if name[:len(_cjk_prefix)] == _cjk_prefix: return _lookup_cjk(name[len(_cjk_prefix):]) if name[:len(_hangul_prefix)] == _hangul_prefix: return _lookup_hangul(name[len(_hangul_prefix):]) if not base_mod: - return trie_lookup(name) + code = trie_lookup(name) else: try: - return _code_by_name[name] + code = _code_by_name[name] except KeyError: if name not in _code_by_name_corrected: - return base_mod.trie_lookup(name) + code = base_mod.trie_lookup(name) else: raise + if not with_named_sequence and %(named_sequence_interval)s: + raise KeyError + return code def name(code): - if %s: + if %(cjk_interval)s: return "CJK UNIFIED IDEOGRAPH-" + hex(code)[2:].upper() if 0xAC00 <= code <= 0xD7A3: # vl_code, t_code = divmod(code - 0xAC00, len(_hangul_T)) @@ -624,6 +627,8 @@ v_code = vl_code %% len(_hangul_V) return ("HANGUL SYLLABLE " + _hangul_L[l_code] + _hangul_V[v_code] + _hangul_T[t_code]) + if %(pua_interval)s: + raise KeyError if not base_mod: return lookup_charcode(code) @@ -635,7 +640,9 @@ return base_mod.lookup_charcode(code) else: raise -''' % (cjk_interval, cjk_interval) +''' % dict(cjk_interval=cjk_interval, + pua_interval="0xF0000 <= code < 0xF0400", + named_sequence_interval="0xF0200 <= code < 0xF0400") # Categories writeDbRecord(outfile, table) @@ -810,8 +817,8 @@ print >> outfile, ']' print >> outfile, ''' -def lookup_with_alias(name): - code = lookup(name) +def lookup_with_alias(name, with_named_sequence=False): + code = lookup(name, with_named_sequence=with_named_sequence) if 0 <= code - %(start)s < len(_name_aliases): return _name_aliases[code - %(start)s] else: diff --git a/rpython/rlib/unicodedata/unicodedb_3_2_0.py b/rpython/rlib/unicodedata/unicodedb_3_2_0.py --- a/rpython/rlib/unicodedata/unicodedb_3_2_0.py +++ b/rpython/rlib/unicodedata/unicodedb_3_2_0.py @@ -16788,22 +16788,25 @@ return code raise KeyError -def lookup(name): +def lookup(name, with_named_sequence=False): if name[:len(_cjk_prefix)] == _cjk_prefix: return _lookup_cjk(name[len(_cjk_prefix):]) if name[:len(_hangul_prefix)] == _hangul_prefix: return _lookup_hangul(name[len(_hangul_prefix):]) if not base_mod: - return trie_lookup(name) + code = trie_lookup(name) else: try: - return _code_by_name[name] + code = _code_by_name[name] except KeyError: if name not in _code_by_name_corrected: - return base_mod.trie_lookup(name) + code = base_mod.trie_lookup(name) else: raise + if not with_named_sequence and 0xF0200 <= code < 0xF0400: + raise KeyError + return code def name(code): if (0x3400 <= code <= 0x4DB5 or 0x4E00 <= code <= 0x9FA5 or 0x20000 <= code <= 0x2A6D6): @@ -16817,6 +16820,8 @@ v_code = vl_code % len(_hangul_V) return ("HANGUL SYLLABLE " + _hangul_L[l_code] + _hangul_V[v_code] + _hangul_T[t_code]) + if 0xF0000 <= code < 0xF0400: + raise KeyError if not base_mod: return lookup_charcode(code) @@ -21288,8 +21293,8 @@ ] -def lookup_with_alias(name): - code = lookup(name) +def lookup_with_alias(name, with_named_sequence=False): + code = lookup(name, with_named_sequence=with_named_sequence) if 0 <= code - 983040 < len(_name_aliases): return _name_aliases[code - 983040] else: diff --git a/rpython/rlib/unicodedata/unicodedb_5_2_0.py b/rpython/rlib/unicodedata/unicodedb_5_2_0.py --- a/rpython/rlib/unicodedata/unicodedb_5_2_0.py +++ b/rpython/rlib/unicodedata/unicodedb_5_2_0.py @@ -136803,22 +136803,25 @@ return code raise KeyError -def lookup(name): +def lookup(name, with_named_sequence=False): if name[:len(_cjk_prefix)] == _cjk_prefix: return _lookup_cjk(name[len(_cjk_prefix):]) if name[:len(_hangul_prefix)] == _hangul_prefix: return _lookup_hangul(name[len(_hangul_prefix):]) if not base_mod: - return trie_lookup(name) + code = trie_lookup(name) else: try: - return _code_by_name[name] + code = _code_by_name[name] except KeyError: if name not in _code_by_name_corrected: - return base_mod.trie_lookup(name) + code = base_mod.trie_lookup(name) else: raise + if not with_named_sequence and 0xF0200 <= code < 0xF0400: + raise KeyError + return code def name(code): if (0x3400 <= code <= 0x4DB5 or 0x4E00 <= code <= 0x9FCB or 0x20000 <= code <= 0x2A6D6 or 0x2A700 <= code <= 0x2B734): @@ -136832,6 +136835,8 @@ v_code = vl_code % len(_hangul_V) return ("HANGUL SYLLABLE " + _hangul_L[l_code] + _hangul_V[v_code] + _hangul_T[t_code]) + if 0xF0000 <= code < 0xF0400: + raise KeyError if not base_mod: return lookup_charcode(code) @@ -157497,8 +157502,8 @@ ] -def lookup_with_alias(name): - code = lookup(name) +def lookup_with_alias(name, with_named_sequence=False): + code = lookup(name, with_named_sequence=with_named_sequence) if 0 <= code - 983040 < len(_name_aliases): return _name_aliases[code - 983040] else: diff --git a/rpython/rlib/unicodedata/unicodedb_6_0_0.py b/rpython/rlib/unicodedata/unicodedb_6_0_0.py --- a/rpython/rlib/unicodedata/unicodedb_6_0_0.py +++ b/rpython/rlib/unicodedata/unicodedb_6_0_0.py @@ -4520,22 +4520,25 @@ return code raise KeyError -def lookup(name): +def lookup(name, with_named_sequence=False): if name[:len(_cjk_prefix)] == _cjk_prefix: return _lookup_cjk(name[len(_cjk_prefix):]) if name[:len(_hangul_prefix)] == _hangul_prefix: return _lookup_hangul(name[len(_hangul_prefix):]) if not base_mod: - return trie_lookup(name) + code = trie_lookup(name) else: try: - return _code_by_name[name] + code = _code_by_name[name] except KeyError: if name not in _code_by_name_corrected: - return base_mod.trie_lookup(name) + code = base_mod.trie_lookup(name) else: raise + if not with_named_sequence and 0xF0200 <= code < 0xF0400: + raise KeyError + return code def name(code): if (0x3400 <= code <= 0x4DB5 or 0x4E00 <= code <= 0x9FCB or 0x20000 <= code <= 0x2A6D6 or 0x2A700 <= code <= 0x2B734 or 0x2B740 <= code <= 0x2B81D): @@ -4549,6 +4552,8 @@ v_code = vl_code % len(_hangul_V) return ("HANGUL SYLLABLE " + _hangul_L[l_code] + _hangul_V[v_code] + _hangul_T[t_code]) + if 0xF0000 <= code < 0xF0400: + raise KeyError if not base_mod: return lookup_charcode(code) @@ -7240,8 +7245,8 @@ ] -def lookup_with_alias(name): - code = lookup(name) +def lookup_with_alias(name, with_named_sequence=False): + code = lookup(name, with_named_sequence=with_named_sequence) if 0 <= code - 983040 < len(_name_aliases): return _name_aliases[code - 983040] else: diff --git a/rpython/rlib/unicodedata/unicodedb_6_2_0.py b/rpython/rlib/unicodedata/unicodedb_6_2_0.py --- a/rpython/rlib/unicodedata/unicodedb_6_2_0.py +++ b/rpython/rlib/unicodedata/unicodedb_6_2_0.py @@ -6890,22 +6890,25 @@ return code raise KeyError -def lookup(name): +def lookup(name, with_named_sequence=False): if name[:len(_cjk_prefix)] == _cjk_prefix: return _lookup_cjk(name[len(_cjk_prefix):]) if name[:len(_hangul_prefix)] == _hangul_prefix: return _lookup_hangul(name[len(_hangul_prefix):]) if not base_mod: - return trie_lookup(name) + code = trie_lookup(name) else: try: - return _code_by_name[name] + code = _code_by_name[name] except KeyError: if name not in _code_by_name_corrected: - return base_mod.trie_lookup(name) + code = base_mod.trie_lookup(name) else: raise + if not with_named_sequence and 0xF0200 <= code < 0xF0400: + raise KeyError + return code def name(code): if (0x3400 <= code <= 0x4DB5 or 0x4E00 <= code <= 0x9FCB or 0x20000 <= code <= 0x2A6D6 or 0x2A700 <= code <= 0x2B734 or 0x2B740 <= code <= 0x2B81D): @@ -6919,6 +6922,8 @@ v_code = vl_code % len(_hangul_V) return ("HANGUL SYLLABLE " + _hangul_L[l_code] + _hangul_V[v_code] + _hangul_T[t_code]) + if 0xF0000 <= code < 0xF0400: + raise KeyError if not base_mod: return lookup_charcode(code) @@ -10521,8 +10526,8 @@ ] -def lookup_with_alias(name): - code = lookup(name) +def lookup_with_alias(name, with_named_sequence=False): + code = lookup(name, with_named_sequence=with_named_sequence) if 0 <= code - 983040 < len(_name_aliases): return _name_aliases[code - 983040] else: From noreply at buildbot.pypy.org Tue Mar 17 11:02:53 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 17 Mar 2015 11:02:53 +0100 (CET) Subject: [pypy-commit] cffi default: Clarify this point in the docs Message-ID: <20150317100253.5A7B91C105F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1673:1e723dc022fe Date: 2015-03-17 11:03 +0100 http://bitbucket.org/cffi/cffi/changeset/1e723dc022fe/ Log: Clarify this point in the docs diff --git a/doc/source/index.rst b/doc/source/index.rst --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -1260,7 +1260,8 @@ buffer interface. This is the opposite of ``ffi.buffer()``. It gives a (read-write) reference to the existing data, not a copy; for this reason, and for PyPy compatibility, it does not work with the built-in -types str or unicode or bytearray. It is meant to be used on objects +types str or unicode or bytearray (or buffers/memoryviews on them). +It is meant to be used on objects containing large quantities of raw data, like ``array.array`` or numpy arrays. It supports both the old buffer API (in Python 2.x) and the new memoryview API. The original object is kept alive (and, in case From noreply at buildbot.pypy.org Tue Mar 17 11:20:34 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 17 Mar 2015 11:20:34 +0100 (CET) Subject: [pypy-commit] pypy default: "//app_main.py" => "/app_main.py" Message-ID: <20150317102034.BB48E1C0FF4@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76425:282f2ae9f319 Date: 2015-03-17 11:20 +0100 http://bitbucket.org/pypy/pypy/changeset/282f2ae9f319/ Log: "//app_main.py" => "/app_main.py" diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -136,7 +136,9 @@ filename = filename[:-1] basename = os.path.basename(filename) lastdirname = os.path.basename(os.path.dirname(filename)) - self.co_filename = '/%s/%s' % (lastdirname, basename) + if lastdirname: + basename = '%s/%s' % (lastdirname, basename) + self.co_filename = '/%s' % (basename,) co_names = property(lambda self: [self.space.unwrap(w_name) for w_name in self.co_names_w]) # for trace From noreply at buildbot.pypy.org Tue Mar 17 11:43:48 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 17 Mar 2015 11:43:48 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: translation fix Message-ID: <20150317104348.E58A21C0FAB@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76426:0a054b97ac06 Date: 2015-03-16 22:14 +0100 http://bitbucket.org/pypy/pypy/changeset/0a054b97ac06/ Log: translation fix diff --git a/rpython/memory/gctransform/stmframework.py b/rpython/memory/gctransform/stmframework.py --- a/rpython/memory/gctransform/stmframework.py +++ b/rpython/memory/gctransform/stmframework.py @@ -234,5 +234,5 @@ # to "stm_enter_callback_call/stm_leave_callback_call". pass - def walk_stack_roots(self, collect_stack_root): + def walk_stack_roots(self, collect_stack_root, is_minor=False): raise NotImplementedError diff --git a/rpython/translator/stm/readbarrier.py b/rpython/translator/stm/readbarrier.py --- a/rpython/translator/stm/readbarrier.py +++ b/rpython/translator/stm/readbarrier.py @@ -240,7 +240,7 @@ join_blocks(graph) annotator = stmtransformer.translator.annotator - insert_empty_startblock(annotator, graph) + insert_empty_startblock(graph) block_transformers = {} From noreply at buildbot.pypy.org Tue Mar 17 11:43:50 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 17 Mar 2015 11:43:50 +0100 (CET) Subject: [pypy-commit] pypy default: Don't release the GIL, but still make a wrapper, which is needed for stm Message-ID: <20150317104350.1E7D21C0FAB@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76427:16d1d97be9f1 Date: 2015-03-17 11:40 +0100 http://bitbucket.org/pypy/pypy/changeset/16d1d97be9f1/ Log: Don't release the GIL, but still make a wrapper, which is needed for stm diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py --- a/rpython/rlib/rthread.py +++ b/rpython/rlib/rthread.py @@ -60,7 +60,7 @@ releasegil=True) # release the GIL c_thread_releaselock = llexternal('RPyThreadReleaseLock', [TLOCKP], lltype.Signed, - _nowrapper=True) # *don't* release the GIL + releasegil=False) # *don't* release the GIL # another set of functions, this time in versions that don't cause the # GIL to be released. Used to be there to handle the GIL lock itself, From noreply at buildbot.pypy.org Tue Mar 17 11:43:51 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 17 Mar 2015 11:43:51 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Don't release the GIL, but still make a wrapper, which is needed for stm Message-ID: <20150317104351.3740C1C0FAB@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76428:1349c7207652 Date: 2015-03-17 11:40 +0100 http://bitbucket.org/pypy/pypy/changeset/1349c7207652/ Log: Don't release the GIL, but still make a wrapper, which is needed for stm diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py --- a/rpython/rlib/rthread.py +++ b/rpython/rlib/rthread.py @@ -60,7 +60,7 @@ releasegil=True) # release the GIL c_thread_releaselock = llexternal('RPyThreadReleaseLock', [TLOCKP], lltype.Signed, - _nowrapper=True) # *don't* release the GIL + releasegil=False) # *don't* release the GIL # another set of functions, this time in versions that don't cause the # GIL to be released. Used to be there to handle the GIL lock itself, From noreply at buildbot.pypy.org Tue Mar 17 11:43:52 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 17 Mar 2015 11:43:52 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: merge heads Message-ID: <20150317104352.55D9E1C0FAB@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76429:6ceb7c1456c3 Date: 2015-03-17 11:41 +0100 http://bitbucket.org/pypy/pypy/changeset/6ceb7c1456c3/ Log: merge heads diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py --- a/rpython/rlib/rthread.py +++ b/rpython/rlib/rthread.py @@ -60,7 +60,7 @@ releasegil=True) # release the GIL c_thread_releaselock = llexternal('RPyThreadReleaseLock', [TLOCKP], lltype.Signed, - _nowrapper=True) # *don't* release the GIL + releasegil=False) # *don't* release the GIL # another set of functions, this time in versions that don't cause the # GIL to be released. Used to be there to handle the GIL lock itself, From noreply at buildbot.pypy.org Tue Mar 17 11:43:53 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 17 Mar 2015 11:43:53 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: hg merge release-2.5.x Message-ID: <20150317104353.81E201C0FAB@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76430:5ed6e6e35d1f Date: 2015-03-17 11:42 +0100 http://bitbucket.org/pypy/pypy/changeset/5ed6e6e35d1f/ Log: hg merge release-2.5.x diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py --- a/pypy/objspace/std/test/test_typeobject.py +++ b/pypy/objspace/std/test/test_typeobject.py @@ -1165,3 +1165,17 @@ return x + 1 a = A() assert a.f(1) == 2 + + def test_eq_returns_notimplemented(self): + assert type.__eq__(int, 42) is NotImplemented + assert type.__ne__(dict, 42) is NotImplemented + assert type.__eq__(int, int) is True + assert type.__eq__(int, dict) is False + + def test_cmp_on_types(self): + class X(type): + def __cmp__(self, other): + return -1 + class Y: + __metaclass__ = X + assert (Y < Y) is True diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py --- a/pypy/objspace/std/typeobject.py +++ b/pypy/objspace/std/typeobject.py @@ -650,9 +650,13 @@ "type object '%N' has no attribute %R", self, w_name) def descr_eq(self, space, w_other): + if not isinstance(w_other, W_TypeObject): + return space.w_NotImplemented return space.is_(self, w_other) def descr_ne(self, space, w_other): + if not isinstance(w_other, W_TypeObject): + return space.w_NotImplemented return space.newbool(not space.is_w(self, w_other)) From noreply at buildbot.pypy.org Tue Mar 17 11:43:54 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 17 Mar 2015 11:43:54 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: lldebug translation fix Message-ID: <20150317104354.A2D7C1C0FAB@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76431:473556af41c9 Date: 2015-03-17 11:43 +0100 http://bitbucket.org/pypy/pypy/changeset/473556af41c9/ Log: lldebug translation fix diff --git a/rpython/translator/c/src/mem.c b/rpython/translator/c/src/mem.c --- a/rpython/translator/c/src/mem.c +++ b/rpython/translator/c/src/mem.c @@ -86,7 +86,7 @@ { struct pypy_debug_alloc_s **p; if (!addr) - return; + return 1; spinlock_acquire(pypy_debug_alloc_lock); for (p = &pypy_debug_alloc_list; *p; p = &((*p)->next)) if ((*p)->addr == addr) From noreply at buildbot.pypy.org Tue Mar 17 11:51:51 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 17 Mar 2015 11:51:51 +0100 (CET) Subject: [pypy-commit] pypy default: Backed out changeset 16d1d97be9f1 Message-ID: <20150317105151.842721C0EFC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76432:b5c2b1e26b71 Date: 2015-03-17 11:49 +0100 http://bitbucket.org/pypy/pypy/changeset/b5c2b1e26b71/ Log: Backed out changeset 16d1d97be9f1 diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py --- a/rpython/rlib/rthread.py +++ b/rpython/rlib/rthread.py @@ -60,7 +60,7 @@ releasegil=True) # release the GIL c_thread_releaselock = llexternal('RPyThreadReleaseLock', [TLOCKP], lltype.Signed, - releasegil=False) # *don't* release the GIL + _nowrapper=True) # *don't* release the GIL # another set of functions, this time in versions that don't cause the # GIL to be released. Used to be there to handle the GIL lock itself, From noreply at buildbot.pypy.org Tue Mar 17 11:51:52 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 17 Mar 2015 11:51:52 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: hg backout 1349c7207652 Message-ID: <20150317105152.9F1181C0EFC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76433:6b0d72fa2e61 Date: 2015-03-17 11:51 +0100 http://bitbucket.org/pypy/pypy/changeset/6b0d72fa2e61/ Log: hg backout 1349c7207652 diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py --- a/rpython/rlib/rthread.py +++ b/rpython/rlib/rthread.py @@ -60,7 +60,7 @@ releasegil=True) # release the GIL c_thread_releaselock = llexternal('RPyThreadReleaseLock', [TLOCKP], lltype.Signed, - releasegil=False) # *don't* release the GIL + _nowrapper=True) # *don't* release the GIL # another set of functions, this time in versions that don't cause the # GIL to be released. Used to be there to handle the GIL lock itself, From noreply at buildbot.pypy.org Tue Mar 17 16:28:46 2015 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 17 Mar 2015 16:28:46 +0100 (CET) Subject: [pypy-commit] pypy dtrace-support: special casing for freebsd, to be tested Message-ID: <20150317152846.B542F1C1E25@cobra.cs.uni-duesseldorf.de> Author: fijal Branch: dtrace-support Changeset: r76434:1f895d6aad66 Date: 2015-03-17 17:25 +0200 http://bitbucket.org/pypy/pypy/changeset/1f895d6aad66/ Log: special casing for freebsd, to be tested diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py --- a/rpython/translator/c/genc.py +++ b/rpython/translator/c/genc.py @@ -488,6 +488,14 @@ mk.definition('DEBUGFLAGS', '-O1 -g -fPIC') else: mk.definition('DEBUGFLAGS', '-O1 -g') + if (self.config.translation.dtrace and + not self.translator.platform.name.startswith('darwin')): + # right now dtrace is incompatible with asmgcc on all platforms + # I think + assert self.config.translation.gcrootfinder != 'asmgcc' + mk.definition('OBJECTS1', '$(subst .c,.o,$(SOURCES))') + mk.definition('OBJECTS', '$(OBJECTS1) dtrace_marker') + mk.rule('dtrace_marker', '', 'dtrace -G -s pypy.d $(OBJECTS)') if self.translator.platform.name == 'msvc': mk.rule('debug_target', 'debugmode_$(DEFAULT_TARGET)', 'rem') else: From noreply at buildbot.pypy.org Tue Mar 17 17:20:07 2015 From: noreply at buildbot.pypy.org (antocuni) Date: Tue, 17 Mar 2015 17:20:07 +0100 (CET) Subject: [pypy-commit] pypy default: extract libpypy-c.so too now that it's needed Message-ID: <20150317162007.4CEF11C00EF@cobra.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: Changeset: r76435:d0354c8dfeb8 Date: 2015-03-17 17:17 +0100 http://bitbucket.org/pypy/pypy/changeset/d0354c8dfeb8/ Log: extract libpypy-c.so too now that it's needed diff --git a/pypy/goal/getnightly.py b/pypy/goal/getnightly.py --- a/pypy/goal/getnightly.py +++ b/pypy/goal/getnightly.py @@ -7,7 +7,7 @@ if sys.platform.startswith('linux'): arch = 'linux' cmd = 'wget "%s"' - tar = "tar -x -v --wildcards --strip-components=2 -f %s '*/bin/pypy'" + tar = "tar -x -v --wildcards --strip-components=2 -f %s '*/bin/pypy' '*/bin/libpypy-c.so'" if os.uname()[-1].startswith('arm'): arch += '-armhf-raspbian' elif sys.platform.startswith('darwin'): From noreply at buildbot.pypy.org Tue Mar 17 17:20:08 2015 From: noreply at buildbot.pypy.org (antocuni) Date: Tue, 17 Mar 2015 17:20:08 +0100 (CET) Subject: [pypy-commit] pypy default: fix issue #2000, which was caused by 2e4e36c84077: partially revert it, by keeping the AST transformation but also allow the possibility to pass a tuple to it Message-ID: <20150317162008.6A7ED1C00EF@cobra.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: Changeset: r76436:9159bcc2b0d0 Date: 2015-03-17 17:20 +0100 http://bitbucket.org/pypy/pypy/changeset/9159bcc2b0d0/ Log: fix issue #2000, which was caused by 2e4e36c84077: partially revert it, by keeping the AST transformation but also allow the possibility to pass a tuple to it diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -1619,6 +1619,13 @@ def prepare_exec(f, prog, globals, locals, compile_flags, builtin, codetype): """Manipulate parameters to exec statement to (codeobject, dict, dict). """ + if (globals is None and locals is None and + isinstance(prog, tuple) and + (len(prog) == 2 or len(prog) == 3)): + globals = prog[1] + if len(prog) == 3: + locals = prog[2] + prog = prog[0] if globals is None: globals = f.f_globals if locals is None: diff --git a/pypy/interpreter/test/test_exec.py b/pypy/interpreter/test/test_exec.py --- a/pypy/interpreter/test/test_exec.py +++ b/pypy/interpreter/test/test_exec.py @@ -262,3 +262,11 @@ """] for c in code: compile(c, "", "exec") + + def test_exec_tuple(self): + # note: this is VERY different than testing exec("a = 42", d), because + # this specific case is handled specially by the AST compiler + d = {} + x = ("a = 42", d) + exec x + assert d['a'] == 42 From noreply at buildbot.pypy.org Tue Mar 17 19:11:51 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 17 Mar 2015 19:11:51 +0100 (CET) Subject: [pypy-commit] pypy default: Mention it as a bug Message-ID: <20150317181151.A1AA11C08AF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76437:cb01edcb5941 Date: 2015-03-17 19:11 +0100 http://bitbucket.org/pypy/pypy/changeset/cb01edcb5941/ Log: Mention it as a bug diff --git a/pypy/doc/stm.rst b/pypy/doc/stm.rst --- a/pypy/doc/stm.rst +++ b/pypy/doc/stm.rst @@ -130,7 +130,11 @@ * Weakrefs might appear to work a bit strangely for now, sometimes staying alive throught ``gc.collect()``, or even dying but then - un-dying for a short time before dying again. + un-dying for a short time before dying again. A similar problem can + show up occasionally elsewhere with accesses to some external + resources, where the (apparent) serialized order doesn't match the + underlying (multithreading) order. These are bugs (partially fixed + already in ``stmgc-c8``). * The STM system is based on very efficient read/write barriers, which are mostly done (their placement could be improved a bit in From noreply at buildbot.pypy.org Tue Mar 17 20:06:43 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 17 Mar 2015 20:06:43 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Temporary workaround: start a new transaction around a lock.release() Message-ID: <20150317190643.AC80A1C046C@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76438:7b2e2b3f44f0 Date: 2015-03-17 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/7b2e2b3f44f0/ Log: Temporary workaround: start a new transaction around a lock.release() diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py --- a/rpython/rlib/rthread.py +++ b/rpython/rlib/rthread.py @@ -61,6 +61,9 @@ c_thread_releaselock = llexternal('RPyThreadReleaseLock', [TLOCKP], lltype.Signed, _nowrapper=True) # *don't* release the GIL +c_thread_releaselock_GIL = llexternal('RPyThreadReleaseLock', [TLOCKP], + lltype.Signed, + releasegil=True) # another set of functions, this time in versions that don't cause the # GIL to be released. Used to be there to handle the GIL lock itself, @@ -158,7 +161,11 @@ return res def release(self): - if c_thread_releaselock(self._lock) != 0: + if rgc.stm_is_enabled(): + func = c_thread_releaselock_GIL # XXX temporary workaround! + else: + func = c_thread_releaselock + if func(self._lock) != 0: raise error("the lock was not previously acquired") def __del__(self): From noreply at buildbot.pypy.org Tue Mar 17 20:29:29 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Tue, 17 Mar 2015 20:29:29 +0100 (CET) Subject: [pypy-commit] pypy exc-later: Delay analysis of specialised exception exit cases until the end of build_flow() Message-ID: <20150317192929.BCF1A1C046C@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: exc-later Changeset: r76439:35a38a39212e Date: 2015-03-13 02:22 +0000 http://bitbucket.org/pypy/pypy/changeset/35a38a39212e/ Log: Delay analysis of specialised exception exit cases until the end of build_flow() This should enable simplifications of exception handling. diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py --- a/rpython/flowspace/flowcontext.py +++ b/rpython/flowspace/flowcontext.py @@ -124,7 +124,7 @@ def guessexception(self, ctx, *cases): block = self.crnt_block links = [] - for case in [None] + list(cases): + for case in [None, Exception]: if case is not None: if case is Exception: last_exc = Variable('last_exception') @@ -398,6 +398,8 @@ block = self.pendingblocks.popleft() if not block.dead: self.record_block(block) + from rpython.translator.simplify import specialize_exceptions + specialize_exceptions(graph) def record_block(self, block): self.setstate(block.framestate) diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py --- a/rpython/translator/simplify.py +++ b/rpython/translator/simplify.py @@ -9,13 +9,13 @@ from rpython.tool.algo.unionfind import UnionFind from rpython.flowspace.model import ( - Variable, Constant, checkgraph, mkentrymap) + Variable, Constant, checkgraph, mkentrymap, const, Link) from rpython.flowspace.operation import OverflowingOperation, op from rpython.rlib import rarithmetic from rpython.translator import unsimplify from rpython.rtyper.lltypesystem import lloperation, lltype from rpython.translator.backendopt.ssa import ( - SSA_to_SSI, DataFlowFamilyBuilder) + SSA_to_SSI, DataFlowFamilyBuilder) def get_graph(arg, translator): if isinstance(arg, Variable): @@ -327,6 +327,62 @@ stack.extend(link.target.exits) seen[link.target] = True +def specialize_exceptions(graph): + for block in list(graph.iterblocks()): + if block.canraise: + op = block.raising_op + if op.canraise != [Exception]: + normal_exit, exc_exit = block.exits + exits = [] + for case in op.canraise: + if case is Exception: + exits.append(exc_exit) + else: + v_exctype = const(case) + v_excvalue = Variable('last_exc_value') + subs = { + exc_exit.last_exception: v_exctype, + exc_exit.last_exc_value: v_excvalue} + link = exc_exit + computed_target = None + seen = set() + while True: + curr_target = link.target + assert link not in seen + seen.add(link) + for v_src, v_target in zip(link.args, curr_target.inputargs): + subs[v_target] = v_src.replace(subs) + for op in curr_target.operations: + new_op = op.replace(subs) + v_const = new_op.constfold() + if v_const is None: + computed_target = link.target + break + subs[op.result] = v_const + if computed_target: + break + if not curr_target.exits: + computed_target = curr_target + break + elif len(curr_target.exits) == 1 or curr_target.canraise: + link = curr_target.exits[0] + else: + v_case = curr_target.exitswitch.replace(subs) + if isinstance(v_case, Constant): + for exit in curr_target.exits: + if exit.exitcase == v_case.value: + link = exit + break + else: + assert False + vars = [v.replace(subs) for v in computed_target.inputargs] + new_link = Link(vars, computed_target, case) + new_link.extravars(v_exctype, v_excvalue) + exits.append(new_link) + exits = [normal_exit] + exits + block.recloseblock(*exits) + + def remove_assertion_errors(graph): """Remove branches that go directly to raising an AssertionError, assuming that AssertionError shouldn't occur at run-time. Note that From noreply at buildbot.pypy.org Tue Mar 17 20:29:31 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Tue, 17 Mar 2015 20:29:31 +0100 (CET) Subject: [pypy-commit] pypy exc-later: simplify guessexception() code Message-ID: <20150317192931.10ABE1C046C@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: exc-later Changeset: r76440:76c1684cab0d Date: 2015-03-13 03:08 +0000 http://bitbucket.org/pypy/pypy/changeset/76c1684cab0d/ Log: simplify guessexception() code diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py --- a/rpython/flowspace/flowcontext.py +++ b/rpython/flowspace/flowcontext.py @@ -74,9 +74,6 @@ curr = prev return recorder - def extravars(self, last_exception=None, last_exc_value=None): - self.last_exception = last_exception - def fixeggblocks(graph): for block in graph.iterblocks(): if isinstance(block, SpamBlock): @@ -124,28 +121,21 @@ def guessexception(self, ctx, *cases): block = self.crnt_block links = [] - for case in [None, Exception]: - if case is not None: - if case is Exception: - last_exc = Variable('last_exception') - else: - last_exc = Constant(case) - last_exc_value = Variable('last_exc_value') - vars = [last_exc, last_exc_value] - vars2 = [Variable(), Variable()] - else: - vars = [] - vars2 = [] - egg = EggBlock(vars2, block, case) - ctx.pendingblocks.append(egg) - link = Link(vars, egg, case) - if case is not None: - link.extravars(last_exception=last_exc, last_exc_value=last_exc_value) - egg.extravars(last_exception=last_exc) - links.append(link) - + normal_block = EggBlock([], block, None) + ctx.pendingblocks.append(normal_block) + normal_exit = Link([], normal_block, exitcase=None) + # + last_exc = Variable('last_exception') + last_exc_value = Variable('last_exc_value') + vars = [last_exc, last_exc_value] + vars2 = [Variable(), Variable()] + exc_block = EggBlock(vars2, block, Exception) + ctx.pendingblocks.append(exc_block) + exc_exit = Link(vars, exc_block, exitcase=Exception) + exc_exit.extravars(last_exception=last_exc, last_exc_value=last_exc_value) + # block.exitswitch = c_last_exception - block.closeblock(*links) + block.closeblock(normal_exit, exc_exit) raise StopFlowing @@ -180,9 +170,6 @@ if outcome is not None: egg = self.nextreplayer.crnt_block w_exc_cls, w_exc_value = egg.inputargs[-2:] - if isinstance(egg.last_exception, Constant): - w_exc_cls = egg.last_exception - assert not isinstance(w_exc_cls.value, list) raise RaiseImplicit(FSException(w_exc_cls, w_exc_value)) # ____________________________________________________________ From noreply at buildbot.pypy.org Tue Mar 17 20:29:32 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Tue, 17 Mar 2015 20:29:32 +0100 (CET) Subject: [pypy-commit] pypy exc-later: Move specialize_exceptions() call to simplify_graph() Message-ID: <20150317192932.3F2F81C046C@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: exc-later Changeset: r76441:62249e9c633f Date: 2015-03-13 04:19 +0000 http://bitbucket.org/pypy/pypy/changeset/62249e9c633f/ Log: Move specialize_exceptions() call to simplify_graph() diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py --- a/rpython/flowspace/flowcontext.py +++ b/rpython/flowspace/flowcontext.py @@ -385,8 +385,6 @@ block = self.pendingblocks.popleft() if not block.dead: self.record_block(block) - from rpython.translator.simplify import specialize_exceptions - specialize_exceptions(graph) def record_block(self, block): self.setstate(block.framestate) diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py --- a/rpython/flowspace/test/test_objspace.py +++ b/rpython/flowspace/test/test_objspace.py @@ -1033,6 +1033,7 @@ f2 = types.FunctionType(new_c, locals(), 'f') graph = self.codetest(f2) + simplify_graph(graph) all_ops = self.all_operations(graph) assert all_ops == {'newlist': 1, 'getattr': 1, 'simple_call': 1, 'iter': 1, 'next': 1} @@ -1239,6 +1240,7 @@ def f(iterable): return [5 for x in iterable] graph = self.codetest(f) + simplify_graph(graph) assert self.all_operations(graph) == {'getattr': 1, 'iter': 1, 'newlist': 1, 'next': 1, 'simple_call': 1} diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py --- a/rpython/translator/simplify.py +++ b/rpython/translator/simplify.py @@ -1116,11 +1116,12 @@ all_passes = [ transform_dead_op_vars, eliminate_empty_blocks, - remove_assertion_errors, remove_identical_vars_SSA, constfold_exitswitch, remove_trivial_links, SSA_to_SSI, + specialize_exceptions, + remove_assertion_errors, coalesce_bool, transform_ovfcheck, simplify_exceptions, @@ -1132,6 +1133,7 @@ """inplace-apply all the existing optimisations to the graph.""" if passes is True: passes = all_passes + #import pdb; pdb.set_trace() for pass_ in passes: pass_(graph) checkgraph(graph) From noreply at buildbot.pypy.org Tue Mar 17 20:29:33 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Tue, 17 Mar 2015 20:29:33 +0100 (CET) Subject: [pypy-commit] pypy exc-later: Reorder simplify_graph() passes and adjust implementations Message-ID: <20150317192933.5F1EF1C046C@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: exc-later Changeset: r76442:a3c18cad3bec Date: 2015-03-13 21:07 +0000 http://bitbucket.org/pypy/pypy/changeset/a3c18cad3bec/ Log: Reorder simplify_graph() passes and adjust implementations diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py --- a/rpython/translator/simplify.py +++ b/rpython/translator/simplify.py @@ -185,9 +185,9 @@ last_op = block.raising_op if last_op.opname == 'getitem': postfx = [] - for exit in block.exits: - if exit.exitcase is IndexError: - postfx.append('idx') + if any(issubclass(IndexError, exit.exitcase) + for exit in block.exits if exit.exitcase): + postfx.append('idx') if postfx: Op = getattr(op, '_'.join(['getitem'] + postfx)) newop = Op(*last_op.args) @@ -332,57 +332,34 @@ if block.canraise: op = block.raising_op if op.canraise != [Exception]: - normal_exit, exc_exit = block.exits + exc_exits = block.exits[1:] exits = [] + has_generic_case = False for case in op.canraise: if case is Exception: - exits.append(exc_exit) + has_generic_case = True + continue + for exit in exc_exits: + if issubclass(case, exit.exitcase): + v_exctype = const(case) + v_excvalue = Variable('last_exc_value') + subs = { + exit.last_exception: v_exctype, + exit.last_exc_value: v_excvalue} + new_link = exit.replace(subs) + new_link.exitcase = case + exits.append(new_link) + break else: - v_exctype = const(case) - v_excvalue = Variable('last_exc_value') - subs = { - exc_exit.last_exception: v_exctype, - exc_exit.last_exc_value: v_excvalue} - link = exc_exit - computed_target = None - seen = set() - while True: - curr_target = link.target - assert link not in seen - seen.add(link) - for v_src, v_target in zip(link.args, curr_target.inputargs): - subs[v_target] = v_src.replace(subs) - for op in curr_target.operations: - new_op = op.replace(subs) - v_const = new_op.constfold() - if v_const is None: - computed_target = link.target - break - subs[op.result] = v_const - if computed_target: - break - if not curr_target.exits: - computed_target = curr_target - break - elif len(curr_target.exits) == 1 or curr_target.canraise: - link = curr_target.exits[0] - else: - v_case = curr_target.exitswitch.replace(subs) - if isinstance(v_case, Constant): - for exit in curr_target.exits: - if exit.exitcase == v_case.value: - link = exit - break - else: - assert False - vars = [v.replace(subs) for v in computed_target.inputargs] - new_link = Link(vars, computed_target, case) - new_link.extravars(v_exctype, v_excvalue) - exits.append(new_link) - exits = [normal_exit] + exits + # ignore the uncaught implicit exception + continue + exits = [block.exits[0]] + exits + if has_generic_case: + exits += exc_exits block.recloseblock(*exits) + def remove_assertion_errors(graph): """Remove branches that go directly to raising an AssertionError, assuming that AssertionError shouldn't occur at run-time. Note that @@ -1120,13 +1097,13 @@ constfold_exitswitch, remove_trivial_links, SSA_to_SSI, - specialize_exceptions, - remove_assertion_errors, coalesce_bool, transform_ovfcheck, simplify_exceptions, + remove_assertion_errors, + specialize_exceptions, + remove_dead_exceptions, transform_xxxitem, - remove_dead_exceptions, ] def simplify_graph(graph, passes=True): # can take a list of passes to apply, True meaning all From noreply at buildbot.pypy.org Tue Mar 17 20:29:34 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Tue, 17 Mar 2015 20:29:34 +0100 (CET) Subject: [pypy-commit] pypy exc-later: Fix handling of implicit exceptions Message-ID: <20150317192934.8901A1C046C@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: exc-later Changeset: r76443:55b6905f0451 Date: 2015-03-14 05:25 +0000 http://bitbucket.org/pypy/pypy/changeset/55b6905f0451/ Log: Fix handling of implicit exceptions diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py --- a/rpython/flowspace/flowcontext.py +++ b/rpython/flowspace/flowcontext.py @@ -335,13 +335,6 @@ self.stack = state.stack[:] self.last_exception = state.last_exception self.blockstack = state.blocklist[:] - self._normalize_raise_signals() - - def _normalize_raise_signals(self): - st = self.stack - for i in range(len(st)): - if isinstance(st[i], RaiseImplicit): - st[i] = Raise(st[i].w_exc) def guessbool(self, w_condition): if isinstance(w_condition, Constant): @@ -677,7 +670,7 @@ return elif isinstance(w_top, FlowSignal): # case of a finally: block - raise w_top + raise w_top.as_explicit() else: # case of an except: block. We popped the exception type self.popvalue() # Now we pop the exception value @@ -1162,6 +1155,9 @@ def __eq__(self, other): return type(other) is type(self) and other.args == self.args + def as_explicit(self): + return self + class Return(FlowSignal): """Signals a 'return' statement. @@ -1223,6 +1219,9 @@ ctx.recorder.crnt_block.closeblock(link) raise StopFlowing + def as_explicit(self): + return Raise(self.w_exc) + class Break(FlowSignal): """Signals a 'break' statement.""" diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py --- a/rpython/flowspace/test/test_objspace.py +++ b/rpython/flowspace/test/test_objspace.py @@ -335,11 +335,8 @@ x = self.codetest(self.implicitException_os_stat) simplify_graph(x) self.show(x) - assert len(x.startblock.exits) == 3 - d = {} - for link in x.startblock.exits: - d[link.exitcase] = True - assert d == {None: True, OSError: True, Exception: True} + exc_cases = [exit.exitcase for exit in x.startblock.exits] + assert exc_cases == [None, OSError] #__________________________________________________________ def reraiseAnythingDicCase(dic): @@ -503,14 +500,8 @@ graph = self.codetest(self.multiple_catch_simple_call) simplify_graph(graph) assert self.all_operations(graph) == {'simple_call': 1} - entrymap = mkentrymap(graph) - links = entrymap[graph.returnblock] - assert len(links) == 3 - assert (dict.fromkeys([link.exitcase for link in links]) == - dict.fromkeys([None, IndexError, OSError])) - links = entrymap[graph.exceptblock] - assert len(links) == 1 - assert links[0].exitcase is Exception + exc_cases = [exit.exitcase for exit in graph.startblock.exits] + assert exc_cases == [None, IndexError, OSError] #__________________________________________________________ def dellocal(): diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py --- a/rpython/translator/simplify.py +++ b/rpython/translator/simplify.py @@ -353,10 +353,11 @@ else: # ignore the uncaught implicit exception continue - exits = [block.exits[0]] + exits if has_generic_case: exits += exc_exits - block.recloseblock(*exits) + if not exits: + block.exitswitch = None + block.recloseblock(block.exits[0], *exits) @@ -1104,6 +1105,7 @@ specialize_exceptions, remove_dead_exceptions, transform_xxxitem, + join_blocks, ] def simplify_graph(graph, passes=True): # can take a list of passes to apply, True meaning all From noreply at buildbot.pypy.org Tue Mar 17 20:29:35 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Tue, 17 Mar 2015 20:29:35 +0100 (CET) Subject: [pypy-commit] pypy exc-later: Move transform_xxitem() from rpython.translator.simplify to rpython.translator.transform Message-ID: <20150317192935.A8BB21C046C@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: exc-later Changeset: r76444:0e069ae6a869 Date: 2015-03-15 02:27 +0000 http://bitbucket.org/pypy/pypy/changeset/0e069ae6a869/ Log: Move transform_xxitem() from rpython.translator.simplify to rpython.translator.transform diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py --- a/rpython/annotator/binaryop.py +++ b/rpython/annotator/binaryop.py @@ -561,7 +561,7 @@ def getitem((lst1, int2)): return lst1.listdef.read_item() - getitem.can_only_throw = [] + getitem.can_only_throw = [IndexError] def getitem_idx((lst1, int2)): return lst1.listdef.read_item() @@ -580,7 +580,7 @@ def getitem((str1, int2)): return SomeChar(no_nul=str1.no_nul) - getitem.can_only_throw = [] + getitem.can_only_throw = [IndexError] def getitem_idx((str1, int2)): return SomeChar(no_nul=str1.no_nul) @@ -592,7 +592,7 @@ class __extend__(pairtype(SomeUnicodeString, SomeInteger)): def getitem((str1, int2)): return SomeUnicodeCodePoint() - getitem.can_only_throw = [] + getitem.can_only_throw = [IndexError] def getitem_idx((str1, int2)): return SomeUnicodeCodePoint() diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py --- a/rpython/translator/simplify.py +++ b/rpython/translator/simplify.py @@ -178,22 +178,6 @@ exits.append(link) block.recloseblock(*(preserve + exits)) -def transform_xxxitem(graph): - # xxx setitem too - for block in graph.iterblocks(): - if block.canraise: - last_op = block.raising_op - if last_op.opname == 'getitem': - postfx = [] - if any(issubclass(IndexError, exit.exitcase) - for exit in block.exits if exit.exitcase): - postfx.append('idx') - if postfx: - Op = getattr(op, '_'.join(['getitem'] + postfx)) - newop = Op(*last_op.args) - newop.result = last_op.result - block.operations[-1] = newop - def remove_dead_exceptions(graph): """Exceptions can be removed if they are unreachable""" @@ -1104,7 +1088,6 @@ remove_assertion_errors, specialize_exceptions, remove_dead_exceptions, - transform_xxxitem, join_blocks, ] diff --git a/rpython/translator/transform.py b/rpython/translator/transform.py --- a/rpython/translator/transform.py +++ b/rpython/translator/transform.py @@ -7,6 +7,7 @@ from rpython.flowspace.model import ( SpaceOperation, Variable, Constant, Link, checkgraph) +from rpython.flowspace.operation import op from rpython.annotator import model as annmodel from rpython.rtyper.lltypesystem import lltype @@ -134,6 +135,23 @@ s_dict.dictdef.generalize_key(self.binding(op.args[1])) +def transform_getitem(ann, blocks): + for block in blocks: + if block.canraise: + last_op = block.raising_op + if last_op.opname == 'getitem': + postfx = [] + if any(issubclass(IndexError, exit.exitcase) + for exit in block.exits if exit.exitcase): + postfx.append('idx') + if postfx: + Op = getattr(op, '_'.join(['getitem'] + postfx)) + newop = Op(*last_op.args) + newop.result = last_op.result + block.operations[-1] = newop + + + def transform_dead_op_vars(self, block_subset): # we redo the same simplification from simplify.py, # to kill dead (never-followed) links, @@ -248,6 +266,7 @@ transform_extend_with_str_slice, transform_extend_with_char_count, transform_list_contains, + transform_getitem, ] def transform_graph(ann, extra_passes=None, block_subset=None): From noreply at buildbot.pypy.org Tue Mar 17 20:29:36 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Tue, 17 Mar 2015 20:29:36 +0100 (CET) Subject: [pypy-commit] pypy exc-later: fix ovfcheck test Message-ID: <20150317192936.BDDDB1C046C@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: exc-later Changeset: r76445:4ef7e35ad896 Date: 2015-03-15 02:49 +0000 http://bitbucket.org/pypy/pypy/changeset/4ef7e35ad896/ Log: fix ovfcheck test diff --git a/rpython/translator/test/test_simplify.py b/rpython/translator/test/test_simplify.py --- a/rpython/translator/test/test_simplify.py +++ b/rpython/translator/test/test_simplify.py @@ -71,9 +71,8 @@ graph, _ = translate(f, [int, int]) assert len(graph.startblock.operations) == 1 assert graph.startblock.operations[0].opname == 'int_floordiv_ovf_zer' - assert len(graph.startblock.exits) == 3 - assert [link.target.operations for link in graph.startblock.exits[1:]] == \ - [(), ()] + assert len(graph.startblock.exits) == 2 + assert graph.startblock.exits[1].target is graph.returnblock def test_remove_direct_call_without_side_effects(): def f(x): From noreply at buildbot.pypy.org Tue Mar 17 20:29:37 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Tue, 17 Mar 2015 20:29:37 +0100 (CET) Subject: [pypy-commit] pypy exc-later: Simplify getitem tests, since flowspace doesn't do anything special with it any more Message-ID: <20150317192937.D271F1C046C@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: exc-later Changeset: r76446:fad5ab2ee8d3 Date: 2015-03-15 03:16 +0000 http://bitbucket.org/pypy/pypy/changeset/fad5ab2ee8d3/ Log: Simplify getitem tests, since flowspace doesn't do anything special with it any more diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py --- a/rpython/flowspace/test/test_objspace.py +++ b/rpython/flowspace/test/test_objspace.py @@ -858,7 +858,7 @@ raise graph = self.codetest(f) simplify_graph(graph) - assert self.all_operations(graph) == {'getitem_idx': 1} + assert self.all_operations(graph) == {'getitem': 1} g = lambda: None def f(c, x): @@ -868,7 +868,7 @@ g() graph = self.codetest(f) simplify_graph(graph) - assert self.all_operations(graph) == {'getitem_idx': 1, + assert self.all_operations(graph) == {'getitem': 1, 'simple_call': 2} def f(c, x): @@ -878,58 +878,12 @@ raise graph = self.codetest(f) simplify_graph(graph) - assert self.all_operations(graph) == {'getitem_idx': 1} + assert self.all_operations(graph) == {'getitem': 1} def f(c, x): try: return c[x] except KeyError: - raise - graph = self.codetest(f) - simplify_graph(graph) - assert self.all_operations(graph) == {'getitem': 1} - - def f(c, x): - try: - return c[x] - except ValueError: - raise - graph = self.codetest(f) - simplify_graph(graph) - assert self.all_operations(graph) == {'getitem': 1} - - def f(c, x): - try: - return c[x] - except Exception: - return -1 - graph = self.codetest(f) - simplify_graph(graph) - self.show(graph) - assert self.all_operations(graph) == {'getitem_idx': 1} - - def f(c, x): - try: - return c[x] - except IndexError: - return -1 - graph = self.codetest(f) - simplify_graph(graph) - assert self.all_operations(graph) == {'getitem_idx': 1} - - def f(c, x): - try: - return c[x] - except KeyError: - return -1 - graph = self.codetest(f) - simplify_graph(graph) - assert self.all_operations(graph) == {'getitem': 1} - - def f(c, x): - try: - return c[x] - except ValueError: return -1 graph = self.codetest(f) simplify_graph(graph) From noreply at buildbot.pypy.org Tue Mar 17 21:15:35 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Tue, 17 Mar 2015 21:15:35 +0100 (CET) Subject: [pypy-commit] pypy exc-later: Kill getitem_idx and all the related machinery Message-ID: <20150317201535.244591C0253@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: exc-later Changeset: r76447:4661b2eab0ed Date: 2015-03-17 20:16 +0000 http://bitbucket.org/pypy/pypy/changeset/4661b2eab0ed/ Log: Kill getitem_idx and all the related machinery diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py --- a/rpython/annotator/binaryop.py +++ b/rpython/annotator/binaryop.py @@ -126,18 +126,6 @@ else: return obj - # checked getitems - - def _getitem_can_only_throw(s_c1, s_o2): - impl = pair(s_c1, s_o2).getitem - return read_can_only_throw(impl, s_c1, s_o2) - - def getitem_idx((s_c1, s_o2)): - impl = pair(s_c1, s_o2).getitem - return impl() - getitem_idx.can_only_throw = _getitem_can_only_throw - - class __extend__(pairtype(SomeType, SomeType), pairtype(SomeType, SomeConstantType), @@ -563,10 +551,6 @@ return lst1.listdef.read_item() getitem.can_only_throw = [IndexError] - def getitem_idx((lst1, int2)): - return lst1.listdef.read_item() - getitem_idx.can_only_throw = [IndexError] - def setitem((lst1, int2), s_value): lst1.listdef.mutate() lst1.listdef.generalize(s_value) @@ -582,10 +566,6 @@ return SomeChar(no_nul=str1.no_nul) getitem.can_only_throw = [IndexError] - def getitem_idx((str1, int2)): - return SomeChar(no_nul=str1.no_nul) - getitem_idx.can_only_throw = [IndexError] - def mul((str1, int2)): # xxx do we want to support this return SomeString(no_nul=str1.no_nul) @@ -594,10 +574,6 @@ return SomeUnicodeCodePoint() getitem.can_only_throw = [IndexError] - def getitem_idx((str1, int2)): - return SomeUnicodeCodePoint() - getitem_idx.can_only_throw = [IndexError] - def mul((str1, int2)): # xxx do we want to support this return SomeUnicodeString() diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py --- a/rpython/flowspace/operation.py +++ b/rpython/flowspace/operation.py @@ -421,7 +421,6 @@ add_operator('setattr', 3, dispatch=1, pyfunc=setattr) add_operator('delattr', 2, dispatch=1, pyfunc=delattr) add_operator('getitem', 2, dispatch=2, pure=True) -add_operator('getitem_idx', 2, dispatch=2, pure=True) add_operator('setitem', 3, dispatch=2) add_operator('delitem', 2, dispatch=2) add_operator('getslice', 3, dispatch=1, pyfunc=do_getslice, pure=True) @@ -683,7 +682,6 @@ # allows the annotator to be more precise, see test_reraiseAnything/KeyError in # the annotator tests op.getitem.canraise = [IndexError, KeyError, Exception] -op.getitem_idx.canraise = [IndexError, KeyError, Exception] op.setitem.canraise = [IndexError, KeyError, Exception] op.delitem.canraise = [IndexError, KeyError, Exception] op.contains.canraise = [Exception] # from an r_dict diff --git a/rpython/rtyper/rlist.py b/rpython/rtyper/rlist.py --- a/rpython/rtyper/rlist.py +++ b/rpython/rtyper/rlist.py @@ -246,9 +246,9 @@ class __extend__(pairtype(AbstractBaseListRepr, IntegerRepr)): - def rtype_getitem((r_lst, r_int), hop, checkidx=False): + def rtype_getitem((r_lst, r_int), hop): v_lst, v_index = hop.inputargs(r_lst, Signed) - if checkidx: + if hop.has_implicit_exception(IndexError): hop.exception_is_here() spec = dum_checkidx else: @@ -268,9 +268,6 @@ v_res = hop.gendirectcall(llfn, c_func_marker, c_basegetitem, v_lst, v_index) return r_lst.recast(hop.llops, v_res) - def rtype_getitem_idx((r_lst, r_int), hop): - return pair(r_lst, r_int).rtype_getitem(hop, checkidx=True) - def rtype_setitem((r_lst, r_int), hop): if hop.has_implicit_exception(IndexError): spec = dum_checkidx diff --git a/rpython/rtyper/rmodel.py b/rpython/rtyper/rmodel.py --- a/rpython/rtyper/rmodel.py +++ b/rpython/rtyper/rmodel.py @@ -283,12 +283,6 @@ return inputconst(Bool, hop.s_result.const) return hop.rtyper.type_system.generic_is(robj1, robj2, hop) - # default implementation for checked getitems - - def rtype_getitem_idx((r_c1, r_o1), hop): - return pair(r_c1, r_o1).rtype_getitem(hop) - - # ____________________________________________________________ diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py --- a/rpython/rtyper/rstr.py +++ b/rpython/rtyper/rstr.py @@ -561,28 +561,23 @@ class __extend__(pairtype(AbstractStringRepr, IntegerRepr)): - def rtype_getitem((r_str, r_int), hop, checkidx=False): + def rtype_getitem((r_str, r_int), hop): string_repr = r_str.repr v_str, v_index = hop.inputargs(string_repr, Signed) - if checkidx: + if hop.has_implicit_exception(IndexError): if hop.args_s[1].nonneg: llfn = r_str.ll.ll_stritem_nonneg_checked else: llfn = r_str.ll.ll_stritem_checked + hop.exception_is_here() else: if hop.args_s[1].nonneg: llfn = r_str.ll.ll_stritem_nonneg else: llfn = r_str.ll.ll_stritem - if checkidx: - hop.exception_is_here() - else: hop.exception_cannot_occur() return hop.gendirectcall(llfn, v_str, v_index) - def rtype_getitem_idx((r_str, r_int), hop): - return pair(r_str, r_int).rtype_getitem(hop, checkidx=True) - def rtype_mul((r_str, r_int), hop): str_repr = r_str.repr v_str, v_int = hop.inputargs(str_repr, Signed) diff --git a/rpython/translator/transform.py b/rpython/translator/transform.py --- a/rpython/translator/transform.py +++ b/rpython/translator/transform.py @@ -135,23 +135,6 @@ s_dict.dictdef.generalize_key(self.binding(op.args[1])) -def transform_getitem(ann, blocks): - for block in blocks: - if block.canraise: - last_op = block.raising_op - if last_op.opname == 'getitem': - postfx = [] - if any(issubclass(IndexError, exit.exitcase) - for exit in block.exits if exit.exitcase): - postfx.append('idx') - if postfx: - Op = getattr(op, '_'.join(['getitem'] + postfx)) - newop = Op(*last_op.args) - newop.result = last_op.result - block.operations[-1] = newop - - - def transform_dead_op_vars(self, block_subset): # we redo the same simplification from simplify.py, # to kill dead (never-followed) links, @@ -266,7 +249,6 @@ transform_extend_with_str_slice, transform_extend_with_char_count, transform_list_contains, - transform_getitem, ] def transform_graph(ann, extra_passes=None, block_subset=None): From noreply at buildbot.pypy.org Wed Mar 18 08:55:43 2015 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 18 Mar 2015 08:55:43 +0100 (CET) Subject: [pypy-commit] pypy dtrace-support: finish dtrace for freebsd Message-ID: <20150318075543.D236B1C0173@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: dtrace-support Changeset: r76448:b4cac385ef48 Date: 2015-03-18 08:55 +0100 http://bitbucket.org/pypy/pypy/changeset/b4cac385ef48/ Log: finish dtrace for freebsd diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py --- a/rpython/translator/c/funcgen.py +++ b/rpython/translator/c/funcgen.py @@ -790,7 +790,7 @@ name_for_op = 'START' else: name_for_op = 'END' - prefix = 'PYPY_PROBES_%s_%s();' % ( + prefix = 'RPYTHON_%s_%s();' % ( val.replace('-', '_').upper(), name_for_op) else: prefix = '' diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py --- a/rpython/translator/c/genc.py +++ b/rpython/translator/c/genc.py @@ -253,9 +253,9 @@ return cfile def _generate_dtrace_probe_file(self, debug_nodes): - name = self.targetdir.join('pypy.d') + name = self.targetdir.join('rpython.d') f = name.open('w') - f.write('provider pypy_probes {\n') + f.write('provider rpython {\n') for debug_node in debug_nodes: debug_node = debug_node.replace('-', '_') f.write(' probe %s__start();\n' % debug_node) @@ -263,7 +263,7 @@ f.write('};\n') f.close() returncode, stdout, stderr = runsubprocess.run_subprocess( - 'dtrace', ['-o', str(self.targetdir.join('pypy_probes.h')), + 'dtrace', ['-o', str(self.targetdir.join('rpython_dtrace.h')), '-h', '-s', str(name)]) if returncode: raise Exception("Dtrace exploded: %s" % stderr) @@ -488,18 +488,16 @@ mk.definition('DEBUGFLAGS', '-O1 -g -fPIC') else: mk.definition('DEBUGFLAGS', '-O1 -g') - if (self.config.translation.dtrace and - not self.translator.platform.name.startswith('darwin')): - # right now dtrace is incompatible with asmgcc on all platforms - # I think - assert self.config.translation.gcrootfinder != 'asmgcc' - mk.definition('OBJECTS1', '$(subst .c,.o,$(SOURCES))') - mk.definition('OBJECTS', '$(OBJECTS1) dtrace_marker') - mk.rule('dtrace_marker', '', 'dtrace -G -s pypy.d $(OBJECTS)') if self.translator.platform.name == 'msvc': mk.rule('debug_target', 'debugmode_$(DEFAULT_TARGET)', 'rem') else: mk.rule('debug_target', '$(DEFAULT_TARGET)', '#') + if (self.config.translation.dtrace and + not self.translator.platform.name.startswith('darwin')): + assert self.config.translation.gcrootfinder != 'asmgcc' + mk.definition('OBJECTS1', '$(subst .c,.o,$(SOURCES))') + mk.definition('OBJECTS', '$(OBJECTS1) rpython_dtrace.o') + mk.rule('rpython_dtrace.o', 'rpython.d $(OBJECTS1)', 'dtrace -G -s rpython.d -o rpython_dtrace.o $(OBJECTS1)') mk.write() #self.translator.platform, # , @@ -854,7 +852,7 @@ eci.write_c_header(fi) print >> fi, '#include "src/g_prerequisite.h"' if dtrace: - print >> fi, '#include "pypy_probes.h"' + print >> fi, '#include "rpython_dtrace.h"' fi.write('#endif /* _PY_COMMON_HEADER_H*/\n') fi.close() diff --git a/rpython/translator/c/test/test_dtrace.py b/rpython/translator/c/test/test_dtrace.py --- a/rpython/translator/c/test/test_dtrace.py +++ b/rpython/translator/c/test/test_dtrace.py @@ -1,4 +1,5 @@ +import subprocess from rpython.translator.c.test.test_standalone import StandaloneTests from rpython.rlib.debug import debug_start, debug_stop from rpython.config.translationoption import get_combined_translation_config @@ -16,4 +17,10 @@ return 0 _, cbuilder = self.compile(f) - cbuilder.cmdexec('') + exe = cbuilder.executable_name + p = subprocess.Popen(['dtrace', '-n', ':' + exe.basename + '::', + '-c', str(exe)], stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out = p.stdout.read() + assert 'pypy_g_f:x-start' in out + assert 'pypy_g_f:x-end' in out diff --git a/rpython/translator/platform/freebsd.py b/rpython/translator/platform/freebsd.py --- a/rpython/translator/platform/freebsd.py +++ b/rpython/translator/platform/freebsd.py @@ -7,7 +7,7 @@ name = "freebsd" link_flags = tuple( - ['-pthread'] + + ['-pthread', '-lelf'] + os.environ.get('LDFLAGS', '').split()) cflags = tuple( ['-O3', '-pthread', '-fomit-frame-pointer'] + diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py --- a/rpython/translator/platform/posix.py +++ b/rpython/translator/platform/posix.py @@ -172,9 +172,11 @@ for args in definitions: m.definition(*args) + target_rule = ('$(CC_LINK) $(LDFLAGSEXTRA) -o $@ $(OBJECTS)' + ' $(LIBDIRS) $(LIBS) $(LINKFILES) $(LDFLAGS)') rules = [ ('all', '$(DEFAULT_TARGET)', []), - ('$(TARGET)', '$(OBJECTS)', '$(CC_LINK) $(LDFLAGSEXTRA) -o $@ $(OBJECTS) $(LIBDIRS) $(LIBS) $(LINKFILES) $(LDFLAGS)'), + ('$(TARGET)', '$(OBJECTS)', target_rule), ('%.o', '%.c', '$(CC) $(CFLAGS) $(CFLAGSEXTRA) -o $@ -c $< $(INCLUDEDIRS)'), ('%.o', '%.cxx', '$(CXX) $(CFLAGS) $(CFLAGSEXTRA) -o $@ -c $< $(INCLUDEDIRS)'), ] From noreply at buildbot.pypy.org Wed Mar 18 09:52:35 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 18 Mar 2015 09:52:35 +0100 (CET) Subject: [pypy-commit] stmgc default: Temporary fix? Add stm_wait_for_current_inevitable_transaction() Message-ID: <20150318085235.DC03A1C0221@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1736:a4fc9f31f925 Date: 2015-03-18 09:53 +0100 http://bitbucket.org/pypy/stmgc/changeset/a4fc9f31f925/ Log: Temporary fix? Add stm_wait_for_current_inevitable_transaction() diff --git a/c7/stm/sync.c b/c7/stm/sync.c --- a/c7/stm/sync.c +++ b/c7/stm/sync.c @@ -141,6 +141,27 @@ } } +void stm_wait_for_current_inevitable_transaction(void) +{ + long i; + for (i = 1; i <= NB_SEGMENTS; i++) { + struct stm_priv_segment_info_s *other_pseg = get_priv_segment(i); + if (other_pseg->transaction_state == TS_INEVITABLE) { + /* Asynchronously found out that there is an inevitable + transaction running. Wait until it signals "done", + then return (even if another inev transaction started + in the meantime). + */ + s_mutex_lock(); + if (other_pseg->transaction_state == TS_INEVITABLE) + cond_wait(C_INEVITABLE); + /*else: this inev transaction finished just now */ + s_mutex_unlock(); + break; + } + } +} + static bool acquire_thread_segment(stm_thread_local_t *tl) { /* This function acquires a segment for the currently running thread, diff --git a/c7/stmgc.h b/c7/stmgc.h --- a/c7/stmgc.h +++ b/c7/stmgc.h @@ -332,6 +332,10 @@ void stm_start_inevitable_transaction(stm_thread_local_t *tl); void stm_commit_transaction(void); +/* Temporary fix? Call this outside a transaction. If there is an + inevitable transaction running somewhere else, wait until it finishes. */ +void stm_wait_for_current_inevitable_transaction(void); + /* Abort the currently running transaction. This function never returns: it jumps back to the stm_start_transaction(). */ void stm_abort_transaction(void) __attribute__((noreturn)); From noreply at buildbot.pypy.org Wed Mar 18 10:14:42 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 18 Mar 2015 10:14:42 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Kill outdated ops Message-ID: <20150318091442.9F3C51C00FA@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76449:a7e972cd321e Date: 2015-03-18 09:56 +0100 http://bitbucket.org/pypy/pypy/changeset/a7e972cd321e/ Log: Kill outdated ops diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py --- a/rpython/rtyper/lltypesystem/opimpl.py +++ b/rpython/rtyper/lltypesystem/opimpl.py @@ -678,12 +678,6 @@ from rpython.rlib.rtimer import read_timestamp return read_timestamp() -def op_stm_start_transaction(): - pass - -def op_stm_stop_transaction(): - pass - def op_debug_fatalerror(ll_msg): from rpython.rtyper.lltypesystem import lltype, rstr from rpython.rtyper.llinterp import LLFatalError From noreply at buildbot.pypy.org Wed Mar 18 10:14:43 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 18 Mar 2015 10:14:43 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: import stmgc/a4fc9f31f925 Message-ID: <20150318091443.F00E91C00FA@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76450:3a58a18e3204 Date: 2015-03-18 09:57 +0100 http://bitbucket.org/pypy/pypy/changeset/3a58a18e3204/ Log: import stmgc/a4fc9f31f925 diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision --- a/rpython/translator/stm/src_stm/revision +++ b/rpython/translator/stm/src_stm/revision @@ -1,1 +1,1 @@ -b26fe28d6f2b +a4fc9f31f925 diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c --- a/rpython/translator/stm/src_stm/stm/core.c +++ b/rpython/translator/stm/src_stm/stm/core.c @@ -833,7 +833,6 @@ void stm_commit_transaction(void) { - restart_all: exec_local_finalizers(); assert(!_has_mutex()); @@ -853,11 +852,6 @@ Important: we should not call cond_wait() in the meantime. */ synchronize_all_threads(STOP_OTHERS_UNTIL_MUTEX_UNLOCK); - if (any_local_finalizers()) { - s_mutex_unlock(); - goto restart_all; - } - /* detect conflicts */ if (detect_write_read_conflicts()) goto restart; diff --git a/rpython/translator/stm/src_stm/stm/extra.c b/rpython/translator/stm/src_stm/stm/extra.c --- a/rpython/translator/stm/src_stm/stm/extra.c +++ b/rpython/translator/stm/src_stm/stm/extra.c @@ -6,15 +6,24 @@ static long register_callbacks(stm_thread_local_t *tl, void *key, void callback(void *), long index) { - if (!_stm_in_transaction(tl)) { - /* check that the current thread-local is really running a + dprintf(("register_callbacks: tl=%p key=%p callback=%p index=%ld\n", + tl, key, callback, index)); + if (tl->associated_segment_num == -1) { + /* check that the provided thread-local is really running a transaction, and do nothing otherwise. */ + dprintf((" NOT IN TRANSACTION\n")); return -1; } - - if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) { + /* The tl was only here to check that. We're really using + STM_PSEGMENT below, which is often but not always the + segment corresponding to the tl. One case where it's not + the case is if this gets called from stmcb_light_finalizer() + from abort_finalizers() from major collections or contention. + */ + if (STM_PSEGMENT->transaction_state != TS_REGULAR) { /* ignore callbacks if we're in an inevitable transaction - (which cannot abort) */ + (which cannot abort) or no transaction at all in this segment */ + dprintf((" STATE = %d\n", (int)STM_PSEGMENT->transaction_state)); return -1; } @@ -23,10 +32,13 @@ if (callback == NULL) { /* double-unregistering works, but return 0 */ - return tree_delete_item(callbacks, (uintptr_t)key); + long res = tree_delete_item(callbacks, (uintptr_t)key); + dprintf((" DELETED %ld\n", res)); + return res; } else { /* double-registering the same key will crash */ + dprintf((" INSERTING\n")); tree_insert(callbacks, (uintptr_t)key, (uintptr_t)callback); return 1; } @@ -39,6 +51,7 @@ if (result < 0 && callback != NULL) { /* no regular transaction running, invoke the callback immediately */ + dprintf(("stm_call_on_commit calls now: %p(%p)\n", callback, key)); callback(key); } return result; @@ -72,8 +85,11 @@ assert(key != NULL); assert(callback != NULL); - /* The callback may call stm_call_on_abort(key, NULL). It is ignored, - because 'callbacks_on_commit_and_abort' was cleared already. */ + /* The callback may call stm_call_on_abort(key, NULL) + (so with callback==NULL). It is ignored, because + 'callbacks_on_commit_and_abort' was cleared already. */ + dprintf(("invoke_and_clear_user_callbacks(%ld): %p(%p)\n", + index, callback, key)); callback(key); } TREE_LOOP_END; diff --git a/rpython/translator/stm/src_stm/stm/forksupport.c b/rpython/translator/stm/src_stm/stm/forksupport.c --- a/rpython/translator/stm/src_stm/stm/forksupport.c +++ b/rpython/translator/stm/src_stm/stm/forksupport.c @@ -220,9 +220,8 @@ if (endpagenum == NB_PAGES) break; /* done */ pagenum = (uninitialized_page_stop - stm_object_pages) / 4096UL; + pagenum--; /* contains data from largemalloc */ endpagenum = NB_PAGES; - if (endpagenum == NB_PAGES) - break; /* done */ } struct page_shared_s ps = pages_privatized[pagenum - PAGE_FLAG_START]; diff --git a/rpython/translator/stm/src_stm/stm/gcpage.c b/rpython/translator/stm/src_stm/stm/gcpage.c --- a/rpython/translator/stm/src_stm/stm/gcpage.c +++ b/rpython/translator/stm/src_stm/stm/gcpage.c @@ -418,12 +418,12 @@ stm_thread_local_t *tl = stm_all_thread_locals; do { - /* If 'tl' is currently running, its 'associated_segment_num' + /* If 'tl' is currently running, its 'last_associated_segment_num' field is the segment number that contains the correct version of its overflowed objects. If not, then the field is still some correct segment number, and it doesn't matter which one we pick. */ - char *segment_base = get_segment_base(tl->associated_segment_num); + char *segment_base = get_segment_base(tl->last_associated_segment_num); struct stm_shadowentry_s *current = tl->shadowstack; struct stm_shadowentry_s *base = tl->shadowstack_base; diff --git a/rpython/translator/stm/src_stm/stm/marker.c b/rpython/translator/stm/src_stm/stm/marker.c --- a/rpython/translator/stm/src_stm/stm/marker.c +++ b/rpython/translator/stm/src_stm/stm/marker.c @@ -58,6 +58,7 @@ */ long i; int in_segment_num = out_marker->tl->associated_segment_num; + assert(in_segment_num >= 1); struct stm_priv_segment_info_s *pseg = get_priv_segment(in_segment_num); struct list_s *mlst = pseg->modified_old_objects; struct list_s *mlstm = pseg->modified_old_objects_markers; diff --git a/rpython/translator/stm/src_stm/stm/nursery.c b/rpython/translator/stm/src_stm/stm/nursery.c --- a/rpython/translator/stm/src_stm/stm/nursery.c +++ b/rpython/translator/stm/src_stm/stm/nursery.c @@ -263,9 +263,9 @@ OPT_ASSERT(write_locks[first_card_index] <= NB_SEGMENTS || write_locks[first_card_index] == 255); /* see gcpage.c */ - dprintf(("mark cards of %p, size %lu with %d, all: %d\n", + /*dprintf(("mark cards of %p, size %lu with %d, all: %d\n", obj, size, mark_value, mark_all)); - dprintf(("obj has %lu cards\n", last_card_index)); + dprintf(("obj has %lu cards\n", last_card_index));*/ while (card_index <= last_card_index) { uintptr_t card_lock_idx = first_card_index + card_index; diff --git a/rpython/translator/stm/src_stm/stm/pages.c b/rpython/translator/stm/src_stm/stm/pages.c --- a/rpython/translator/stm/src_stm/stm/pages.c +++ b/rpython/translator/stm/src_stm/stm/pages.c @@ -63,12 +63,12 @@ static void d_remap_file_pages(char *addr, size_t size, ssize_t pgoff) { - dprintf(("remap_file_pages: 0x%lx bytes: (seg%ld %p) --> (seg%ld %p)\n", + /*dprintf(("remap_file_pages: 0x%lx bytes: (seg%ld %p) --> (seg%ld %p)\n", (long)size, (long)((addr - stm_object_pages) / 4096UL) / NB_PAGES, (void *)((addr - stm_object_pages) % (4096UL * NB_PAGES)), (long)pgoff / NB_PAGES, - (void *)((pgoff % NB_PAGES) * 4096UL))); + (void *)((pgoff % NB_PAGES) * 4096UL)));*/ assert(size % 4096 == 0); assert(size <= TOTAL_MEMORY); assert(((uintptr_t)addr) % 4096 == 0); diff --git a/rpython/translator/stm/src_stm/stm/setup.c b/rpython/translator/stm/src_stm/stm/setup.c --- a/rpython/translator/stm/src_stm/stm/setup.c +++ b/rpython/translator/stm/src_stm/stm/setup.c @@ -255,7 +255,7 @@ tl->prev = stm_all_thread_locals->prev; stm_all_thread_locals->prev->next = tl; stm_all_thread_locals->prev = tl; - num = tl->prev->associated_segment_num; + num = tl->prev->last_associated_segment_num; } tl->thread_local_obj = NULL; @@ -263,7 +263,8 @@ assign the same number to all of them and they would get their own numbers automatically. */ num = (num % NB_SEGMENTS) + 1; - tl->associated_segment_num = num; + tl->associated_segment_num = -1; + tl->last_associated_segment_num = num; tl->thread_local_counter = ++thread_local_counters; *_get_cpth(tl) = pthread_self(); _init_shadow_stack(tl); diff --git a/rpython/translator/stm/src_stm/stm/sync.c b/rpython/translator/stm/src_stm/stm/sync.c --- a/rpython/translator/stm/src_stm/stm/sync.c +++ b/rpython/translator/stm/src_stm/stm/sync.c @@ -141,6 +141,27 @@ } } +void stm_wait_for_current_inevitable_transaction(void) +{ + long i; + for (i = 1; i <= NB_SEGMENTS; i++) { + struct stm_priv_segment_info_s *other_pseg = get_priv_segment(i); + if (other_pseg->transaction_state == TS_INEVITABLE) { + /* Asynchronously found out that there is an inevitable + transaction running. Wait until it signals "done", + then return (even if another inev transaction started + in the meantime). + */ + s_mutex_lock(); + if (other_pseg->transaction_state == TS_INEVITABLE) + cond_wait(C_INEVITABLE); + /*else: this inev transaction finished just now */ + s_mutex_unlock(); + break; + } + } +} + static bool acquire_thread_segment(stm_thread_local_t *tl) { /* This function acquires a segment for the currently running thread, @@ -148,7 +169,7 @@ assert(_has_mutex()); assert(_is_tl_registered(tl)); - int num = tl->associated_segment_num; + int num = tl->last_associated_segment_num; if (sync_ctl.in_use1[num - 1] == 0) { /* fast-path: we can get the same segment number than the one we had before. The value stored in GS is still valid. */ @@ -167,8 +188,9 @@ num = (num % NB_SEGMENTS) + 1; if (sync_ctl.in_use1[num - 1] == 0) { /* we're getting 'num', a different number. */ - dprintf(("acquired different segment: %d->%d\n", tl->associated_segment_num, num)); - tl->associated_segment_num = num; + dprintf(("acquired different segment: %d->%d\n", + tl->last_associated_segment_num, num)); + tl->last_associated_segment_num = num; set_gs_register(get_segment_base(num)); goto got_num; } @@ -186,6 +208,7 @@ sync_ctl.in_use1[num - 1] = 1; assert(STM_SEGMENT->segment_num == num); assert(STM_SEGMENT->running_thread == NULL); + tl->associated_segment_num = tl->last_associated_segment_num; STM_SEGMENT->running_thread = tl; return true; } @@ -204,10 +227,12 @@ } assert(STM_SEGMENT->running_thread == tl); + assert(tl->associated_segment_num == tl->last_associated_segment_num); + tl->associated_segment_num = -1; STM_SEGMENT->running_thread = NULL; - assert(sync_ctl.in_use1[tl->associated_segment_num - 1] == 1); - sync_ctl.in_use1[tl->associated_segment_num - 1] = 0; + assert(sync_ctl.in_use1[tl->last_associated_segment_num - 1] == 1); + sync_ctl.in_use1[tl->last_associated_segment_num - 1] = 0; } __attribute__((unused)) @@ -218,9 +243,16 @@ bool _stm_in_transaction(stm_thread_local_t *tl) { - int num = tl->associated_segment_num; - assert(1 <= num && num <= NB_SEGMENTS); - return get_segment(num)->running_thread == tl; + if (tl->associated_segment_num == -1) { + return false; + } + else { + int num = tl->associated_segment_num; + OPT_ASSERT(1 <= num && num <= NB_SEGMENTS); + OPT_ASSERT(num == tl->last_associated_segment_num); + OPT_ASSERT(get_segment(num)->running_thread == tl); + return true; + } } void _stm_test_switch(stm_thread_local_t *tl) diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h --- a/rpython/translator/stm/src_stm/stmgc.h +++ b/rpython/translator/stm/src_stm/stmgc.h @@ -69,6 +69,7 @@ long last_abort__bytes_in_nursery; /* the next fields are handled internally by the library */ int associated_segment_num; + int last_associated_segment_num; int thread_local_counter; struct stm_thread_local_s *prev, *next; void *creating_pthread[2]; @@ -331,6 +332,10 @@ void stm_start_inevitable_transaction(stm_thread_local_t *tl); void stm_commit_transaction(void); +/* Temporary fix? Call this outside a transaction. If there is an + inevitable transaction running somewhere else, wait until it finishes. */ +void stm_wait_for_current_inevitable_transaction(void); + /* Abort the currently running transaction. This function never returns: it jumps back to the stm_start_transaction(). */ void stm_abort_transaction(void) __attribute__((noreturn)); From noreply at buildbot.pypy.org Wed Mar 18 10:14:45 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 18 Mar 2015 10:14:45 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Insert a call to stm_wait_for_current_inevitable_transaction() Message-ID: <20150318091445.10E5C1C00FA@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76451:cc6db35b39d6 Date: 2015-03-18 10:08 +0100 http://bitbucket.org/pypy/pypy/changeset/cc6db35b39d6/ Log: Insert a call to stm_wait_for_current_inevitable_transaction() diff --git a/rpython/translator/stm/src_stm/stmgcintf.c b/rpython/translator/stm/src_stm/stmgcintf.c --- a/rpython/translator/stm/src_stm/stmgcintf.c +++ b/rpython/translator/stm/src_stm/stmgcintf.c @@ -167,7 +167,7 @@ pypy_stm_nursery_low_fill_mark = _stm_nursery_start + limit; } -long _pypy_stm_start_transaction(void) +void _pypy_stm_start_transaction(void) { pypy_stm_nursery_low_fill_mark = 1; /* will be set to a correct value below */ long counter = stm_start_transaction(&stm_thread_local); @@ -175,8 +175,14 @@ _pypy_stm_initialize_nursery_low_fill_mark(counter); pypy_stm_ready_atomic = 1; /* reset after abort */ +} - return counter; +void _pypy_stm_start_transaction_save_errno_wait_inev(void) +{ + int e = errno; + stm_wait_for_current_inevitable_transaction(); + _pypy_stm_start_transaction(); + errno = e; } void pypy_stm_transaction_break(void) diff --git a/rpython/translator/stm/src_stm/stmgcintf.h b/rpython/translator/stm/src_stm/stmgcintf.h --- a/rpython/translator/stm/src_stm/stmgcintf.h +++ b/rpython/translator/stm/src_stm/stmgcintf.h @@ -26,7 +26,8 @@ void _pypy_stm_initialize_nursery_low_fill_mark(long v_counter); void _pypy_stm_inev_state(void); -long _pypy_stm_start_transaction(void); +void _pypy_stm_start_transaction(void); +void _pypy_stm_start_transaction_save_errno_wait_inev(void); void _pypy_stm_become_inevitable(const char *); @@ -57,11 +58,8 @@ errno = e; } static inline void pypy_stm_start_if_not_atomic(void) { - if (pypy_stm_ready_atomic == 1) { - int e = errno; - _pypy_stm_start_transaction(); - errno = e; - } + if (pypy_stm_ready_atomic == 1) + _pypy_stm_start_transaction_save_errno_wait_inev(); } static inline void pypy_stm_start_inevitable_if_not_atomic(void) { if (pypy_stm_ready_atomic == 1) { From noreply at buildbot.pypy.org Wed Mar 18 10:14:46 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 18 Mar 2015 10:14:46 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Backout 7b2e2b3f44f0: it should now work in general (needs to be tested) Message-ID: <20150318091446.3AD361C00FA@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76452:e092e933aae7 Date: 2015-03-18 10:09 +0100 http://bitbucket.org/pypy/pypy/changeset/e092e933aae7/ Log: Backout 7b2e2b3f44f0: it should now work in general (needs to be tested) diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py --- a/rpython/rlib/rthread.py +++ b/rpython/rlib/rthread.py @@ -61,9 +61,6 @@ c_thread_releaselock = llexternal('RPyThreadReleaseLock', [TLOCKP], lltype.Signed, _nowrapper=True) # *don't* release the GIL -c_thread_releaselock_GIL = llexternal('RPyThreadReleaseLock', [TLOCKP], - lltype.Signed, - releasegil=True) # another set of functions, this time in versions that don't cause the # GIL to be released. Used to be there to handle the GIL lock itself, @@ -161,11 +158,7 @@ return res def release(self): - if rgc.stm_is_enabled(): - func = c_thread_releaselock_GIL # XXX temporary workaround! - else: - func = c_thread_releaselock - if func(self._lock) != 0: + if c_thread_releaselock(self._lock) != 0: raise error("the lock was not previously acquired") def __del__(self): From noreply at buildbot.pypy.org Wed Mar 18 11:29:03 2015 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 18 Mar 2015 11:29:03 +0100 (CET) Subject: [pypy-commit] pypy release-2.5.x: merge default into branch Message-ID: <20150318102903.50C731C1F23@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: release-2.5.x Changeset: r76453:e3d046c43451 Date: 2015-03-18 12:28 +0200 http://bitbucket.org/pypy/pypy/changeset/e3d046c43451/ Log: merge default into branch diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -11,3 +11,4 @@ 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1 10f1b29a2bd21f837090286174a9ca030b8680b2 release-2.5.0 +8e24dac0b8e2db30d46d59f2c4daa3d4aaab7861 release-2.5.1 diff --git a/pypy/doc/stm.rst b/pypy/doc/stm.rst --- a/pypy/doc/stm.rst +++ b/pypy/doc/stm.rst @@ -46,13 +46,14 @@ multiple cores. * ``pypy-stm`` provides (but does not impose) a special API to the - user in the pure Python module `transaction`_. This module is based - on the lower-level module `pypystm`_, but also provides some + user in the pure Python module ``transaction``. This module is based + on the lower-level module ``pypystm``, but also provides some compatibily with non-STM PyPy's or CPython's. * Building on top of the way the GIL is removed, we will talk - about `Atomic sections, Transactions, etc.: a better way to write - parallel programs`_. + about `How to write multithreaded programs: the 10'000-feet view`_ + and `transaction.TransactionQueue`_. + Getting Started @@ -89,7 +90,7 @@ Current status (stmgc-c7) ------------------------- -* It seems to work fine, without crashing any more. Please `report +* **NEW:** It seems to work fine, without crashing any more. Please `report any crash`_ you find (or other bugs). * It runs with an overhead as low as 20% on examples like "richards". @@ -97,33 +98,47 @@ 2x for "translate.py"-- which we are still trying to understand. One suspect is our partial GC implementation, see below. +* **NEW:** the ``PYPYSTM`` environment variable and the + ``pypy/stm/print_stm_log.py`` script let you know exactly which + "conflicts" occurred. This is described in the section + `transaction.TransactionQueue`_ below. + +* **NEW:** special transaction-friendly APIs (like ``stmdict``), + described in the section `transaction.TransactionQueue`_ below. The + old API changed again, mostly moving to different modules. Sorry + about that. I feel it's a better idea to change the API early + instead of being stuck with a bad one later... + * Currently limited to 1.5 GB of RAM (this is just a parameter in `core.h`__ -- theoretically. In practice, increase it too much and clang crashes again). Memory overflows are not correctly handled; they cause segfaults. -* The JIT warm-up time improved recently but is still bad. In order to - produce machine code, the JIT needs to enter a special single-threaded - mode for now. This means that you will get bad performance results if - your program doesn't run for several seconds, where *several* can mean - *many.* When trying benchmarks, be sure to check that you have - reached the warmed state, i.e. the performance is not improving any - more. This should be clear from the fact that as long as it's - producing more machine code, ``pypy-stm`` will run on a single core. +* **NEW:** The JIT warm-up time improved again, but is still + relatively large. In order to produce machine code, the JIT needs + to enter "inevitable" mode. This means that you will get bad + performance results if your program doesn't run for several seconds, + where *several* can mean *many.* When trying benchmarks, be sure to + check that you have reached the warmed state, i.e. the performance + is not improving any more. * The GC is new; although clearly inspired by PyPy's regular GC, it misses a number of optimizations for now. Programs allocating large numbers of small objects that don't immediately die (surely a common - situation) suffer from these missing optimizations. + situation) suffer from these missing optimizations. (The bleeding + edge ``stmgc-c8`` is better at that.) * Weakrefs might appear to work a bit strangely for now, sometimes staying alive throught ``gc.collect()``, or even dying but then - un-dying for a short time before dying again. + un-dying for a short time before dying again. A similar problem can + show up occasionally elsewhere with accesses to some external + resources, where the (apparent) serialized order doesn't match the + underlying (multithreading) order. These are bugs (partially fixed + already in ``stmgc-c8``). * The STM system is based on very efficient read/write barriers, which are mostly done (their placement could be improved a bit in - JIT-generated machine code). But the overall bookkeeping logic could - see more improvements (see `Low-level statistics`_ below). + JIT-generated machine code). * Forking the process is slow because the complete memory needs to be copied manually. A warning is printed to this effect. @@ -132,7 +147,8 @@ crash on an assertion error because of a non-implemented overflow of an internal 28-bit counter. -.. _`report bugs`: https://bugs.pypy.org/ + +.. _`report any crash`: https://bitbucket.org/pypy/pypy/issues?status=new&status=open .. __: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/rpython/translator/stm/src_stm/stm/core.h @@ -155,7 +171,6 @@ interpreter and other ones might have slightly different needs. - User Guide ========== @@ -181,8 +196,33 @@ order. -A better way to write parallel programs ---------------------------------------- +How to write multithreaded programs: the 10'000-feet view +--------------------------------------------------------- + +PyPy-STM offers two ways to write multithreaded programs: + +* the traditional way, using the ``thread`` or ``threading`` modules. + +* using ``TransactionQueue``, described next__, as a way to hide the + low-level notion of threads. + +.. __: `transaction.TransactionQueue`_ + +``TransactionQueue`` hides the hard multithreading-related issues that +we typically encounter when using low-level threads. This is not the +first alternative approach to avoid dealing with low-level threads; +for example, OpenMP_ is one. However, it is one of the first ones +which does not require the code to be organized in a particular +fashion. Instead, it works on any Python program which has got +*latent* and *imperfect* parallelism. Ideally, it only requires that +the end programmer identifies where this parallelism is likely to be +found, and communicates it to the system using a simple API. + +.. _OpenMP: http://en.wikipedia.org/wiki/OpenMP + + +transaction.TransactionQueue +---------------------------- In CPU-hungry programs, we can often easily identify outermost loops over some data structure, or other repetitive algorithm, where each @@ -216,41 +256,115 @@ behavior did not change because we are using ``TransactionQueue``. All the calls still *appear* to execute in some serial order. -Now the performance should ideally be improved: if the function calls -turn out to be actually independent (most of the time), then it will -be. But if the function calls are not, then the total performance -will crawl back to the previous case, with additionally some small -penalty for the overhead. +However, the performance typically does not increase out of the box. +In fact, it is likely to be worse at first. Typically, this is +indicated by the total CPU usage, which remains low (closer to 1 than +N cores). First note that it is expected that the CPU usage should +not go much higher than 1 in the JIT warm-up phase: you must run a +program for several seconds, or for larger programs at least one +minute, to give the JIT a chance to warm up enough. But if CPU usage +remains low even afterwards, then the ``PYPYSTM`` environment variable +can be used to track what is going on. -This case occurs typically when you see the total CPU usage remaining -low (closer to 1 than N cores). Note first that it is expected that -the CPU usage should not go much higher than 1 in the JIT warm-up -phase. You must run a program for several seconds, or for larger -programs at least one minute, to give the JIT a chance to warm up -correctly. But if CPU usage remains low even though all code is -executing in a ``TransactionQueue.run()``, then the ``PYPYSTM`` -environment variable can be used to track what is going on. +Run your program with ``PYPYSTM=logfile`` to produce a log file called +``logfile``. Afterwards, use the ``pypy/stm/print_stm_log.py`` +utility to inspect the content of this log file. It produces output +like this (sorted by amount of time lost, largest first):: -Run your program with ``PYPYSTM=stmlog`` to produce a log file called -``stmlog``. Afterwards, use the ``pypy/stm/print_stm_log.py`` utility -to inspect the content of this log file. It produces output like -this:: + 10.5s lost in aborts, 1.25s paused (12412x STM_CONTENTION_WRITE_WRITE) + File "foo.py", line 10, in f + someobj.stuff = 5 + File "bar.py", line 20, in g + someobj.other = 10 - documentation in progress! +This means that 10.5 seconds were lost running transactions that were +aborted (which caused another 1.25 seconds of lost time by pausing), +because of the reason shown in the two independent single-entry +tracebacks: one thread ran the line ``someobj.stuff = 5``, whereas +another thread concurrently ran the line ``someobj.other = 10`` on the +same object. Two writes to the same object cause a conflict, which +aborts one of the two transactions. In the example above this +occurred 12412 times. +The two other conflict sources are ``STM_CONTENTION_INEVITABLE``, +which means that two transactions both tried to do an external +operation, like printing or reading from a socket or accessing an +external array of raw data; and ``STM_CONTENTION_WRITE_READ``, which +means that one transaction wrote to an object but the other one merely +read it, not wrote to it (in that case only the writing transaction is +reported; the location for the reads is not recorded because doing so +is not possible without a very large performance impact). + +Common causes of conflicts: + +* First of all, any I/O or raw manipulation of memory turns the + transaction inevitable ("must not abort"). There can be only one + inevitable transaction running at any time. A common case is if + each transaction starts with sending data to a log file. You should + refactor this case so that it occurs either near the end of the + transaction (which can then mostly run in non-inevitable mode), or + even delegate it to a separate thread. + +* Writing to a list or a dictionary conflicts with any read from the + same list or dictionary, even one done with a different key. For + dictionaries and sets, you can try the types ``transaction.stmdict`` + and ``transaction.stmset``, which behave mostly like ``dict`` and + ``set`` but allow concurrent access to different keys. (What is + missing from them so far is lazy iteration: for example, + ``stmdict.iterkeys()`` is implemented as ``iter(stmdict.keys())``; + and, unlike PyPy's dictionaries and sets, the STM versions are not + ordered.) There are also experimental ``stmiddict`` and + ``stmidset`` classes using the identity of the key. + +* ``time.time()`` and ``time.clock()`` turn the transaction inevitable + in order to guarantee that a call that appears to be later will + really return a higher number. If getting slightly unordered + results is fine, use ``transaction.time()`` or + ``transaction.clock()``. + +* ``transaction.threadlocalproperty`` can be used as class-level:: + + class Foo(object): # must be a new-style class! + x = transaction.threadlocalproperty() + y = transaction.threadlocalproperty(dict) + + This declares that instances of ``Foo`` have two attributes ``x`` + and ``y`` that are thread-local: reading or writing them from + concurrently-running transactions will return independent results. + (Any other attributes of ``Foo`` instances will be globally visible + from all threads, as usual.) The optional argument to + ``threadlocalproperty()`` is the default value factory: in case no + value was assigned in the current thread yet, the factory is called + and its result becomes the value in that thread (like + ``collections.defaultdict``). If no default value factory is + specified, uninitialized reads raise ``AttributeError``. Note that + with ``TransactionQueue`` you get a pool of a fixed number of + threads, each running the transactions one after the other; such + thread-local properties will have the value last stored in them in + the same thread,, which may come from a random previous transaction. + ``threadlocalproperty`` is still useful to avoid conflicts from + cache-like data structures. + +Note that Python is a complicated language; there are a number of less +common cases that may cause conflict (of any type) where we might not +expect it at priori. In many of these cases it could be fixed; please +report any case that you don't understand. (For example, so far, +creating a weakref to an object requires attaching an auxiliary +internal object to that object, and so it can cause write-write +conflicts.) Atomic sections --------------- -PyPy supports *atomic sections,* which are blocks of code which you -want to execute without "releasing the GIL". In STM terms, this means -blocks of code that are executed while guaranteeing that the -transaction is not interrupted in the middle. *This is experimental -and may be removed in the future* if `lock elision`_ is ever -implemented. +The ``TransactionQueue`` class described above is based on *atomic +sections,* which are blocks of code which you want to execute without +"releasing the GIL". In STM terms, this means blocks of code that are +executed while guaranteeing that the transaction is not interrupted in +the middle. *This is experimental and may be removed in the future* +if `Software lock elision`_ is ever implemented. -Here is a usage example:: +Here is a direct usage example:: with transaction.atomic: assert len(lst1) == 10 @@ -295,7 +409,8 @@ including with a ``print`` to standard output. If one thread tries to acquire a lock while running in an atomic block, and another thread has got the same lock at that point, then the former may fail with a -``thread.error``. The reason is that "waiting" for some condition to +``thread.error``. (Don't rely on it; it may also deadlock.) +The reason is that "waiting" for some condition to become true --while running in an atomic block-- does not really make sense. For now you can work around it by making sure that, say, all your prints are either in an ``atomic`` block or none of them are. @@ -354,106 +469,38 @@ .. _`software lock elision`: https://www.repository.cam.ac.uk/handle/1810/239410 -Atomic sections, Transactions, etc.: a better way to write parallel programs ----------------------------------------------------------------------------- +Miscellaneous functions +----------------------- -(This section is based on locks as we plan to implement them, but also -works with the existing atomic sections.) - -In the cases where elision works, the block of code can run in parallel -with other blocks of code *even if they are protected by the same lock.* -You still get the illusion that the blocks are run sequentially. This -works even for multiple threads that run each a series of such blocks -and nothing else, protected by one single global lock. This is -basically the Python application-level equivalent of what was done with -the interpreter in ``pypy-stm``: while you think you are writing -thread-unfriendly code because of this global lock, actually the -underlying system is able to make it run on multiple cores anyway. - -This capability can be hidden in a library or in the framework you use; -the end user's code does not need to be explicitly aware of using -threads. For a simple example of this, there is `transaction.py`_ in -``lib_pypy``. The idea is that you write, or already have, some program -where the function ``f(key, value)`` runs on every item of some big -dictionary, say:: - - for key, value in bigdict.items(): - f(key, value) - -Then you simply replace the loop with:: - - for key, value in bigdict.items(): - transaction.add(f, key, value) - transaction.run() - -This code runs the various calls to ``f(key, value)`` using a thread -pool, but every single call is executed under the protection of a unique -lock. The end result is that the behavior is exactly equivalent --- in -fact it makes little sense to do it in this way on a non-STM PyPy or on -CPython. But on ``pypy-stm``, the various locked calls to ``f(key, -value)`` can tentatively be executed in parallel, even if the observable -result is as if they were executed in some serial order. - -This approach hides the notion of threads from the end programmer, -including all the hard multithreading-related issues. This is not the -first alternative approach to explicit threads; for example, OpenMP_ is -one. However, it is one of the first ones which does not require the -code to be organized in a particular fashion. Instead, it works on any -Python program which has got latent, imperfect parallelism. Ideally, it -only requires that the end programmer identifies where this parallelism -is likely to be found, and communicates it to the system, using for -example the ``transaction.add()`` scheme. - -.. _`transaction.py`: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/lib_pypy/transaction.py -.. _OpenMP: http://en.wikipedia.org/wiki/OpenMP - - -.. _`transactional_memory`: - -API of transactional_memory ---------------------------- - -The new pure Python module ``transactional_memory`` runs on both CPython -and PyPy, both with and without STM. It contains: - -* ``getsegmentlimit()``: return the number of "segments" in +* ``transaction.getsegmentlimit()``: return the number of "segments" in this pypy-stm. This is the limit above which more threads will not be able to execute on more cores. (Right now it is limited to 4 due to inter-segment overhead, but should be increased in the future. It should also be settable, and the default value should depend on the number of actual CPUs.) If STM is not available, this returns 1. -* ``print_abort_info(minimum_time=0.0)``: debugging help. Each thread - remembers the longest abort or pause it did because of cross-thread - contention_. This function prints it to ``stderr`` if the time lost - is greater than ``minimum_time`` seconds. The record is then - cleared, to make it ready for new events. This function returns - ``True`` if it printed a report, and ``False`` otherwise. +* ``__pypy__.thread.signals_enabled``: a context manager that runs its + block of code with signals enabled. By default, signals are only + enabled in the main thread; a non-main thread will not receive + signals (this is like CPython). Enabling signals in non-main + threads is useful for libraries where threads are hidden and the end + user is not expecting his code to run elsewhere than in the main + thread. +* ``pypystm.exclusive_atomic``: a context manager similar to + ``transaction.atomic`` but which complains if it is nested. -API of __pypy__.thread ----------------------- +* ``transaction.is_atomic()``: return True if called from an atomic + context. -The ``__pypy__.thread`` submodule is a built-in module of PyPy that -contains a few internal built-in functions used by the -``transactional_memory`` module, plus the following: +* ``pypystm.count()``: return a different positive integer every time + it is called. This works without generating conflicts. The + returned integers are only roughly in increasing order; this should + not be relied upon. -* ``__pypy__.thread.atomic``: a context manager to run a block in - fully atomic mode, without "releasing the GIL". (May be eventually - removed?) -* ``__pypy__.thread.signals_enabled``: a context manager that runs its - block with signals enabled. By default, signals are only enabled in - the main thread; a non-main thread will not receive signals (this is - like CPython). Enabling signals in non-main threads is useful for - libraries where threads are hidden and the end user is not expecting - his code to run elsewhere than in the main thread. - - -.. _contention: - -Conflicts ---------- +More details about conflicts +---------------------------- Based on Software Transactional Memory, the ``pypy-stm`` solution is prone to "conflicts". To repeat the basic idea, threads execute their code @@ -469,7 +516,7 @@ the transaction). If this occurs too often, parallelization fails. How much actual parallelization a multithreaded program can see is a bit -subtle. Basically, a program not using ``__pypy__.thread.atomic`` or +subtle. Basically, a program not using ``transaction.atomic`` or eliding locks, or doing so for very short amounts of time, will parallelize almost freely (as long as it's not some artificial example where, say, all threads try to increase the same global counter and do @@ -481,13 +528,14 @@ overview. Parallelization works as long as two principles are respected. The -first one is that the transactions must not *conflict* with each other. -The most obvious sources of conflicts are threads that all increment a -global shared counter, or that all store the result of their -computations into the same list --- or, more subtly, that all ``pop()`` -the work to do from the same list, because that is also a mutation of -the list. (It is expected that some STM-aware library will eventually -be designed to help with conflict problems, like a STM-aware queue.) +first one is that the transactions must not *conflict* with each +other. The most obvious sources of conflicts are threads that all +increment a global shared counter, or that all store the result of +their computations into the same list --- or, more subtly, that all +``pop()`` the work to do from the same list, because that is also a +mutation of the list. (You can work around it with +``transaction.stmdict``, but for that specific example, some STM-aware +queue should eventually be designed.) A conflict occurs as follows: when a transaction commits (i.e. finishes successfully) it may cause other transactions that are still in progress @@ -503,22 +551,23 @@ Another issue is that of avoiding long-running so-called "inevitable" transactions ("inevitable" is taken in the sense of "which cannot be avoided", i.e. transactions which cannot abort any more). Transactions -like that should only occur if you use ``__pypy__.thread.atomic``, -generally become of I/O in atomic blocks. They work, but the +like that should only occur if you use ``atomic``, +generally because of I/O in atomic blocks. They work, but the transaction is turned inevitable before the I/O is performed. For all the remaining execution time of the atomic block, they will impede parallel work. The best is to organize the code so that such operations -are done completely outside ``__pypy__.thread.atomic``. +are done completely outside ``atomic``. -(This is related to the fact that blocking I/O operations are +(This is not unrelated to the fact that blocking I/O operations are discouraged with Twisted, and if you really need them, you should do them on their own separate thread.) -In case of lock elision, we don't get long-running inevitable -transactions, but a different problem can occur: doing I/O cancels lock -elision, and the lock turns into a real lock, preventing other threads -from committing if they also need this lock. (More about it when lock -elision is implemented and tested.) +In case lock elision eventually replaces atomic sections, we wouldn't +get long-running inevitable transactions, but the same problem occurs +in a different way: doing I/O cancels lock elision, and the lock turns +into a real lock. This prevents other threads from committing if they +also need this lock. (More about it when lock elision is implemented +and tested.) @@ -528,56 +577,18 @@ XXX this section mostly empty for now -Low-level statistics --------------------- - -When a non-main thread finishes, you get low-level statistics printed to -stderr, looking like that:: - - thread 0x7f73377fe600: - outside transaction 42182 0.506 s - run current 85466 0.000 s - run committed 34262 3.178 s - run aborted write write 6982 0.083 s - run aborted write read 550 0.005 s - run aborted inevitable 388 0.010 s - run aborted other 0 0.000 s - wait free segment 0 0.000 s - wait write read 78 0.027 s - wait inevitable 887 0.490 s - wait other 0 0.000 s - sync commit soon 1 0.000 s - bookkeeping 51418 0.606 s - minor gc 162970 1.135 s - major gc 1 0.019 s - sync pause 59173 1.738 s - longest recordered marker 0.000826 s - "File "x.py", line 5, in f" - -On each line, the first number is a counter, and the second number gives -the associated time --- the amount of real time that the thread was in -this state. The sum of all the times should be equal to the total time -between the thread's start and the thread's end. The most important -points are "run committed", which gives the amount of useful work, and -"outside transaction", which should give the time spent e.g. in library -calls (right now it seems to be larger than that; to investigate). The -various "run aborted" and "wait" entries are time lost due to -conflicts_. Everything else is overhead of various forms. (Short-, -medium- and long-term future work involves reducing this overhead :-) - -The last two lines are special; they are an internal marker read by -``transactional_memory.print_abort_info()``. - - Reference to implementation details ----------------------------------- -The core of the implementation is in a separate C library called stmgc_, -in the c7_ subdirectory. Please see the `README.txt`_ for more -information. In particular, the notion of segment is discussed there. +The core of the implementation is in a separate C library called +stmgc_, in the c7_ subdirectory (current version of pypy-stm) and in +the c8_ subdirectory (bleeding edge version). Please see the +`README.txt`_ for more information. In particular, the notion of +segment is discussed there. .. _stmgc: https://bitbucket.org/pypy/stmgc/src/default/ .. _c7: https://bitbucket.org/pypy/stmgc/src/default/c7/ +.. _c8: https://bitbucket.org/pypy/stmgc/src/default/c8/ .. _`README.txt`: https://bitbucket.org/pypy/stmgc/raw/default/c7/README.txt PyPy itself adds on top of it the automatic placement of read__ and write__ diff --git a/pypy/goal/getnightly.py b/pypy/goal/getnightly.py --- a/pypy/goal/getnightly.py +++ b/pypy/goal/getnightly.py @@ -7,7 +7,7 @@ if sys.platform.startswith('linux'): arch = 'linux' cmd = 'wget "%s"' - tar = "tar -x -v --wildcards --strip-components=2 -f %s '*/bin/pypy'" + tar = "tar -x -v --wildcards --strip-components=2 -f %s '*/bin/pypy' '*/bin/libpypy-c.so'" if os.uname()[-1].startswith('arm'): arch += '-armhf-raspbian' elif sys.platform.startswith('darwin'): diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -136,7 +136,9 @@ filename = filename[:-1] basename = os.path.basename(filename) lastdirname = os.path.basename(os.path.dirname(filename)) - self.co_filename = '/%s/%s' % (lastdirname, basename) + if lastdirname: + basename = '%s/%s' % (lastdirname, basename) + self.co_filename = '/%s' % (basename,) co_names = property(lambda self: [self.space.unwrap(w_name) for w_name in self.co_names_w]) # for trace diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -1619,6 +1619,13 @@ def prepare_exec(f, prog, globals, locals, compile_flags, builtin, codetype): """Manipulate parameters to exec statement to (codeobject, dict, dict). """ + if (globals is None and locals is None and + isinstance(prog, tuple) and + (len(prog) == 2 or len(prog) == 3)): + globals = prog[1] + if len(prog) == 3: + locals = prog[2] + prog = prog[0] if globals is None: globals = f.f_globals if locals is None: diff --git a/pypy/interpreter/test/test_exec.py b/pypy/interpreter/test/test_exec.py --- a/pypy/interpreter/test/test_exec.py +++ b/pypy/interpreter/test/test_exec.py @@ -262,3 +262,11 @@ """] for c in code: compile(c, "", "exec") + + def test_exec_tuple(self): + # note: this is VERY different than testing exec("a = 42", d), because + # this specific case is handled specially by the AST compiler + d = {} + x = ("a = 42", d) + exec x + assert d['a'] == 42 diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -30,7 +30,7 @@ space.wrap(addr.get_protocol()), space.wrap(addr.get_pkttype()), space.wrap(addr.get_hatype()), - space.wrap(addr.get_addr())]) + space.wrap(addr.get_haddr())]) elif rsocket.HAS_AF_UNIX and isinstance(addr, rsocket.UNIXAddress): return space.wrap(addr.get_path()) elif rsocket.HAS_AF_NETLINK and isinstance(addr, rsocket.NETLINKAddress): @@ -79,7 +79,7 @@ raise NotImplementedError # XXX Hack to seperate rpython and pypy -def addr_from_object(family, space, w_address): +def addr_from_object(family, fd, space, w_address): if family == rsocket.AF_INET: w_host, w_port = space.unpackiterable(w_address, 2) host = space.str_w(w_host) @@ -89,8 +89,9 @@ if family == rsocket.AF_INET6: pieces_w = space.unpackiterable(w_address) if not (2 <= len(pieces_w) <= 4): - raise TypeError("AF_INET6 address must be a tuple of length 2 " - "to 4, not %d" % len(pieces_w)) + raise oefmt(space.w_TypeError, + "AF_INET6 address must be a tuple of length 2 " + "to 4, not %d", len(pieces_w)) host = space.str_w(pieces_w[0]) port = space.int_w(pieces_w[1]) port = make_ushort_port(space, port) @@ -105,6 +106,28 @@ if rsocket.HAS_AF_NETLINK and family == rsocket.AF_NETLINK: w_pid, w_groups = space.unpackiterable(w_address, 2) return rsocket.NETLINKAddress(space.uint_w(w_pid), space.uint_w(w_groups)) + if rsocket.HAS_AF_PACKET and family == rsocket.AF_PACKET: + pieces_w = space.unpackiterable(w_address) + if not (2 <= len(pieces_w) <= 5): + raise oefmt(space.w_TypeError, + "AF_PACKET address must be a tuple of length 2 " + "to 5, not %d", len(pieces_w)) + ifname = space.str_w(pieces_w[0]) + ifindex = rsocket.PacketAddress.get_ifindex_from_ifname(fd, ifname) + protocol = space.int_w(pieces_w[1]) + if len(pieces_w) > 2: pkttype = space.int_w(pieces_w[2]) + else: pkttype = 0 + if len(pieces_w) > 3: hatype = space.int_w(pieces_w[3]) + else: hatype = 0 + if len(pieces_w) > 4: haddr = space.str_w(pieces_w[4]) + else: haddr = "" + if len(haddr) > 8: + raise OperationError(space.w_ValueError, space.wrap( + "Hardware address must be 8 bytes or less")) + if protocol < 0 or protocol > 0xfffff: + raise OperationError(space.w_OverflowError, space.wrap( + "protoNumber must be 0-65535.")) + return rsocket.PacketAddress(ifindex, protocol, pkttype, hatype, haddr) raise RSocketError("unknown address family") # XXX Hack to seperate rpython and pypy @@ -172,7 +195,8 @@ # convert an app-level object into an Address # based on the current socket's family def addr_from_object(self, space, w_address): - return addr_from_object(self.sock.family, space, w_address) + fd = intmask(self.sock.fd) + return addr_from_object(self.sock.family, fd, space, w_address) def bind_w(self, space, w_addr): """bind(address) diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -1,4 +1,4 @@ -import sys +import sys, os import py from pypy.tool.pytest.objspace import gettestobjspace from rpython.tool.udir import udir @@ -615,6 +615,28 @@ os.chdir(oldcwd) +class AppTestPacket: + def setup_class(cls): + if not hasattr(os, 'getuid') or os.getuid() != 0: + py.test.skip("AF_PACKET needs to be root for testing") + w_ok = space.appexec([], "(): import _socket; " + + "return hasattr(_socket, 'AF_PACKET')") + if not space.is_true(w_ok): + py.test.skip("no AF_PACKET on this platform") + cls.space = space + + def test_convert_between_tuple_and_sockaddr_ll(self): + import _socket + s = _socket.socket(_socket.AF_PACKET, _socket.SOCK_RAW) + assert s.getsockname() == ('', 0, 0, 0, '') + s.bind(('lo', 123)) + a, b, c, d, e = s.getsockname() + assert (a, b, c) == ('lo', 123, 0) + assert isinstance(d, int) + assert isinstance(e, str) + assert 0 <= len(e) <= 8 + + class AppTestSocketTCP: HOST = 'localhost' diff --git a/pypy/module/zipimport/test/test_zipimport_deflated.py b/pypy/module/zipimport/test/test_zipimport_deflated.py --- a/pypy/module/zipimport/test/test_zipimport_deflated.py +++ b/pypy/module/zipimport/test/test_zipimport_deflated.py @@ -14,7 +14,7 @@ def setup_class(cls): try: import rpython.rlib.rzlib - except ImportError: + except CompilationError: py.test.skip("zlib not available, cannot test compressed zipfiles") cls.make_class() cls.w_BAD_ZIP = cls.space.wrap(BAD_ZIP) diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py --- a/rpython/annotator/binaryop.py +++ b/rpython/annotator/binaryop.py @@ -132,13 +132,11 @@ impl = pair(s_c1, s_o2).getitem return read_can_only_throw(impl, s_c1, s_o2) - def getitem_idx_key((s_c1, s_o2)): + def getitem_idx((s_c1, s_o2)): impl = pair(s_c1, s_o2).getitem return impl() - getitem_idx_key.can_only_throw = _getitem_can_only_throw + getitem_idx.can_only_throw = _getitem_can_only_throw - getitem_idx = getitem_idx_key - getitem_key = getitem_idx_key class __extend__(pairtype(SomeType, SomeType), @@ -565,14 +563,10 @@ return lst1.listdef.read_item() getitem.can_only_throw = [] - getitem_key = getitem - def getitem_idx((lst1, int2)): return lst1.listdef.read_item() getitem_idx.can_only_throw = [IndexError] - getitem_idx_key = getitem_idx - def setitem((lst1, int2), s_value): lst1.listdef.mutate() lst1.listdef.generalize(s_value) @@ -588,14 +582,10 @@ return SomeChar(no_nul=str1.no_nul) getitem.can_only_throw = [] - getitem_key = getitem - def getitem_idx((str1, int2)): return SomeChar(no_nul=str1.no_nul) getitem_idx.can_only_throw = [IndexError] - getitem_idx_key = getitem_idx - def mul((str1, int2)): # xxx do we want to support this return SomeString(no_nul=str1.no_nul) @@ -604,14 +594,10 @@ return SomeUnicodeCodePoint() getitem.can_only_throw = [] - getitem_key = getitem - def getitem_idx((str1, int2)): return SomeUnicodeCodePoint() getitem_idx.can_only_throw = [IndexError] - getitem_idx_key = getitem_idx - def mul((str1, int2)): # xxx do we want to support this return SomeUnicodeString() diff --git a/rpython/doc/jit/index.rst b/rpython/doc/jit/index.rst --- a/rpython/doc/jit/index.rst +++ b/rpython/doc/jit/index.rst @@ -23,11 +23,15 @@ overview pyjitpl5 + optimizer virtualizable - :doc:`Overview `: motivating our approach - :doc:`Notes ` about the current work in PyPy +- :doc:`Optimizer `: the step between tracing and writing + machine code + - :doc:`Virtulizable ` how virtualizables work and what they are (in other words how to make frames more efficient). diff --git a/rpython/doc/jit/optimizer.rst b/rpython/doc/jit/optimizer.rst new file mode 100644 --- /dev/null +++ b/rpython/doc/jit/optimizer.rst @@ -0,0 +1,196 @@ +.. _trace_optimizer: + +Trace Optimizer +=============== + +Traces of user programs are not directly translated into machine code. +The optimizer module implements several different semantic preserving +transformations that either allow operations to be swept from the trace +or convert them to operations that need less time or space. + +The optimizer is in `rpython/jit/metainterp/optimizeopt/`. +When you try to make sense of this module, this page might get you started. + +Before some optimizations are explained in more detail, it is essential to +understand how traces look like. +The optimizer comes with a test suit. It contains many trace +examples and you might want to take a look at it +(in `rpython/jit/metainterp/optimizeopt/test/*.py`). +The allowed operations can be found in `rpython/jit/metainterp/resoperation.py`. +Here is an example of a trace:: + + [p0,i0,i1] + label(p0, i0, i1) + i2 = getarray_item_raw(p0, i0, descr=) + i3 = int_add(i1,i2) + i4 = int_add(i0,1) + i5 = int_le(i4, 100) # lower-or-equal + guard_true(i5) + jump(p0, i4, i3) + +At the beginning it might be clumsy to read but it makes sense when you start +to compare the Python code that constructed the trace:: + + from array import array + a = array('i',range(101)) + sum = 0; i = 0 + while i <= 100: # can be seen as label + sum += a[i] + i += 1 + # jumps back to the while header + +There are better ways to compute the sum from ``[0..100]``, but it gives a better intuition on how +traces are constructed than ``sum(range(101))``. +Note that the trace syntax is the one used in the test suite. It is also very +similar to traces printed at runtime by PYPYLOG_. The first line gives the input variables, the +second line is a ``label`` operation, the last one is the backwards ``jump`` operation. + +.. _PYPYLOG: logging.html + +These instructions mentioned earlier are special: + +* the input defines the input parameter type and name to enter the trace. +* ``label`` is the instruction a ``jump`` can target. Label instructions have + a ``JitCellToken`` associated that uniquely identifies the label. Any jump + has a target token of a label. + +The token is saved in a so called `descriptor` of the instruction. It is +not written explicitly because it is not done in the tests either. But +the test suite creates a dummy token for each trace and adds it as descriptor +to ``label`` and ``jump``. Of course the optimizer does the same at runtime, +but using real values. +The sample trace includes a descriptor in ``getarrayitem_raw``. Here it +annotates the type of the array. It is a signed integer array. + +High level overview +------------------- + +Before the JIT backend transforms any trace into machine code, it tries to +transform the trace into an equivalent trace that executes faster. The method +`optimize_trace` in `rpython/jit/metainterp/optimizeopt/__init__.py` is the +main entry point. + +Optimizations are applied in a sequence one after another and the base +sequence is as follows:: + + intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll + +Each of the colon-separated name has a class attached, inheriting from +the `Optimization` class. The `Optimizer` class itself also +derives from the `Optimization` class and implements the control logic for +the optimization. Most of the optimizations only require a single forward pass. +The trace is 'propagated' into each optimization using the method +`propagate_forward`. Instruction by instruction, it flows from the +first optimization to the last optimization. The method `emit_operation` +is called for every operation that is passed to the next optimizer. + +A frequently encountered pattern +-------------------------------- + +To find potential optimization targets it is necessary to know the instruction +type. Simple solution is to switch using the operation number (= type):: + + for op in operations: + if op.getopnum() == rop.INT_ADD: + # handle this instruction + pass + elif op.getopnum() == rop.INT_FLOOR_DIV: + pass + # and many more + +Things get worse if you start to match the arguments +(is argument one constant and two variable or vice versa?). The pattern to tackle +this code bloat is to move it to a separate method using +`make_dispatcher_method`. It associates methods with instruction types:: + + class OptX(Optimization): + def prefix_INT_ADD(self, op): + pass # emit, transform, ... + + dispatch_opt = make_dispatcher_method(OptX, 'prefix_', + default=OptX.emit_operation) + OptX.propagate_forward = dispatch_opt + + optX = OptX() + for op in operations: + optX.propagate_forward(op) + +``propagate_forward`` searches for the method that is able to handle the instruction +type. As an example `INT_ADD` will invoke `prefix_INT_ADD`. If there is no function +for the instruction, it is routed to the default implementation (``emit_operation`` +in this example). + +Rewrite optimization +-------------------- + +The second optimization is called 'rewrite' and is commonly also known as +strength reduction. A simple example would be that an integer multiplied +by 2 is equivalent to the bits shifted to the left once +(e.g. ``x * 2 == x << 1``). Not only strength reduction is done in this +optimization but also boolean or arithmetic simplifications. Other examples +would be: ``x & 0 == 0``, ``x - 0 == x`` + +Whenever such an operation is encountered (e.g. ``y = x & 0``), no operation is +emitted. Instead the variable y is made equal to 0 +(= ``make_equal_to(op.result, 0)``). The variables found in a trace are +instances of Box classes that can be found in +`rpython/jit/metainterp/history.py`. `OptValue` wraps those variables again +and maps the boxes to the optimization values in the optimizer. When a +value is made equal, the two variable's boxes are made to point to the same +`OptValue` instance. + +**NOTE: this OptValue organization is currently being refactored in a branch.** + +Pure optimization +----------------- + +Is interwoven into the basic optimizer. It saves operations, results, +arguments to be known to have pure semantics. + +"Pure" here means the same as the ``jit.elidable`` decorator: +free of "observable" side effects and referentially transparent +(the operation can be replaced with its result without changing the program +semantics). The operations marked as ALWAYS_PURE in `resoperation.py` are a +subset of the NOSIDEEFFECT operations. Operations such as new, new array, +getfield_(raw/gc) are marked as NOSIDEEFFECT but not as ALWAYS_PURE. + +Pure operations are optimized in two different ways. If their arguments +are constants, the operation is removed and the result is turned into a +constant. If not, we can still use a memoization technique: if, later, +we see the same operation on the same arguments again, we don't need to +recompute its result, but can simply reuse the previous operation's +result. + +Unroll optimization +------------------- + +A detailed description can be found the document +`Loop-Aware Optimizations in PyPy's Tracing JIT`__ + +.. __: http://www2.maths.lth.se/matematiklth/vision/publdb/reports/pdf/ardo-bolz-etal-dls-12.pdf + +This optimization does not fall into the traditional scheme of one forward +pass only. In a nutshell it unrolls the trace _once_, connects the two +traces (by inserting parameters into the jump and label of the peeled trace) +and uses information to iron out allocations, propagate constants and +do any other optimization currently present in the 'optimizeopt' module. + +It is prepended to all optimizations and thus extends the Optimizer class +and unrolls the loop once before it proceeds. + + +What is missing from this document +---------------------------------- + +* Guards are not explained +* Several optimizations are not explained + + +Further references +------------------ + +* `Allocation Removal by Partial Evaluation in a Tracing JIT`__ +* `Loop-Aware Optimizations in PyPy's Tracing JIT`__ + +.. __: http://www.stups.uni-duesseldorf.de/mediawiki/images/b/b0/Pub-BoCuFiLePeRi2011.pdf +.. __: http://www2.maths.lth.se/matematiklth/vision/publdb/reports/pdf/ardo-bolz-etal-dls-12.pdf diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py --- a/rpython/flowspace/model.py +++ b/rpython/flowspace/model.py @@ -140,6 +140,12 @@ newlink.llexitcase = self.llexitcase return newlink + def replace(self, mapping): + def rename(v): + if v is not None: + return v.replace(mapping) + return self.copy(rename) + def settarget(self, targetblock): assert len(self.args) == len(targetblock.inputargs), ( "output args mismatch") @@ -215,13 +221,12 @@ return uniqueitems([w for w in result if isinstance(w, Constant)]) def renamevariables(self, mapping): - self.inputargs = [mapping.get(a, a) for a in self.inputargs] - for op in self.operations: - op.args = [mapping.get(a, a) for a in op.args] - op.result = mapping.get(op.result, op.result) - self.exitswitch = mapping.get(self.exitswitch, self.exitswitch) + self.inputargs = [a.replace(mapping) for a in self.inputargs] + self.operations = [op.replace(mapping) for op in self.operations] + if self.exitswitch is not None: + self.exitswitch = self.exitswitch.replace(mapping) for link in self.exits: - link.args = [mapping.get(a, a) for a in link.args] + link.args = [a.replace(mapping) for a in link.args] def closeblock(self, *exits): assert self.exits == [], "block already closed" @@ -327,6 +332,8 @@ newvar.concretetype = self.concretetype return newvar + def replace(self, mapping): + return mapping.get(self, self) class Constant(Hashable): @@ -356,6 +363,9 @@ # cannot count on it not mutating at runtime! return False + def replace(self, mapping): + return self + class FSException(object): def __init__(self, w_type, w_value): @@ -431,8 +441,8 @@ ", ".join(map(repr, self.args))) def replace(self, mapping): - newargs = [mapping.get(arg, arg) for arg in self.args] - newresult = mapping.get(self.result, self.result) + newargs = [arg.replace(mapping) for arg in self.args] + newresult = self.result.replace(mapping) return type(self)(self.opname, newargs, newresult, self.offset) class Atom(object): diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py --- a/rpython/flowspace/operation.py +++ b/rpython/flowspace/operation.py @@ -76,8 +76,8 @@ self.offset = -1 def replace(self, mapping): - newargs = [mapping.get(arg, arg) for arg in self.args] - newresult = mapping.get(self.result, self.result) + newargs = [arg.replace(mapping) for arg in self.args] + newresult = self.result.replace(mapping) newop = type(self)(*newargs) newop.result = newresult newop.offset = self.offset @@ -422,8 +422,6 @@ add_operator('delattr', 2, dispatch=1, pyfunc=delattr) add_operator('getitem', 2, dispatch=2, pure=True) add_operator('getitem_idx', 2, dispatch=2, pure=True) -add_operator('getitem_key', 2, dispatch=2, pure=True) -add_operator('getitem_idx_key', 2, dispatch=2, pure=True) add_operator('setitem', 3, dispatch=2) add_operator('delitem', 2, dispatch=2) add_operator('getslice', 3, dispatch=1, pyfunc=do_getslice, pure=True) @@ -686,8 +684,6 @@ # the annotator tests op.getitem.canraise = [IndexError, KeyError, Exception] op.getitem_idx.canraise = [IndexError, KeyError, Exception] -op.getitem_key.canraise = [IndexError, KeyError, Exception] -op.getitem_idx_key.canraise = [IndexError, KeyError, Exception] op.setitem.canraise = [IndexError, KeyError, Exception] op.delitem.canraise = [IndexError, KeyError, Exception] op.contains.canraise = [Exception] # from an r_dict diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py --- a/rpython/flowspace/test/test_objspace.py +++ b/rpython/flowspace/test/test_objspace.py @@ -867,7 +867,7 @@ raise graph = self.codetest(f) simplify_graph(graph) - assert self.all_operations(graph) == {'getitem_idx_key': 1} + assert self.all_operations(graph) == {'getitem_idx': 1} g = lambda: None def f(c, x): @@ -877,7 +877,7 @@ g() graph = self.codetest(f) simplify_graph(graph) - assert self.all_operations(graph) == {'getitem_idx_key': 1, + assert self.all_operations(graph) == {'getitem_idx': 1, 'simple_call': 2} def f(c, x): @@ -896,7 +896,7 @@ raise graph = self.codetest(f) simplify_graph(graph) - assert self.all_operations(graph) == {'getitem_key': 1} + assert self.all_operations(graph) == {'getitem': 1} def f(c, x): try: @@ -915,7 +915,7 @@ graph = self.codetest(f) simplify_graph(graph) self.show(graph) - assert self.all_operations(graph) == {'getitem_idx_key': 1} + assert self.all_operations(graph) == {'getitem_idx': 1} def f(c, x): try: @@ -933,7 +933,7 @@ return -1 graph = self.codetest(f) simplify_graph(graph) - assert self.all_operations(graph) == {'getitem_key': 1} + assert self.all_operations(graph) == {'getitem': 1} def f(c, x): try: diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py --- a/rpython/rlib/_rsocket_rffi.py +++ b/rpython/rlib/_rsocket_rffi.py @@ -199,7 +199,7 @@ WSA_INVALID_PARAMETER WSA_NOT_ENOUGH_MEMORY WSA_OPERATION_ABORTED SIO_RCVALL SIO_KEEPALIVE_VALS -SIOCGIFNAME +SIOCGIFNAME SIOCGIFINDEX '''.split() for name in constant_names: @@ -328,7 +328,8 @@ if _HAS_AF_PACKET: CConfig.sockaddr_ll = platform.Struct('struct sockaddr_ll', - [('sll_ifindex', rffi.INT), + [('sll_family', rffi.INT), + ('sll_ifindex', rffi.INT), ('sll_protocol', rffi.INT), ('sll_pkttype', rffi.INT), ('sll_hatype', rffi.INT), diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py --- a/rpython/rlib/rsocket.py +++ b/rpython/rlib/rsocket.py @@ -5,15 +5,8 @@ a drop-in replacement for the 'socket' module. """ -# Known missing features: -# -# - address families other than AF_INET, AF_INET6, AF_UNIX, AF_PACKET -# - AF_PACKET is only supported on Linux -# - methods makefile(), -# - SSL -# -# It's unclear if makefile() and SSL support belong here or only as -# app-level code for PyPy. +# XXX this does not support yet the least common AF_xxx address families +# supported by CPython. See http://bugs.pypy.org/issue1942 from rpython.rlib import _rsocket_rffi as _c, jit, rgc from rpython.rlib.objectmodel import instantiate, keepalive_until_here @@ -200,23 +193,49 @@ family = AF_PACKET struct = _c.sockaddr_ll maxlen = minlen = sizeof(struct) + ifr_name_size = _c.ifreq.c_ifr_name.length + sll_addr_size = _c.sockaddr_ll.c_sll_addr.length + + def __init__(self, ifindex, protocol, pkttype=0, hatype=0, haddr=""): + addr = lltype.malloc(_c.sockaddr_ll, flavor='raw', zero=True, + track_allocation=False) + self.setdata(addr, PacketAddress.maxlen) + rffi.setintfield(addr, 'c_sll_family', AF_PACKET) + rffi.setintfield(addr, 'c_sll_protocol', htons(protocol)) + rffi.setintfield(addr, 'c_sll_ifindex', ifindex) + rffi.setintfield(addr, 'c_sll_pkttype', pkttype) + rffi.setintfield(addr, 'c_sll_hatype', hatype) + halen = rffi.str2chararray(haddr, + rffi.cast(rffi.CCHARP, addr.c_sll_addr), + PacketAddress.sll_addr_size) + rffi.setintfield(addr, 'c_sll_halen', halen) + + @staticmethod + def get_ifindex_from_ifname(fd, ifname): + p = lltype.malloc(_c.ifreq, flavor='raw') + iflen = rffi.str2chararray(ifname, + rffi.cast(rffi.CCHARP, p.c_ifr_name), + PacketAddress.ifr_name_size - 1) + p.c_ifr_name[iflen] = '\0' + err = _c.ioctl(fd, _c.SIOCGIFINDEX, p) + ifindex = p.c_ifr_ifindex + lltype.free(p, flavor='raw') + if err != 0: + raise RSocketError("invalid interface name") + return ifindex def get_ifname(self, fd): + ifname = "" a = self.lock(_c.sockaddr_ll) - p = lltype.malloc(_c.ifreq, flavor='raw') - rffi.setintfield(p, 'c_ifr_ifindex', - rffi.getintfield(a, 'c_sll_ifindex')) - if (_c.ioctl(fd, _c.SIOCGIFNAME, p) == 0): - # eh, the iface name is a constant length array - i = 0 - d = [] - while p.c_ifr_name[i] != '\x00' and i < len(p.c_ifr_name): - d.append(p.c_ifr_name[i]) - i += 1 - ifname = ''.join(d) - else: - ifname = "" - lltype.free(p, flavor='raw') + ifindex = rffi.getintfield(a, 'c_sll_ifindex') + if ifindex: + p = lltype.malloc(_c.ifreq, flavor='raw') + rffi.setintfield(p, 'c_ifr_ifindex', ifindex) + if (_c.ioctl(fd, _c.SIOCGIFNAME, p) == 0): + ifname = rffi.charp2strn( + rffi.cast(rffi.CCHARP, p.c_ifr_name), + PacketAddress.ifr_name_size) + lltype.free(p, flavor='raw') self.unlock() return ifname @@ -235,11 +254,11 @@ def get_hatype(self): a = self.lock(_c.sockaddr_ll) - res = bool(rffi.getintfield(a, 'c_sll_hatype')) + res = rffi.getintfield(a, 'c_sll_hatype') self.unlock() return res - def get_addr(self): + def get_haddr(self): a = self.lock(_c.sockaddr_ll) lgt = rffi.getintfield(a, 'c_sll_halen') d = [] diff --git a/rpython/rlib/rzipfile.py b/rpython/rlib/rzipfile.py --- a/rpython/rlib/rzipfile.py +++ b/rpython/rlib/rzipfile.py @@ -8,7 +8,7 @@ try: from rpython.rlib import rzlib -except (ImportError, CompilationError): +except CompilationError: rzlib = None crc_32_tab = [ diff --git a/rpython/rlib/rzlib.py b/rpython/rlib/rzlib.py --- a/rpython/rlib/rzlib.py +++ b/rpython/rlib/rzlib.py @@ -22,13 +22,10 @@ includes=['zlib.h'], testonly_libraries = testonly_libraries ) -try: - eci = rffi_platform.configure_external_library( - libname, eci, - [dict(prefix='zlib-'), - ]) -except CompilationError: - raise ImportError("Could not find a zlib library") +eci = rffi_platform.configure_external_library( + libname, eci, + [dict(prefix='zlib-'), + ]) constantnames = ''' diff --git a/rpython/rlib/test/test_rzipfile.py b/rpython/rlib/test/test_rzipfile.py --- a/rpython/rlib/test/test_rzipfile.py +++ b/rpython/rlib/test/test_rzipfile.py @@ -9,7 +9,7 @@ try: from rpython.rlib import rzlib -except ImportError, e: +except CompilationError as e: py.test.skip("zlib not installed: %s " % (e, )) class BaseTestRZipFile(BaseRtypingTest): diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py --- a/rpython/rtyper/lltypesystem/rffi.py +++ b/rpython/rtyper/lltypesystem/rffi.py @@ -794,6 +794,14 @@ else: lltype.free(cp, flavor='raw', track_allocation=False) + # str -> already-existing char[maxsize] + def str2chararray(s, array, maxsize): + length = min(len(s), maxsize) + ll_s = llstrtype(s) + copy_string_to_raw(ll_s, array, 0, length) + return length + str2chararray._annenforceargs_ = [strtype, None, int] + # char* -> str # doesn't free char* def charp2str(cp): @@ -944,19 +952,19 @@ return (str2charp, free_charp, charp2str, get_nonmovingbuffer, free_nonmovingbuffer, alloc_buffer, str_from_buffer, keep_buffer_alive_until_here, - charp2strn, charpsize2str, + charp2strn, charpsize2str, str2chararray, ) (str2charp, free_charp, charp2str, get_nonmovingbuffer, free_nonmovingbuffer, alloc_buffer, str_from_buffer, keep_buffer_alive_until_here, - charp2strn, charpsize2str, + charp2strn, charpsize2str, str2chararray, ) = make_string_mappings(str) (unicode2wcharp, free_wcharp, wcharp2unicode, get_nonmoving_unicodebuffer, free_nonmoving_unicodebuffer, alloc_unicodebuffer, unicode_from_buffer, keep_unicodebuffer_alive_until_here, - wcharp2unicoden, wcharpsize2unicode, + wcharp2unicoden, wcharpsize2unicode, unicode2wchararray, ) = make_string_mappings(unicode) # char** diff --git a/rpython/rtyper/lltypesystem/test/test_rffi.py b/rpython/rtyper/lltypesystem/test/test_rffi.py --- a/rpython/rtyper/lltypesystem/test/test_rffi.py +++ b/rpython/rtyper/lltypesystem/test/test_rffi.py @@ -102,24 +102,19 @@ #include #include - char *f(char* arg) + void f(char *target, char* arg) { - char *ret; - /* lltype.free uses OP_RAW_FREE, we must allocate - * with the matching function - */ - OP_RAW_MALLOC(strlen(arg) + 1, ret, char*) - strcpy(ret, arg); - return ret; + strcpy(target, arg); } """) eci = ExternalCompilationInfo(separate_module_sources=[c_source], - post_include_bits=['char *f(char*);']) - z = llexternal('f', [CCHARP], CCHARP, compilation_info=eci) + post_include_bits=['void f(char*,char*);']) + z = llexternal('f', [CCHARP, CCHARP], lltype.Void, compilation_info=eci) def f(): s = str2charp("xxx") - l_res = z(s) + l_res = lltype.malloc(CCHARP.TO, 10, flavor='raw') + z(l_res, s) res = charp2str(l_res) lltype.free(l_res, flavor='raw') free_charp(s) @@ -676,6 +671,23 @@ assert interpret(f, [], backendopt=True) == 43 + def test_str2chararray(self): + eci = ExternalCompilationInfo(includes=['string.h']) + strlen = llexternal('strlen', [CCHARP], SIZE_T, + compilation_info=eci) + def f(): + raw = str2charp("XxxZy") + n = str2chararray("abcdef", raw, 4) + assert raw[0] == 'a' + assert raw[1] == 'b' + assert raw[2] == 'c' + assert raw[3] == 'd' + assert raw[4] == 'y' + lltype.free(raw, flavor='raw') + return n + + assert interpret(f, []) == 4 + def test_around_extcall(self): if sys.platform == "win32": py.test.skip('No pipes on windows') diff --git a/rpython/rtyper/rlist.py b/rpython/rtyper/rlist.py --- a/rpython/rtyper/rlist.py +++ b/rpython/rtyper/rlist.py @@ -268,13 +268,9 @@ v_res = hop.gendirectcall(llfn, c_func_marker, c_basegetitem, v_lst, v_index) return r_lst.recast(hop.llops, v_res) - rtype_getitem_key = rtype_getitem - def rtype_getitem_idx((r_lst, r_int), hop): return pair(r_lst, r_int).rtype_getitem(hop, checkidx=True) - rtype_getitem_idx_key = rtype_getitem_idx - def rtype_setitem((r_lst, r_int), hop): if hop.has_implicit_exception(IndexError): spec = dum_checkidx diff --git a/rpython/rtyper/rmodel.py b/rpython/rtyper/rmodel.py --- a/rpython/rtyper/rmodel.py +++ b/rpython/rtyper/rmodel.py @@ -285,11 +285,9 @@ # default implementation for checked getitems - def rtype_getitem_idx_key((r_c1, r_o1), hop): + def rtype_getitem_idx((r_c1, r_o1), hop): return pair(r_c1, r_o1).rtype_getitem(hop) - rtype_getitem_idx = rtype_getitem_idx_key - rtype_getitem_key = rtype_getitem_idx_key # ____________________________________________________________ diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py --- a/rpython/rtyper/rstr.py +++ b/rpython/rtyper/rstr.py @@ -580,13 +580,9 @@ hop.exception_cannot_occur() return hop.gendirectcall(llfn, v_str, v_index) - rtype_getitem_key = rtype_getitem - def rtype_getitem_idx((r_str, r_int), hop): return pair(r_str, r_int).rtype_getitem(hop, checkidx=True) - rtype_getitem_idx_key = rtype_getitem_idx - def rtype_mul((r_str, r_int), hop): str_repr = r_str.repr v_str, v_int = hop.inputargs(str_repr, Signed) diff --git a/rpython/translator/platform/arm.py b/rpython/translator/platform/arm.py --- a/rpython/translator/platform/arm.py +++ b/rpython/translator/platform/arm.py @@ -21,11 +21,14 @@ available_librarydirs = [SB2 + '/lib/arm-linux-gnueabi/', SB2 + '/lib/arm-linux-gnueabihf/', + SB2 + '/lib/aarch64-linux-gnu/', SB2 + '/usr/lib/arm-linux-gnueabi/', - SB2 + '/usr/lib/arm-linux-gnueabihf/'] + SB2 + '/usr/lib/arm-linux-gnueabihf/', + SB2 + '/usr/lib/aarch64-linux-gnu/'] available_includedirs = [SB2 + '/usr/include/arm-linux-gnueabi/', - SB2 + '/usr/include/arm-linux-gnueabihf/'] + SB2 + '/usr/include/arm-linux-gnueabihf/', + SB2 + '/usr/include/aarch64-linux-gnu/'] copied_cache = {} diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py --- a/rpython/translator/platform/posix.py +++ b/rpython/translator/platform/posix.py @@ -167,7 +167,8 @@ ('CC', self.cc), ('CC_LINK', eci.use_cpp_linker and 'g++' or '$(CC)'), ('LINKFILES', eci.link_files), - ('RPATH_FLAGS', self.rpath_flags), + ('RPATH_FLAGS', self.rpath_flags + ['-Wl,-rpath-link=\'%s\'' % ldir + for ldir in rel_libdirs]), ] for args in definitions: m.definition(*args) diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py --- a/rpython/translator/simplify.py +++ b/rpython/translator/simplify.py @@ -63,28 +63,19 @@ When this happens, we need to replace the preceeding link with the following link. Arguments of the links should be updated.""" for link in list(graph.iterlinks()): - while not link.target.operations: - block1 = link.target - if block1.exitswitch is not None: - break - if not block1.exits: - break - exit = block1.exits[0] - assert block1 is not exit.target, ( - "the graph contains an empty infinite loop") - outputargs = [] - for v in exit.args: - if isinstance(v, Variable): - try: - i = block1.inputargs.index(v) - v = link.args[i] - except ValueError: - # the variable was passed implicitly to block1 - pass - outputargs.append(v) - link.args = outputargs - link.target = exit.target - # the while loop above will simplify recursively the new link + while not link.target.operations: + block1 = link.target + if block1.exitswitch is not None: + break + if not block1.exits: + break + exit = block1.exits[0] + assert block1 is not exit.target, ( + "the graph contains an empty infinite loop") + subst = dict(zip(block1.inputargs, link.args)) + link.args = [v.replace(subst) for v in exit.args] + link.target = exit.target + # the while loop above will simplify recursively the new link def transform_ovfcheck(graph): """The special function calls ovfcheck needs to @@ -132,9 +123,6 @@ the block's single list of exits. """ renaming = {} - def rename(v): - return renaming.get(v, v) - for block in graph.iterblocks(): if not (block.canraise and block.exits[-1].exitcase is Exception): @@ -151,10 +139,10 @@ while len(query.exits) == 2: newrenaming = {} for lprev, ltarg in zip(exc.args, query.inputargs): - newrenaming[ltarg] = rename(lprev) + newrenaming[ltarg] = lprev.replace(renaming) op = query.operations[0] if not (op.opname in ("is_", "issubtype") and - newrenaming.get(op.args[0]) == last_exception): + op.args[0].replace(newrenaming) == last_exception): break renaming.update(newrenaming) case = query.operations[0].args[-1].value @@ -177,19 +165,16 @@ # construct the block's new exits exits = [] for case, oldlink in switches: - link = oldlink.copy(rename) + link = oldlink.replace(renaming) assert case is not None link.last_exception = last_exception link.last_exc_value = last_exc_value # make the above two variables unique renaming2 = {} - def rename2(v): - return renaming2.get(v, v) for v in link.getextravars(): renaming2[v] = Variable(v) - link = link.copy(rename2) + link = link.replace(renaming2) link.exitcase = case - link.prevblock = block exits.append(link) block.recloseblock(*(preserve + exits)) @@ -203,8 +188,6 @@ for exit in block.exits: if exit.exitcase is IndexError: postfx.append('idx') - elif exit.exitcase is KeyError: - postfx.append('key') if postfx: Op = getattr(op, '_'.join(['getitem'] + postfx)) newop = Op(*last_op.args) @@ -313,8 +296,6 @@ renaming = {} for vprev, vtarg in zip(link.args, link.target.inputargs): renaming[vtarg] = vprev - def rename(v): - return renaming.get(v, v) def rename_op(op): op = op.replace(renaming) # special case... @@ -328,9 +309,12 @@ link.prevblock.operations.append(rename_op(op)) exits = [] for exit in link.target.exits: - newexit = exit.copy(rename) + newexit = exit.replace(renaming) exits.append(newexit) - newexitswitch = rename(link.target.exitswitch) + if link.target.exitswitch: + newexitswitch = link.target.exitswitch.replace(renaming) + else: + newexitswitch = None link.prevblock.exitswitch = newexitswitch link.prevblock.recloseblock(*exits) if (isinstance(newexitswitch, Constant) and @@ -618,7 +602,7 @@ assert len(links) == len(new_args) for link, args in zip(links, new_args): link.args = args - for block in graph.iterblocks(): + for block in entrymap: block.renamevariables(renaming) def remove_identical_vars(graph): From noreply at buildbot.pypy.org Wed Mar 18 11:40:36 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 18 Mar 2015 11:40:36 +0100 (CET) Subject: [pypy-commit] stmgc default: Oups, use cond_broadcast instead of cond_signal here Message-ID: <20150318104036.03A0A1C0173@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1737:7861b2a77263 Date: 2015-03-18 11:40 +0100 http://bitbucket.org/pypy/stmgc/changeset/7861b2a77263/ Log: Oups, use cond_broadcast instead of cond_signal here diff --git a/c7/stm/core.c b/c7/stm/core.c --- a/c7/stm/core.c +++ b/c7/stm/core.c @@ -889,7 +889,8 @@ /* send what is hopefully the correct signals */ if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) { /* wake up one thread in wait_for_end_of_inevitable_transaction() */ - cond_signal(C_INEVITABLE); + STM_PSEGMENT->transaction_state = TS_NONE; + cond_broadcast(C_INEVITABLE); if (globally_unique_transaction) committed_globally_unique_transaction(); } From noreply at buildbot.pypy.org Wed Mar 18 11:43:00 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 18 Mar 2015 11:43:00 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: import stmgc/7861b2a77263 Message-ID: <20150318104300.458121C0221@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76454:d54c94dbf79c Date: 2015-03-18 11:41 +0100 http://bitbucket.org/pypy/pypy/changeset/d54c94dbf79c/ Log: import stmgc/7861b2a77263 diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision --- a/rpython/translator/stm/src_stm/revision +++ b/rpython/translator/stm/src_stm/revision @@ -1,1 +1,1 @@ -a4fc9f31f925 +7861b2a77263 diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c --- a/rpython/translator/stm/src_stm/stm/core.c +++ b/rpython/translator/stm/src_stm/stm/core.c @@ -889,7 +889,8 @@ /* send what is hopefully the correct signals */ if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) { /* wake up one thread in wait_for_end_of_inevitable_transaction() */ - cond_signal(C_INEVITABLE); + STM_PSEGMENT->transaction_state = TS_NONE; + cond_broadcast(C_INEVITABLE); if (globally_unique_transaction) committed_globally_unique_transaction(); } From noreply at buildbot.pypy.org Wed Mar 18 16:16:46 2015 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 18 Mar 2015 16:16:46 +0100 (CET) Subject: [pypy-commit] pypy dtrace-support: try to support debug_probe Message-ID: <20150318151646.98FA61C0221@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: dtrace-support Changeset: r76455:5190ebff847e Date: 2015-03-18 17:16 +0200 http://bitbucket.org/pypy/pypy/changeset/5190ebff847e/ Log: try to support debug_probe diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -557,6 +557,8 @@ # __________ debugging __________ 'debug_view': LLOp(), 'debug_print': LLOp(canrun=True), + 'debug_probe': LLOp(canrun=True), + # dtrace or similar probe call 'debug_start': LLOp(canrun=True), 'debug_stop': LLOp(canrun=True), 'have_debug_prints': LLOp(canrun=True), diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py --- a/rpython/rtyper/lltypesystem/opimpl.py +++ b/rpython/rtyper/lltypesystem/opimpl.py @@ -594,6 +594,9 @@ def op_debug_flush(): pass +def op_debug_probe(probename, arg): + pass + def op_have_debug_prints(): return debug.have_debug_prints() diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py --- a/rpython/translator/c/funcgen.py +++ b/rpython/translator/c/funcgen.py @@ -808,6 +808,14 @@ def OP_DEBUG_STOP(self, op): return self._op_debug('PYPY_DEBUG_STOP', op.args[0]) + def OP_DEBUG_PROBE(self, op): + if not self.db.translator.config.translation.dtrace: + return + assert isinstance(op.args[0], Constant) + val = ''.join(op.args[0].value.chars) + string_literal = c_string_constant(val) + return 'RPYTHON_%s(%d);' % (string_literal, self.expr(op.args[1])) + def OP_DEBUG_ASSERT(self, op): return 'RPyAssert(%s, %s);' % (self.expr(op.args[0]), c_string_constant(op.args[1].value)) diff --git a/rpython/translator/c/test/test_dtrace.py b/rpython/translator/c/test/test_dtrace.py --- a/rpython/translator/c/test/test_dtrace.py +++ b/rpython/translator/c/test/test_dtrace.py @@ -1,12 +1,18 @@ -import subprocess +import subprocess, py, sys from rpython.translator.c.test.test_standalone import StandaloneTests from rpython.rlib.debug import debug_start, debug_stop from rpython.config.translationoption import get_combined_translation_config +from rpython.rtyper.lltypesystem.lloperation import llop +from rpython.rtyper.lltypesystem import lltype class TestDTrace(StandaloneTests): config = get_combined_translation_config(translating=True) config.translation.dtrace = True + + def setup_class(cls): + if sys.platform not in ['freebsd', 'darwin']: + py.test.skip("not supported on other platforms") def test_dtrace_probes(self): def f(argv): @@ -24,3 +30,16 @@ out = p.stdout.read() assert 'pypy_g_f:x-start' in out assert 'pypy_g_f:x-end' in out + + def test_debug_probe(self): + def f(argv): + llop.debug_probe(lltype.Void, "foo", 13) + return 0 + + _, cbuilder = self.compile(f) + exe = cbuilder.executable_name + p = subprocess.Popen(['dtrace', '-n', ':' + exe.basename + '::', + '-c', str(exe)], stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out = p.stdout.read() + assert 'pypy_g_f:foo' in out From noreply at buildbot.pypy.org Wed Mar 18 16:34:48 2015 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 18 Mar 2015 16:34:48 +0100 (CET) Subject: [pypy-commit] pypy dtrace-support: write support for debug_probe Message-ID: <20150318153448.94B001C0221@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: dtrace-support Changeset: r76456:311c07895ec9 Date: 2015-03-18 16:34 +0100 http://bitbucket.org/pypy/pypy/changeset/311c07895ec9/ Log: write support for debug_probe diff --git a/rpython/translator/c/database.py b/rpython/translator/c/database.py --- a/rpython/translator/c/database.py +++ b/rpython/translator/c/database.py @@ -48,7 +48,7 @@ self.containerstats = {} self.externalfuncs = {} self.helper2ptr = {} - self.debug_nodes = set() + self.debug_nodes = {} # late_initializations is for when the value you want to # assign to a constant object is something C doesn't think is @@ -234,12 +234,12 @@ else: raise Exception("don't know about %r" % (obj,)) - def seen_debug_start(self, op): + def seen_debug_node(self, op, suffix, args=''): arg = op.args[0] if not isinstance(arg, Constant): return - name = ''.join(arg.value.chars) - self.debug_nodes.add(name) + name = ''.join(arg.value.chars) + suffix + self.debug_nodes[name] = args def complete(self, show_progress=True): assert not self.completed diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py --- a/rpython/translator/c/funcgen.py +++ b/rpython/translator/c/funcgen.py @@ -802,7 +802,8 @@ return x def OP_DEBUG_START(self, op): - self.db.seen_debug_start(op) + self.db.seen_debug_node(op, '__start') + self.db.seen_debug_node(op, '__end') return self._op_debug('PYPY_DEBUG_START', op.args[0]) def OP_DEBUG_STOP(self, op): @@ -811,10 +812,11 @@ def OP_DEBUG_PROBE(self, op): if not self.db.translator.config.translation.dtrace: return + self.db.seen_debug_node(op, '', 'long') assert isinstance(op.args[0], Constant) val = ''.join(op.args[0].value.chars) - string_literal = c_string_constant(val) - return 'RPYTHON_%s(%d);' % (string_literal, self.expr(op.args[1])) + string_literal = val.upper() + return 'RPYTHON_%s(%s);' % (string_literal, self.expr(op.args[1])) def OP_DEBUG_ASSERT(self, op): return 'RPyAssert(%s, %s);' % (self.expr(op.args[0]), diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py --- a/rpython/translator/c/genc.py +++ b/rpython/translator/c/genc.py @@ -256,10 +256,9 @@ name = self.targetdir.join('rpython.d') f = name.open('w') f.write('provider rpython {\n') - for debug_node in debug_nodes: + for debug_node, args in debug_nodes.iteritems(): debug_node = debug_node.replace('-', '_') - f.write(' probe %s__start();\n' % debug_node) - f.write(' probe %s__end();\n' % debug_node) + f.write(' probe %s(%s);\n' % (debug_node, args)) f.write('};\n') f.close() returncode, stdout, stderr = runsubprocess.run_subprocess( diff --git a/rpython/translator/c/test/test_dtrace.py b/rpython/translator/c/test/test_dtrace.py --- a/rpython/translator/c/test/test_dtrace.py +++ b/rpython/translator/c/test/test_dtrace.py @@ -11,7 +11,8 @@ config.translation.dtrace = True def setup_class(cls): - if sys.platform not in ['freebsd', 'darwin']: + if not (sys.platform.startswith('freebsd') or + sys.platform.startswith('darwin')): py.test.skip("not supported on other platforms") def test_dtrace_probes(self): From noreply at buildbot.pypy.org Wed Mar 18 17:48:09 2015 From: noreply at buildbot.pypy.org (vext01) Date: Wed, 18 Mar 2015 17:48:09 +0100 (CET) Subject: [pypy-commit] pypy default: Add missing __init__.py files. Message-ID: <20150318164809.D71831C0455@cobra.cs.uni-duesseldorf.de> Author: Edd Barrett Branch: Changeset: r76457:31f9be34f0b2 Date: 2015-03-18 16:48 +0000 http://bitbucket.org/pypy/pypy/changeset/31f9be34f0b2/ Log: Add missing __init__.py files. Enables (e.g.) `py.test pypy/` to work. Fix proposed by Ronan, thanks. diff --git a/pypy/module/_csv/test/__init__.py b/pypy/module/_csv/test/__init__.py new file mode 100644 diff --git a/pypy/module/_io/test/__init__.py b/pypy/module/_io/test/__init__.py new file mode 100644 diff --git a/pypy/module/_multiprocessing/test/__init__.py b/pypy/module/_multiprocessing/test/__init__.py new file mode 100644 diff --git a/pypy/module/_ssl/test/__init__.py b/pypy/module/_ssl/test/__init__.py new file mode 100644 diff --git a/pypy/module/itertools/test/__init__.py b/pypy/module/itertools/test/__init__.py new file mode 100644 diff --git a/pypy/module/pwd/test/__init__.py b/pypy/module/pwd/test/__init__.py new file mode 100644 diff --git a/pypy/module/select/test/__init__.py b/pypy/module/select/test/__init__.py new file mode 100644 diff --git a/pypy/module/struct/test/__init__.py b/pypy/module/struct/test/__init__.py new file mode 100644 From noreply at buildbot.pypy.org Wed Mar 18 18:19:15 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 18 Mar 2015 18:19:15 +0100 (CET) Subject: [pypy-commit] pypy default: Complain more cleanly about reversed(x) or enumerate(x) for objects of Message-ID: <20150318171915.CBD2E1C00FA@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76458:31c12715fa13 Date: 2015-03-18 18:19 +0100 http://bitbucket.org/pypy/pypy/changeset/31c12715fa13/ Log: Complain more cleanly about reversed(x) or enumerate(x) for objects of unsupported type diff --git a/rpython/rtyper/lltypesystem/rlist.py b/rpython/rtyper/lltypesystem/rlist.py --- a/rpython/rtyper/lltypesystem/rlist.py +++ b/rpython/rtyper/lltypesystem/rlist.py @@ -57,7 +57,7 @@ elif variant == ("reversed",): return ReversedListIteratorRepr(self) else: - raise NotImplementedError(variant) + raise TyperError("unsupported %r iterator over a list" % (variant,)) def get_itemarray_lowleveltype(self): ITEM = self.item_repr.lowleveltype diff --git a/rpython/rtyper/lltypesystem/rrange.py b/rpython/rtyper/lltypesystem/rrange.py --- a/rpython/rtyper/lltypesystem/rrange.py +++ b/rpython/rtyper/lltypesystem/rrange.py @@ -59,7 +59,10 @@ self.ll_newrange = ll_newrange self.ll_newrangest = ll_newrangest - def make_iterator_repr(self): + def make_iterator_repr(self, variant=None): + if variant is not None: + raise TyperError("unsupported %r iterator over a range list" % + (variant,)) return RangeIteratorRepr(self) diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py --- a/rpython/rtyper/lltypesystem/rstr.py +++ b/rpython/rtyper/lltypesystem/rstr.py @@ -156,7 +156,10 @@ self.CACHE[value] = p return p - def make_iterator_repr(self): + def make_iterator_repr(self, variant=None): + if variant is not None: + raise TyperError("unsupported %r iterator over a str/unicode" % + (variant,)) return self.repr.iterator_repr def can_ll_be_null(self, s_value): diff --git a/rpython/rtyper/rtuple.py b/rpython/rtyper/rtuple.py --- a/rpython/rtyper/rtuple.py +++ b/rpython/rtyper/rtuple.py @@ -210,7 +210,10 @@ ll_str = property(gen_str_function) - def make_iterator_repr(self): + def make_iterator_repr(self, variant=None): + if variant is not None: + raise TyperError("unsupported %r iterator over a tuple" % + (variant,)) if len(self.items_r) == 1: # subclasses are supposed to set the IteratorRepr attribute return self.IteratorRepr(self) diff --git a/rpython/rtyper/test/test_rstr.py b/rpython/rtyper/test/test_rstr.py --- a/rpython/rtyper/test/test_rstr.py +++ b/rpython/rtyper/test/test_rstr.py @@ -116,6 +116,16 @@ res = self.interpret(fn, [1]) assert res == 1 + ord('a') + 10000 + def test_str_iterator_reversed_unsupported(self): + const = self.const + def fn(): + total = 0 + t = const('foo') + for x in reversed(t): + total += ord(x) + return total + py.test.raises(TyperError, self.interpret, fn, []) + def test_char_constant(self): const = self.const def fn(s): diff --git a/rpython/rtyper/test/test_rtuple.py b/rpython/rtyper/test/test_rtuple.py --- a/rpython/rtyper/test/test_rtuple.py +++ b/rpython/rtyper/test/test_rtuple.py @@ -1,8 +1,10 @@ +import py from rpython.rtyper.rtuple import TUPLE_TYPE, TupleRepr from rpython.rtyper.lltypesystem.lltype import Signed, Bool from rpython.rtyper.rbool import bool_repr from rpython.rtyper.rint import signed_repr from rpython.rtyper.test.tool import BaseRtypingTest +from rpython.rtyper.error import TyperError from rpython.rlib.objectmodel import compute_hash from rpython.translator.translator import TranslationContext @@ -228,6 +230,15 @@ res = self.interpret(f, [93813]) assert res == 93813 + def test_tuple_iterator_reversed_unsupported(self): + def f(i): + total = 0 + t = (i,) + for x in reversed(t): + total += x + return total + py.test.raises(TyperError, self.interpret, f, [93813]) + def test_inst_tuple_iter(self): class A: pass From noreply at buildbot.pypy.org Wed Mar 18 18:22:31 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 18 Mar 2015 18:22:31 +0100 (CET) Subject: [pypy-commit] pypy default: Missing imports Message-ID: <20150318172232.00B341C00FA@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76459:e4fbc76c0922 Date: 2015-03-18 18:22 +0100 http://bitbucket.org/pypy/pypy/changeset/e4fbc76c0922/ Log: Missing imports diff --git a/rpython/rtyper/lltypesystem/rlist.py b/rpython/rtyper/lltypesystem/rlist.py --- a/rpython/rtyper/lltypesystem/rlist.py +++ b/rpython/rtyper/lltypesystem/rlist.py @@ -1,6 +1,7 @@ from rpython.rlib import rgc, jit, types from rpython.rlib.debug import ll_assert from rpython.rlib.signature import signature +from rpython.rtyper.error import TyperError from rpython.rtyper.lltypesystem import rstr from rpython.rtyper.lltypesystem.lltype import (GcForwardReference, Ptr, GcArray, GcStruct, Void, Signed, malloc, typeOf, nullptr, typeMethod) diff --git a/rpython/rtyper/lltypesystem/rrange.py b/rpython/rtyper/lltypesystem/rrange.py --- a/rpython/rtyper/lltypesystem/rrange.py +++ b/rpython/rtyper/lltypesystem/rrange.py @@ -1,5 +1,6 @@ from rpython.rtyper.lltypesystem.lltype import Ptr, GcStruct, Signed, malloc, Void from rpython.rtyper.rrange import AbstractRangeRepr, AbstractRangeIteratorRepr +from rpython.rtyper.error import TyperError # ____________________________________________________________ # From noreply at buildbot.pypy.org Wed Mar 18 18:26:42 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Wed, 18 Mar 2015 18:26:42 +0100 (CET) Subject: [pypy-commit] pypy exc-later: Add a hack to make implicit Exception exitcases around function calls explicit again Message-ID: <20150318172642.B44191C0173@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: exc-later Changeset: r76460:53e4a161bd86 Date: 2015-03-18 17:22 +0000 http://bitbucket.org/pypy/pypy/changeset/53e4a161bd86/ Log: Add a hack to make implicit Exception exitcases around function calls explicit again diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py --- a/rpython/translator/simplify.py +++ b/rpython/translator/simplify.py @@ -352,25 +352,34 @@ flowcontext.py). """ for block in list(graph.iterblocks()): - for i in range(len(block.exits)-1, -1, -1): - exit = block.exits[i] - if not (exit.target is graph.exceptblock and - exit.args[0] == Constant(AssertionError)): + for i in range(len(block.exits)-1, -1, -1): + exit = block.exits[i] + if not (exit.target is graph.exceptblock and + exit.args[0] == Constant(AssertionError)): + continue + # can we remove this exit without breaking the graph? + if len(block.exits) < 2: + break + if block.canraise: + if exit.exitcase is None: + break + elif (exit.exitcase is Exception and + block.raising_op.opname + in ('simple_call', 'call_args')): + v_etype = Variable('last_exception') + v_exc = Variable('last_exc_value') + exit.args = [v_etype, v_exc] + exit.last_exception = v_etype + exit.last_exc_value = v_exc continue - # can we remove this exit without breaking the graph? - if len(block.exits) < 2: - break - if block.canraise: - if exit.exitcase is None: - break - if len(block.exits) == 2: - # removing the last non-exceptional exit - block.exitswitch = None - exit.exitcase = None - # remove this exit - lst = list(block.exits) - del lst[i] - block.recloseblock(*lst) + if len(block.exits) == 2: + # removing the last non-exceptional exit + block.exitswitch = None + exit.exitcase = None + # remove this exit + lst = list(block.exits) + del lst[i] + block.recloseblock(*lst) # _____________________________________________________________________ From noreply at buildbot.pypy.org Wed Mar 18 20:09:47 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Wed, 18 Mar 2015 20:09:47 +0100 (CET) Subject: [pypy-commit] pypy exc-later: fix annotation for os.stat(fname)[i] Message-ID: <20150318190947.A73211C0173@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: exc-later Changeset: r76461:d21442bffbcf Date: 2015-03-18 19:10 +0000 http://bitbucket.org/pypy/pypy/changeset/d21442bffbcf/ Log: fix annotation for os.stat(fname)[i] diff --git a/rpython/rtyper/module/ll_os_stat.py b/rpython/rtyper/module/ll_os_stat.py --- a/rpython/rtyper/module/ll_os_stat.py +++ b/rpython/rtyper/module/ll_os_stat.py @@ -131,6 +131,7 @@ assert 0 <= index < N_INDEXABLE_FIELDS, "os.stat()[index] out of range" name, TYPE = STAT_FIELDS[index] return lltype_to_annotation(TYPE) + getitem.can_only_throw = [] class __extend__(pairtype(SomeStatvfsResult, annmodel.SomeInteger)): @@ -138,6 +139,7 @@ assert s_int.is_constant() name, TYPE = STATVFS_FIELDS[s_int.const] return lltype_to_annotation(TYPE) + getitem.can_only_throw = [] s_StatResult = SomeStatResult() From noreply at buildbot.pypy.org Thu Mar 19 09:12:02 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 19 Mar 2015 09:12:02 +0100 (CET) Subject: [pypy-commit] pypy.org extradoc: update the values Message-ID: <20150319081202.08D901C019D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r582:11f7ea9d253a Date: 2015-03-19 09:12 +0100 http://bitbucket.org/pypy/pypy.org/changeset/11f7ea9d253a/ Log: update the values diff --git a/don1.html b/don1.html --- a/don1.html +++ b/don1.html @@ -15,7 +15,7 @@ - $58925 of $105000 (56.1%) + $58935 of $105000 (56.1%)
From noreply at buildbot.pypy.org Thu Mar 19 10:24:40 2015 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 19 Mar 2015 10:24:40 +0100 (CET) Subject: [pypy-commit] pypy vmprof: shuffle do_get_objects to rgc in a bit more general way Message-ID: <20150319092440.C2FB11C019D@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76462:b4cd3d16f845 Date: 2015-03-19 11:24 +0200 http://bitbucket.org/pypy/pypy/changeset/b4cd3d16f845/ Log: shuffle do_get_objects to rgc in a bit more general way diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py --- a/pypy/module/gc/referents.py +++ b/pypy/module/gc/referents.py @@ -44,30 +44,6 @@ return OperationError(space.w_NotImplementedError, space.wrap("operation not implemented by this GC")) -# ____________________________________________________________ - -def clear_gcflag_extra(fromlist): - pending = fromlist[:] - while pending: - gcref = pending.pop() - if rgc.get_gcflag_extra(gcref): - rgc.toggle_gcflag_extra(gcref) - pending.extend(rgc.get_rpy_referents(gcref)) - -def do_get_objects(): - roots = [gcref for gcref in rgc.get_rpy_roots() if gcref] - pending = roots[:] - result_w = [] - while pending: - gcref = pending.pop() - if not rgc.get_gcflag_extra(gcref): - rgc.toggle_gcflag_extra(gcref) - w_obj = try_cast_gcref_to_w_root(gcref) - if w_obj is not None: - result_w.append(w_obj) - pending.extend(rgc.get_rpy_referents(gcref)) - clear_gcflag_extra(roots) - return result_w # ____________________________________________________________ @@ -116,8 +92,8 @@ break # done. Clear flags carefully rgc.toggle_gcflag_extra(gcarg) - clear_gcflag_extra(roots) - clear_gcflag_extra([gcarg]) + rgc.clear_gcflag_extra(roots) + rgc.clear_gcflag_extra([gcarg]) return result_w # ____________________________________________________________ @@ -189,8 +165,7 @@ """Return a list of all app-level objects.""" if not rgc.has_gcflag_extra(): raise missing_operation(space) - result_w = do_get_objects() - rgc.assert_no_more_gcflags() + result_w = rgc.do_get_objects(try_cast_gcref_to_w_root) return space.newlist(result_w) def get_referents(space, args_w): diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -692,3 +692,31 @@ lambda_func = hop.args_s[1].const hop.exception_cannot_occur() hop.rtyper.custom_trace_funcs.append((TP, lambda_func())) + + at specialize.arg(0) +def do_get_objects(callback): + """ Get all the objects that satisfy callback(gcref) -> True + """ + roots = [gcref for gcref in get_rpy_roots() if gcref] + pending = roots[:] + result_w = [] + while pending: + gcref = pending.pop() + if not get_gcflag_extra(gcref): + toggle_gcflag_extra(gcref) + w_obj = callback(gcref) + if w_obj is not None: + result_w.append(w_obj) + pending.extend(get_rpy_referents(gcref)) + clear_gcflag_extra(roots) + assert_no_more_gcflags() + return result_w + + +def clear_gcflag_extra(fromlist): + pending = fromlist[:] + while pending: + gcref = pending.pop() + if get_gcflag_extra(gcref): + toggle_gcflag_extra(gcref) + pending.extend(get_rpy_referents(gcref)) From noreply at buildbot.pypy.org Thu Mar 19 10:24:42 2015 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 19 Mar 2015 10:24:42 +0100 (CET) Subject: [pypy-commit] pypy vmprof: merge Message-ID: <20150319092442.6F4F31C019D@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76463:b1f4cf8e7d91 Date: 2015-03-19 11:24 +0200 http://bitbucket.org/pypy/pypy/changeset/b1f4cf8e7d91/ Log: merge diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -141,4 +141,4 @@ try: return space.wrap(ec.code_info_file.read()) finally: - ec.code_info_file.seek(2, 0) + ec.code_info_file.seek(0, 2) diff --git a/pypy/module/_vmprof/src/fake_pypy_api.c b/pypy/module/_vmprof/src/fake_pypy_api.c --- a/pypy/module/_vmprof/src/fake_pypy_api.c +++ b/pypy/module/_vmprof/src/fake_pypy_api.c @@ -1,25 +1,15 @@ - -long pypy_jit_start_addr(void) -{ - return 3; -} - -long pypy_jit_end_addr(void) -{ - return 3; -} long pypy_jit_stack_depth_at_loc(long x) { return 0; } -long pypy_find_codemap_at_addr(long x) +void *pypy_find_codemap_at_addr(long x) { - return 0; + return (void *)0; } -long pypy_yield_codemap_at_addr(long x, long y, long *a) +long pypy_yield_codemap_at_addr(void *x, long y, long *a) { return 0; } diff --git a/pypy/module/_vmprof/src/get_custom_offset.c b/pypy/module/_vmprof/src/get_custom_offset.c --- a/pypy/module/_vmprof/src/get_custom_offset.c +++ b/pypy/module/_vmprof/src/get_custom_offset.c @@ -1,12 +1,12 @@ - -long pypy_jit_start_addr(); -long pypy_jit_end_addr(); -long pypy_jit_stack_depth_at_loc(long); -long pypy_find_codemap_at_addr(long); -long pypy_yield_codemap_at_addr(long, long, long*); extern volatile int pypy_codemap_currently_invalid; +void *pypy_find_codemap_at_addr(long addr); +long pypy_yield_codemap_at_addr(void *codemap_raw, long addr, + long *current_pos_addr); +long pypy_jit_stack_depth_at_loc(long loc); + + void vmprof_set_tramp_range(void* start, void* end) { } @@ -18,36 +18,26 @@ static ptrdiff_t vmprof_unw_get_custom_offset(void* ip, unw_cursor_t *cp) { intptr_t ip_l = (intptr_t)ip; - - if (ip_l < pypy_jit_start_addr() || ip_l > pypy_jit_end_addr()) { - return -1; - } - return (void*)pypy_jit_stack_depth_at_loc(ip_l); + return pypy_jit_stack_depth_at_loc(ip_l); } static long vmprof_write_header_for_jit_addr(void **result, long n, void *ip, int max_depth) { - long codemap_pos; + void *codemap; long current_pos = 0; intptr_t id; intptr_t addr = (intptr_t)ip; - if (addr < pypy_jit_start_addr() || addr > pypy_jit_end_addr()) { + codemap = pypy_find_codemap_at_addr(addr); + if (codemap == NULL) return n; + + while (n < max_depth) { + id = pypy_yield_codemap_at_addr(codemap, addr, ¤t_pos); + if (id == 0) + break; + result[n++] = (void *)id; } - codemap_pos = pypy_find_codemap_at_addr(addr); - if (codemap_pos == -1) { - return n; - } - while (1) { - id = pypy_yield_codemap_at_addr(codemap_pos, addr, ¤t_pos); - if (id == 0) { - return n; - } - result[n++] = id; - if (n >= max_depth) { - return n; - } - } + return n; } diff --git a/rpython/jit/backend/llsupport/asmmemmgr.py b/rpython/jit/backend/llsupport/asmmemmgr.py --- a/rpython/jit/backend/llsupport/asmmemmgr.py +++ b/rpython/jit/backend/llsupport/asmmemmgr.py @@ -314,7 +314,8 @@ assert gcrootmap is not None for pos, mark in self.gcroot_markers: gcrootmap.register_asm_addr(rawstart + pos, mark) - cpu.codemap.register_frame_depth_map(rawstart, self.frame_positions, + cpu.codemap.register_frame_depth_map(rawstart, rawstart + size, + self.frame_positions, self.frame_assignments) self.frame_positions = None self.frame_assignments = None diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py --- a/rpython/jit/backend/llsupport/codemap.py +++ b/rpython/jit/backend/llsupport/codemap.py @@ -9,6 +9,7 @@ """ +import os from rpython.rlib import rgc from rpython.rlib.objectmodel import specialize, we_are_translated from rpython.rlib.entrypoint import jit_entrypoint @@ -16,158 +17,55 @@ from rpython.rlib.rbisect import bisect_left, bisect_left_addr from rpython.rtyper.lltypesystem import lltype, rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo +from rpython.translator import cdir -INT_LIST = rffi.CArray(lltype.Signed) -CODEMAP = lltype.Struct( - 'pypy_codemap_item', - ('addr', lltype.Signed), - ('machine_code_size', lltype.Signed), - ('bytecode_info_size', lltype.Signed), - ('bytecode_info', lltype.Ptr(INT_LIST)), - hints=dict(external=True, c_name='pypy_codemap_item')) -CODEMAP_LIST = rffi.CArray(CODEMAP) +INT_LIST_PTR = rffi.CArrayPtr(lltype.Signed) -CODEMAP_STORAGE = lltype.Struct( - 'pypy_codemap_storage', - ('jit_addr_map_used', lltype.Signed), - ('jit_frame_depth_map_used', lltype.Signed), - ('jit_codemap_used', lltype.Signed), - ('jit_addr_map', lltype.Ptr(INT_LIST)), - ('jit_frame_depth_map', lltype.Ptr(INT_LIST)), - ('jit_codemap', lltype.Ptr(CODEMAP_LIST)), - hints=dict(external=True, c_name='pypy_codemap_storage')) -CODEMAP_GCARRAY = lltype.GcArray(CODEMAP) - -_codemap = None +srcdir = os.path.join(os.path.dirname(__file__), 'src') eci = ExternalCompilationInfo(post_include_bits=[""" -RPY_EXTERN volatile int pypy_codemap_currently_invalid; -RPY_EXTERN void pypy_codemap_invalid_set(int); +#include +RPY_EXTERN long pypy_jit_codemap_add(uintptr_t addr, + unsigned int machine_code_size, + long *bytecode_info, + unsigned int bytecode_info_size); +RPY_EXTERN long *pypy_jit_codemap_del(uintptr_t addr); +RPY_EXTERN uintptr_t pypy_jit_codemap_firstkey(void); +RPY_EXTERN void *pypy_find_codemap_at_addr(long addr); +RPY_EXTERN long pypy_yield_codemap_at_addr(void *codemap_raw, long addr, + long *current_pos_addr); -typedef struct pypy_codemap_item { - long addr, machine_code_size, bytecode_info_size; - long* bytecode_info; -} pypy_codemap_item; +RPY_EXTERN long pypy_jit_depthmap_add(uintptr_t addr, unsigned int size, + unsigned int stackdepth); +RPY_EXTERN void pypy_jit_depthmap_clear(uintptr_t addr, unsigned int size); -typedef struct pypy_codemap_storage { - long jit_addr_map_used; - long jit_frame_depth_map_used; - long jit_codemap_used; - long* jit_addr_map; - long* jit_frame_depth_map; - pypy_codemap_item* jit_codemap; -} pypy_codemap_storage; - -RPY_EXTERN pypy_codemap_storage *pypy_get_codemap_storage(); -RPY_EXTERN long pypy_jit_stack_depth_at_loc(long loc); -RPY_EXTERN long pypy_find_codemap_at_addr(long addr); -RPY_EXTERN long pypy_jit_start_addr(void); -RPY_EXTERN long pypy_jit_end_addr(void); -RPY_EXTERN long pypy_yield_codemap_at_addr(long, long, long*); - -"""], separate_module_sources=[""" -volatile int pypy_codemap_currently_invalid = 0; - -static pypy_codemap_storage pypy_cs_g; - -long bisect_right(long *a, long x, long hi) -{ - long lo, mid; - lo = 0; - while (lo < hi) { - mid = (lo+hi) / 2; - if (x < a[mid]) { hi = mid; } - else { lo = mid+1; } - } - return lo; -} - -long bisect_right_addr(pypy_codemap_item *a, long x, long hi) -{ - long lo, mid; - lo = 0; - while (lo < hi) { - mid = (lo+hi) / 2; - if (x < a[mid].addr) { hi = mid; } - else { lo = mid+1; } - } - return lo; -} - -long pypy_jit_stack_depth_at_loc(long loc) -{ - long pos; - pos = bisect_right(pypy_cs_g.jit_addr_map, loc, - pypy_cs_g.jit_addr_map_used); - if (pos == 0 || pos == pypy_cs_g.jit_addr_map_used) - return -1; - return pypy_cs_g.jit_frame_depth_map[pos - 1]; -} - -long pypy_find_codemap_at_addr(long addr) -{ - return bisect_right_addr(pypy_cs_g.jit_codemap, addr, - pypy_cs_g.jit_codemap_used) - 1; -} - -long pypy_jit_start_addr(void) -{ - return pypy_cs_g.jit_addr_map[0]; -} - -long pypy_jit_end_addr(void) -{ - return pypy_cs_g.jit_addr_map[pypy_cs_g.jit_addr_map_used - 1]; -} - -long pypy_yield_codemap_at_addr(long codemap_no, long addr, - long* current_pos_addr) -{ - // will return consecutive unique_ids from codemap, starting from position - // `pos` until addr - pypy_codemap_item *codemap = &(pypy_cs_g.jit_codemap[codemap_no]); - long current_pos = *current_pos_addr; - long start_addr = codemap->addr; - long rel_addr = addr - start_addr; - long next_start, next_stop; - - while (1) { - if (current_pos >= codemap->bytecode_info_size) - return 0; - next_start = codemap->bytecode_info[current_pos + 1]; - if (next_start > rel_addr) - return 0; - next_stop = codemap->bytecode_info[current_pos + 2]; - if (next_stop > rel_addr) { - *current_pos_addr = current_pos + 4; - return codemap->bytecode_info[current_pos]; - } - // we need to skip potentially more than one - current_pos = codemap->bytecode_info[current_pos + 3]; - } -} - -pypy_codemap_storage *pypy_get_codemap_storage(void) -{ - return &pypy_cs_g; -} - -void pypy_codemap_invalid_set(int value) -{ - pypy_codemap_currently_invalid = value; -} -"""]) +"""], separate_module_sources=[ + open(os.path.join(srcdir, 'skiplist.c'), 'r').read() + + open(os.path.join(srcdir, 'codemap.c'), 'r').read() +], include_dirs=[cdir]) def llexternal(name, args, res): return rffi.llexternal(name, args, res, compilation_info=eci, releasegil=False) -ll_pypy_codemap_invalid_set = llexternal('pypy_codemap_invalid_set', - [rffi.INT], lltype.Void) -pypy_get_codemap_storage = llexternal('pypy_get_codemap_storage', - [], lltype.Ptr(CODEMAP_STORAGE)) +pypy_jit_codemap_add = llexternal('pypy_jit_codemap_add', + [lltype.Signed, lltype.Signed, + INT_LIST_PTR, lltype.Signed], + lltype.Signed) +pypy_jit_codemap_del = llexternal('pypy_jit_codemap_del', + [lltype.Signed], INT_LIST_PTR) +pypy_jit_codemap_firstkey = llexternal('pypy_jit_codemap_firstkey', + [], lltype.Signed) + +pypy_jit_depthmap_add = llexternal('pypy_jit_depthmap_add', + [lltype.Signed, lltype.Signed, + lltype.Signed], lltype.Signed) +pypy_jit_depthmap_clear = llexternal('pypy_jit_depthmap_clear', + [lltype.Signed, lltype.Signed], + lltype.Void) + stack_depth_at_loc = llexternal('pypy_jit_stack_depth_at_loc', [lltype.Signed], lltype.Signed) find_codemap_at_addr = llexternal('pypy_find_codemap_at_addr', @@ -177,186 +75,62 @@ rffi.CArrayPtr(lltype.Signed)], lltype.Signed) -def pypy_codemap_invalid_set(val): - #if we_are_translated(): - ll_pypy_codemap_invalid_set(val) - at specialize.ll() -def copy_item(source, dest, si, di, baseline=0): - TP = lltype.typeOf(dest) - if isinstance(TP.TO.OF, lltype.Struct): - rgc.copy_struct_item(source, dest, si, di) - else: - dest[di] = source[si] + baseline - -class ListStorageMixin(object): - # XXX this code has wrong complexity, we should come up with a better - # data structure ideally - _mixin_ = True - jit_addr_map_allocated = 0 - jit_codemap_allocated = 0 - jit_frame_depth_map_allocated = 0 - - @specialize.arg(1) - def extend_with(self, name, to_insert, pos, baseline=0): - # first check if we need to reallocate - g = pypy_get_codemap_storage() - used = getattr(g, name + '_used') - allocated = getattr(self, name + '_allocated') - lst = getattr(g, name) - if used + len(to_insert) > allocated or pos != used: - old_lst = lst - if used + len(to_insert) > allocated: - new_size = max(4 * allocated, - (allocated + len(to_insert)) * 2) - else: - new_size = allocated - lst = lltype.malloc(lltype.typeOf(lst).TO, new_size, - flavor='raw', - track_allocation=False) - setattr(self, name + '_allocated', new_size) - for i in range(0, pos): - copy_item(old_lst, lst, i, i) - j = 0 - for i in range(pos, pos + len(to_insert)): - copy_item(to_insert, lst, j, i, baseline) - j += 1 - j = pos - for i in range(pos + len(to_insert), len(to_insert) + used): - copy_item(old_lst, lst, j, i) - j += 1 - self.free_lst(name, old_lst) - else: - for i in range(len(to_insert)): - copy_item(to_insert, lst, i, i + pos, baseline) - setattr(g, name, lst) - setattr(g, name + '_used', len(to_insert) + used) - - @specialize.arg(1) - def remove(self, name, start, end): - g = pypy_get_codemap_storage() - lst = getattr(g, name) - used = getattr(g, name + '_used') - j = end - for i in range(start, used - (end - start)): - info = lltype.nullptr(INT_LIST) - if name == 'jit_codemap': - if i < end: - info = lst[i].bytecode_info - copy_item(lst, lst, j, i) - if name == 'jit_codemap': - if info: - lltype.free(info, flavor='raw', track_allocation=False) - j += 1 - setattr(g, name + '_used', used - (end - start)) - - def free(self): - g = pypy_get_codemap_storage() - # if setup has not been called - if g.jit_addr_map_used: - lltype.free(g.jit_addr_map, flavor='raw', track_allocation=False) - g.jit_addr_map_used = 0 - g.jit_addr_map = lltype.nullptr(INT_LIST) - i = 0 - while i < g.jit_codemap_used: - lltype.free(g.jit_codemap[i].bytecode_info, flavor='raw', - track_allocation=False) - i += 1 - if g.jit_codemap_used: - lltype.free(g.jit_codemap, flavor='raw', - track_allocation=False) - g.jit_codemap_used = 0 - g.jit_codemap = lltype.nullptr(CODEMAP_LIST) - if g.jit_frame_depth_map_used: - lltype.free(g.jit_frame_depth_map, flavor='raw', - track_allocation=False) - g.jit_frame_depth_map_used = 0 - g.jit_frame_depth_map = lltype.nullptr(INT_LIST) - - @specialize.arg(1) - def free_lst(self, name, lst): - if lst: - lltype.free(lst, flavor='raw', track_allocation=False) - -class CodemapStorage(ListStorageMixin): +class CodemapStorage(object): """ An immortal wrapper around underlaying jit codemap data """ def setup(self): - g = pypy_get_codemap_storage() - if g.jit_addr_map_used != 0: - # someone failed to call free(), in tests only anyway + if not we_are_translated(): + # in case someone failed to call free(), in tests only anyway self.free() + def free(self): + while True: + key = pypy_jit_codemap_firstkey() + if not key: + break + items = pypy_jit_codemap_del(key) + lltype.free(items, flavor='raw', track_allocation=False) + def free_asm_block(self, start, stop): - # fix up jit_addr_map - g = pypy_get_codemap_storage() - jit_adr_start = bisect_left(g.jit_addr_map, start, - g.jit_addr_map_used) - jit_adr_stop = bisect_left(g.jit_addr_map, stop, - g.jit_addr_map_used) - pypy_codemap_invalid_set(1) - self.remove('jit_addr_map', jit_adr_start, jit_adr_stop) - self.remove('jit_frame_depth_map', jit_adr_start, jit_adr_stop) - # fix up codemap - # (there should only be zero or one codemap entry in that range, - # but still we use a range to distinguish between zero and one) - codemap_adr_start = bisect_left_addr(g.jit_codemap, start, - g.jit_codemap_used) - codemap_adr_stop = bisect_left_addr(g.jit_codemap, stop, - g.jit_codemap_used) - self.remove('jit_codemap', codemap_adr_start, codemap_adr_stop) - pypy_codemap_invalid_set(0) + items = pypy_jit_codemap_del(start) + if items: + lltype.free(items, flavor='raw', track_allocation=False) + pypy_jit_depthmap_clear(start, stop - start) - def register_frame_depth_map(self, rawstart, frame_positions, + def register_frame_depth_map(self, rawstart, rawstop, frame_positions, frame_assignments): if not frame_positions: return - pypy_codemap_invalid_set(1) - g = pypy_get_codemap_storage() - if (not g.jit_addr_map_used or - rawstart > g.jit_addr_map[g.jit_addr_map_used - 1]): - start = g.jit_addr_map_used - self.extend_with('jit_addr_map', frame_positions, - g.jit_addr_map_used, rawstart) - self.extend_with('jit_frame_depth_map', frame_assignments, - g.jit_frame_depth_map_used) - else: - start = bisect_left(g.jit_addr_map, rawstart, - g.jit_addr_map_used) - self.extend_with('jit_addr_map', frame_positions, start, rawstart) - self.extend_with('jit_frame_depth_map', frame_assignments, - start) - pypy_codemap_invalid_set(0) + assert len(frame_positions) == len(frame_assignments) + for i in range(len(frame_positions)-1, -1, -1): + pos = rawstart + frame_positions[i] + length = rawstop - pos + if length > 0: + #print "ADD:", pos, length, frame_assignments[i] + pypy_jit_depthmap_add(pos, length, frame_assignments[i]) + rawstop = pos - def register_codemap(self, codemap): - start = codemap[0] - g = pypy_get_codemap_storage() - pos = bisect_left_addr(g.jit_codemap, start, g.jit_codemap_used) - items = lltype.malloc(INT_LIST, len(codemap[2]), flavor='raw', + def register_codemap(self, (start, size, l)): + items = lltype.malloc(INT_LIST_PTR.TO, len(l), flavor='raw', track_allocation=False) - for i in range(len(codemap[2])): - items[i] = codemap[2][i] - s = lltype.malloc(CODEMAP_GCARRAY, 1) - s[0].addr = codemap[0] - s[0].machine_code_size = codemap[1] - s[0].bytecode_info = items - s[0].bytecode_info_size = len(codemap[2]) - pypy_codemap_invalid_set(1) - self.extend_with('jit_codemap', s, pos) - pypy_codemap_invalid_set(0) + for i in range(len(l)): + items[i] = l[i] + if pypy_jit_codemap_add(start, size, items, len(l)) < 0: + lltype.free(items, flavor='raw', track_allocation=False) def finish_once(self): self.free() def unpack_traceback(addr): - codemap_pos = find_codemap_at_addr(addr) - if codemap_pos == -1: + codemap_raw = find_codemap_at_addr(addr) + if not codemap_raw: return [] # no codemap for that position storage = lltype.malloc(rffi.CArray(lltype.Signed), 1, flavor='raw') storage[0] = 0 res = [] while True: - item = yield_bytecode_at_addr(codemap_pos, addr, storage) + item = yield_bytecode_at_addr(codemap_raw, addr, storage) if item == 0: break res.append(item) diff --git a/rpython/jit/backend/llsupport/src/codemap.c b/rpython/jit/backend/llsupport/src/codemap.c new file mode 100644 --- /dev/null +++ b/rpython/jit/backend/llsupport/src/codemap.c @@ -0,0 +1,196 @@ +#include "src/precommondefs.h" + +#ifndef HAS_SKIPLIST +# error "skiplist.c needs to be included before" +#endif + +volatile int pypy_codemap_currently_invalid = 0; + +void pypy_codemap_invalid_set(int value) +{ + if (value) + __sync_lock_test_and_set(&pypy_codemap_currently_invalid, 1); + else + __sync_lock_release(&pypy_codemap_currently_invalid); +} + + +/************************************************************/ +/*** codemap storage ***/ +/************************************************************/ + +typedef struct { + unsigned int machine_code_size; + unsigned int bytecode_info_size; + long *bytecode_info; +} codemap_data_t; + +static skipnode_t jit_codemap_head; + +/*** interface used from codemap.py ***/ + +RPY_EXTERN +long pypy_jit_codemap_add(uintptr_t addr, unsigned int machine_code_size, + long *bytecode_info, unsigned int bytecode_info_size) +{ + skipnode_t *new = skiplist_malloc(sizeof(codemap_data_t)); + codemap_data_t *data; + if (new == NULL) + return -1; /* too bad */ + + new->key = addr; + data = (codemap_data_t *)new->data; + data->machine_code_size = machine_code_size; + data->bytecode_info = bytecode_info; + data->bytecode_info_size = bytecode_info_size; + + pypy_codemap_invalid_set(1); + skiplist_insert(&jit_codemap_head, new); + pypy_codemap_invalid_set(0); + return 0; +} + +RPY_EXTERN +long *pypy_jit_codemap_del(uintptr_t addr) +{ + long *result; + skipnode_t *node; + + pypy_codemap_invalid_set(1); + node = skiplist_remove(&jit_codemap_head, addr); + pypy_codemap_invalid_set(0); + + if (node == NULL) + return NULL; + result = ((codemap_data_t *)node->data)->bytecode_info; + free(node); + return result; +} + +RPY_EXTERN +uintptr_t pypy_jit_codemap_firstkey(void) +{ + return skiplist_firstkey(&jit_codemap_head); +} + +/*** interface used from pypy/module/_vmprof ***/ + +RPY_EXTERN +void *pypy_find_codemap_at_addr(long addr) +{ + skipnode_t *codemap = skiplist_search(&jit_codemap_head, addr); + codemap_data_t *data; + uintptr_t rel_addr; + + if (codemap == &jit_codemap_head) + return NULL; + + rel_addr = (uintptr_t)addr - codemap->key; + data = (codemap_data_t *)codemap->data; + if (rel_addr >= data->machine_code_size) + return NULL; + + return (void *)codemap; +} + +RPY_EXTERN +long pypy_yield_codemap_at_addr(void *codemap_raw, long addr, + long *current_pos_addr) +{ + // will return consecutive unique_ids from codemap, starting from position + // `pos` until addr + skipnode_t *codemap = (skipnode_t *)codemap_raw; + long current_pos = *current_pos_addr; + long rel_addr = addr - codemap->key; + long next_start, next_stop; + codemap_data_t *data = (codemap_data_t *)codemap->data; + + while (1) { + if (current_pos >= data->bytecode_info_size) + return 0; + next_start = data->bytecode_info[current_pos + 1]; + if (next_start > rel_addr) + return 0; + next_stop = data->bytecode_info[current_pos + 2]; + if (next_stop > rel_addr) { + *current_pos_addr = current_pos + 4; + return data->bytecode_info[current_pos]; + } + // we need to skip potentially more than one + current_pos = data->bytecode_info[current_pos + 3]; + } +} + +/************************************************************/ +/*** depthmap storage ***/ +/************************************************************/ + +typedef struct { + unsigned int block_size; + unsigned int stack_depth; +} depthmap_data_t; + +static skipnode_t jit_depthmap_head; + +/*** interface used from codemap.py ***/ + +RPY_EXTERN +long pypy_jit_depthmap_add(uintptr_t addr, unsigned int size, + unsigned int stackdepth) +{ + skipnode_t *new = skiplist_malloc(sizeof(depthmap_data_t)); + depthmap_data_t *data; + if (new == NULL) + return -1; /* too bad */ + + new->key = addr; + data = (depthmap_data_t *)new->data; + data->block_size = size; + data->stack_depth = stackdepth; + + pypy_codemap_invalid_set(1); + skiplist_insert(&jit_depthmap_head, new); + pypy_codemap_invalid_set(0); + return 0; +} + +RPY_EXTERN +void pypy_jit_depthmap_clear(uintptr_t addr, unsigned int size) +{ + uintptr_t search_key = addr + size - 1; + if (size == 0) + return; + + pypy_codemap_invalid_set(1); + while (1) { + /* search for all nodes belonging to the range, and remove them */ + skipnode_t *node = skiplist_search(&jit_depthmap_head, search_key); + if (node->key < addr) + break; /* exhausted */ + skiplist_remove(&jit_depthmap_head, node->key); + free(node); + } + pypy_codemap_invalid_set(0); +} + +/*** interface used from pypy/module/_vmprof ***/ + +RPY_EXTERN +long pypy_jit_stack_depth_at_loc(long loc) +{ + skipnode_t *depthmap = skiplist_search(&jit_depthmap_head, (uintptr_t)loc); + depthmap_data_t *data; + uintptr_t rel_addr; + + if (depthmap == &jit_depthmap_head) + return -1; + + rel_addr = (uintptr_t)loc - depthmap->key; + data = (codemap_data_t *)depthmap->data; + if (rel_addr >= data->block_size) + return -1; + + return data->stack_depth; +} + +/************************************************************/ diff --git a/rpython/jit/backend/llsupport/src/skiplist.c b/rpython/jit/backend/llsupport/src/skiplist.c new file mode 100644 --- /dev/null +++ b/rpython/jit/backend/llsupport/src/skiplist.c @@ -0,0 +1,103 @@ +#include +#include + +#define HAS_SKIPLIST +#define SKIPLIST_HEIGHT 8 + +typedef struct skipnode_s { + uintptr_t key; + char *data; + struct skipnode_s *next[SKIPLIST_HEIGHT]; /* may be smaller */ +} skipnode_t; + +static skipnode_t *skiplist_malloc(uintptr_t datasize) +{ + char *result; + uintptr_t basesize; + uintptr_t length = 1; + while (length < SKIPLIST_HEIGHT && (rand() & 3) == 0) + length++; + basesize = sizeof(skipnode_t) - + (SKIPLIST_HEIGHT - length) * sizeof(skipnode_t *); + result = malloc(basesize + datasize); + if (result != NULL) { + ((skipnode_t *)result)->data = result + basesize; + } + return (skipnode_t *)result; +} + +static skipnode_t *skiplist_search(skipnode_t *head, uintptr_t searchkey) +{ + /* Returns the skipnode with key closest (but <=) searchkey. + Note that if there is no item with key <= searchkey in the list, + this will return the head node. */ + uintptr_t level = SKIPLIST_HEIGHT - 1; + while (1) { + skipnode_t *next = head->next[level]; + if (next != NULL && next->key <= searchkey) { + head = next; + } + else { + if (level == 0) + break; + level -= 1; + } + } + return head; +} + +static void skiplist_insert(skipnode_t *head, skipnode_t *new) +{ + uintptr_t size0 = sizeof(skipnode_t) - + SKIPLIST_HEIGHT * sizeof(skipnode_t *); + uintptr_t height_of_new = (new->data - ((char *)new + size0)) / + sizeof(skipnode_t *); + + uintptr_t level = SKIPLIST_HEIGHT - 1; + uintptr_t searchkey = new->key; + while (1) { + skipnode_t *next = head->next[level]; + if (next != NULL && next->key <= searchkey) { + head = next; + } + else { + if (level < height_of_new) { + new->next[level] = next; + head->next[level] = new; + if (level == 0) + break; + } + level -= 1; + } + } +} + +static skipnode_t *skiplist_remove(skipnode_t *head, uintptr_t exact_key) +{ + uintptr_t level = SKIPLIST_HEIGHT - 1; + while (1) { + skipnode_t *next = head->next[level]; + if (next != NULL && next->key <= exact_key) { + if (next->key == exact_key) { + head->next[level] = next->next[level]; + if (level == 0) + return next; /* successfully removed */ + level -= 1; + } + else + head = next; + } + else { + if (level == 0) + return NULL; /* 'exact_key' not found! */ + level -= 1; + } + } +} + +static uintptr_t skiplist_firstkey(skipnode_t *head) +{ + if (head->next[0] == NULL) + return 0; + return head->next[0]->key; +} diff --git a/rpython/jit/backend/llsupport/test/test_codemap.py b/rpython/jit/backend/llsupport/test/test_codemap.py --- a/rpython/jit/backend/llsupport/test/test_codemap.py +++ b/rpython/jit/backend/llsupport/test/test_codemap.py @@ -1,51 +1,55 @@ from rpython.jit.backend.llsupport.codemap import stack_depth_at_loc -from rpython.jit.backend.llsupport.codemap import CodemapStorage,\ - ListStorageMixin, CodemapBuilder, unpack_traceback,\ - pypy_get_codemap_storage +from rpython.jit.backend.llsupport.codemap import CodemapStorage, \ + CodemapBuilder, unpack_traceback, find_codemap_at_addr -g = pypy_get_codemap_storage() - -def test_list_storage_mixin(): - class X(ListStorageMixin): - def unpack(self): - return [g.jit_addr_map[i] for i in range(g.jit_addr_map_used)] - - x = X() - x.extend_with('jit_addr_map', [1, 2, 3], 0) - assert x.unpack() == [1, 2, 3] - x.extend_with('jit_addr_map', [4, 5, 6], 3) - assert x.unpack() == [1, 2, 3, 4, 5, 6] - x.extend_with('jit_addr_map', [7, 8, 9], 2, baseline=10) - assert x.unpack() == [1, 2, 17, 18, 19, 3, 4, 5, 6] - x.remove('jit_addr_map', 3, 6) - assert x.unpack() == [1, 2, 17, 4, 5, 6] - x.extend_with('jit_addr_map', [1] * 6, 6) - assert x.unpack() == [1, 2, 17, 4, 5, 6, 1, 1, 1, 1, 1, 1] - x.extend_with('jit_addr_map', [10] * 4, 5) - assert x.unpack() == [1, 2, 17, 4, 5, 10, 10, 10, 10, 6, - 1, 1, 1, 1, 1, 1] - x.free() +def test_register_codemap(): + codemap = CodemapStorage() + codemap.setup() + codemap.register_codemap((100, 20, [13, 14, 15])) + codemap.register_codemap((300, 30, [16, 17, 18])) + codemap.register_codemap((200, 100, [19, 20, 21, 22, 23])) + # + raw100 = find_codemap_at_addr(100) + assert find_codemap_at_addr(119) == raw100 + assert not find_codemap_at_addr(120) + # + raw200 = find_codemap_at_addr(200) + assert raw200 != raw100 + assert find_codemap_at_addr(299) == raw200 + # + raw300 = find_codemap_at_addr(329) + assert raw300 != raw100 and raw300 != raw200 + assert find_codemap_at_addr(300) == raw300 + # + codemap.free() def test_find_jit_frame_depth(): codemap = CodemapStorage() codemap.setup() - codemap.register_frame_depth_map(11, [0, 5, 10], [1, 2, 3]) - codemap.register_frame_depth_map(30, [0, 5, 10], [4, 5, 6]) - codemap.register_frame_depth_map(0, [0, 5, 10], [7, 8, 9]) + codemap.register_frame_depth_map(11, 26, [0, 5, 10], [1, 2, 3]) + codemap.register_frame_depth_map(30, 41, [0, 5, 10], [4, 5, 6]) + codemap.register_frame_depth_map(0, 11, [0, 5, 10], [7, 8, 9]) assert stack_depth_at_loc(13) == 1 assert stack_depth_at_loc(-3) == -1 + assert stack_depth_at_loc(40) == 6 assert stack_depth_at_loc(41) == -1 assert stack_depth_at_loc(5) == 8 assert stack_depth_at_loc(17) == 2 assert stack_depth_at_loc(38) == 5 - codemap.free_asm_block(11, 22) - assert stack_depth_at_loc(13) == 9 + assert stack_depth_at_loc(25) == 3 + assert stack_depth_at_loc(26) == -1 + assert stack_depth_at_loc(11) == 1 + assert stack_depth_at_loc(10) == 9 + codemap.free_asm_block(11, 26) + assert stack_depth_at_loc(11) == -1 + assert stack_depth_at_loc(13) == -1 assert stack_depth_at_loc(-3) == -1 + assert stack_depth_at_loc(40) == 6 assert stack_depth_at_loc(41) == -1 assert stack_depth_at_loc(5) == 8 - assert stack_depth_at_loc(17) == 9 assert stack_depth_at_loc(38) == 5 + assert stack_depth_at_loc(10) == 9 codemap.free() def test_codemaps(): diff --git a/rpython/jit/backend/llsupport/test/test_skiplist.py b/rpython/jit/backend/llsupport/test/test_skiplist.py new file mode 100644 --- /dev/null +++ b/rpython/jit/backend/llsupport/test/test_skiplist.py @@ -0,0 +1,89 @@ +import random, os +import cffi + +ffi = cffi.FFI() + +ffi.cdef(""" +typedef struct { + uintptr_t key; + char *data; + ...; +} skipnode_t; + +skipnode_t *skiplist_malloc(uintptr_t datasize); +skipnode_t *skiplist_search(skipnode_t *head, uintptr_t searchkey); +void skiplist_insert(skipnode_t *head, skipnode_t *new); +skipnode_t *skiplist_remove(skipnode_t *head, uintptr_t exact_key); +uintptr_t skiplist_firstkey(skipnode_t *head); +""") + +filename = os.path.join(os.path.dirname(__file__), '..', 'src', 'skiplist.c') +lib = ffi.verify(open(filename).read()) + + +def test_insert_search_remove(): + my_head = ffi.new("skipnode_t *") + assert lib.skiplist_search(my_head, 0) == my_head + # + keys = random.sample(xrange(2, 10**9), 50000) + nodes = {} + for key in keys: + node = lib.skiplist_malloc(4) + node.key = key + ffi.cast("int *", node.data)[0] = key + lib.skiplist_insert(my_head, node) + nodes[key] = node + # + random.shuffle(keys) + for key in keys: + node = lib.skiplist_search(my_head, key) + assert nodes[key] == node + if key + 1 not in nodes: + assert node == lib.skiplist_search(my_head, key + 1) + # + keys.sort() + following = {} + preceeding = {} + for key, next_key in zip(keys[:-1], keys[1:]): + following[key] = next_key + preceeding[next_key] = key + following[0] = keys[0] + following[keys[-1]] = 10**9 + preceeding[keys[0]] = 0 + # + for i in range(100000): + random_key = random.randrange(2, 10**9) + node = lib.skiplist_search(my_head, random_key) + assert node.key <= random_key + if node == my_head: + assert random_key < following[0] + else: + assert node == nodes[node.key] + assert following[node.key] > random_key + # + random_keys = list(keys) + random.shuffle(random_keys) + for i in range(10000): + node = nodes.pop(random_keys.pop()) + prev = preceeding[node.key] + next = following[node.key] + following[prev] = next + preceeding[next] = prev + res = lib.skiplist_remove(my_head, node.key) + assert res == node + if prev == 0: + assert lib.skiplist_search(my_head, node.key) == my_head + else: + assert lib.skiplist_search(my_head, node.key) == nodes[prev] + res = lib.skiplist_remove(my_head, node.key) + assert res == ffi.NULL + # + for i in range(100000): + random_key = random.randrange(2, 10**9) + node = lib.skiplist_search(my_head, random_key) + assert node.key <= random_key + if node == my_head: + assert random_key < following[0] + else: + assert node == nodes[node.key] + assert following[node.key] > random_key diff --git a/rpython/translator/c/gcc/trackgcroot.py b/rpython/translator/c/gcc/trackgcroot.py --- a/rpython/translator/c/gcc/trackgcroot.py +++ b/rpython/translator/c/gcc/trackgcroot.py @@ -1069,6 +1069,7 @@ visit_leaq = FunctionGcRootTracker._visit_lea visit_xorq = FunctionGcRootTracker.binary_insn + visit_xchgl = FunctionGcRootTracker._visit_xchg visit_xchgq = FunctionGcRootTracker._visit_xchg visit_testq = FunctionGcRootTracker._visit_test From noreply at buildbot.pypy.org Thu Mar 19 10:39:28 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 19 Mar 2015 10:39:28 +0100 (CET) Subject: [pypy-commit] pypy default: Tests for issue #2001. Only failure is in Message-ID: <20150319093928.092F01C009D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76464:a8f27887f7d9 Date: 2015-03-19 10:11 +0100 http://bitbucket.org/pypy/pypy/changeset/a8f27887f7d9/ Log: Tests for issue #2001. Only failure is in TestIncrementalMiniMarkGC.test_limited_memory_linux. diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py --- a/rpython/translator/c/test/test_newgc.py +++ b/rpython/translator/c/test/test_newgc.py @@ -2,6 +2,7 @@ import inspect import os import sys +import subprocess import py @@ -50,8 +51,8 @@ t.viewcg() exename = t.compile() - def run(s, i): - data = py.process.cmdexec("%s %s %d" % (exename, s, i)) + def run(s, i, runner=subprocess.check_output): + data = runner([str(exename), str(s), str(i)]) data = data.strip() if data == 'MEMORY-ERROR': raise MemoryError @@ -115,11 +116,11 @@ cls.c_allfuncs.close_isolate() cls.c_allfuncs = None - def run(self, name, *args): + def run(self, name, *args, **kwds): if not args: args = (-1, ) print 'Running %r)' % name - res = self.c_allfuncs(name, *args) + res = self.c_allfuncs(name, *args, **kwds) num = self.name_to_func[name] if self.funcsstr[num]: return res @@ -1524,6 +1525,57 @@ res = self.run("nongc_opaque_attached_to_gc") assert res == 0 + def define_limited_memory(self): + class A(object): + def __init__(self, next): + self.next = next + def g(i): + a = None + while i != 0: + a = A(a) + i -= 1 + return 0 + def f(i): + try: + return g(i) + except MemoryError: + g(20) # allocate some more stuff, but exit + return 42 + return f + + def test_limited_memory(self): + import random + # + def myrunner(args): + env = os.environ.copy() + env['PYPY_GC_MAX'] = '%dKB' % gcmax + return subprocess.check_output(args, env=env) + # + for i in range(10): + gcmax = random.randrange(50000, 100000) + print gcmax + res = self.run("limited_memory", -1, runner=myrunner) + assert res == 42 + + define_limited_memory_linux = define_limited_memory + + def test_limited_memory_linux(self): + if not sys.platform.startswith('linux'): + py.test.skip("linux only") + # + import random + # + def myrunner(args): + args1 = ['/bin/bash', '-c', 'ulimit -v %d && %s' % + (ulimitv, ' '.join(args),)] + return subprocess.check_output(args1) + # + for i in range(10): + ulimitv = random.randrange(50000, 100000) + print ulimitv + res = self.run("limited_memory_linux", -1, runner=myrunner) + assert res == 42 + class TestIncrementalMiniMarkGC(TestMiniMarkGC): gcpolicy = "incminimark" From noreply at buildbot.pypy.org Thu Mar 19 10:39:29 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 19 Mar 2015 10:39:29 +0100 (CET) Subject: [pypy-commit] pypy default: Fix: if minor_collection() raises MemoryError, we used to be left with Message-ID: <20150319093929.3DCB61C009D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76465:15b2dab1fa08 Date: 2015-03-19 10:34 +0100 http://bitbucket.org/pypy/pypy/changeset/15b2dab1fa08/ Log: Fix: if minor_collection() raises MemoryError, we used to be left with nursery_free==NULL diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -62,7 +62,7 @@ from rpython.rlib.rarithmetic import ovfcheck, LONG_BIT, intmask, r_uint from rpython.rlib.rarithmetic import LONG_BIT_SHIFT from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop -from rpython.rlib.objectmodel import specialize +from rpython.rlib.objectmodel import specialize, we_are_translated # # Handles the objects in 2 generations: @@ -695,7 +695,13 @@ minor_collection_count = 0 while True: - self.nursery_free = llmemory.NULL # debug: don't use me + if not we_are_translated(): + # debug: don't use 'nursery_free', but only if not translated; + # in real code we might get a MemoryError in minor_collection() + # and exit this function unexpectedly, but still catch the + # MemoryError somewhere and continue afterwards --- if we then + # see 'nursery_free == NULL', we segfault. + del self.nursery_free if self.nursery_barriers.non_empty(): size_gc_header = self.gcheaderbuilder.size_gc_header From noreply at buildbot.pypy.org Thu Mar 19 10:39:30 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 19 Mar 2015 10:39:30 +0100 (CET) Subject: [pypy-commit] pypy default: More issues left... skip the test until we decide if we should address them Message-ID: <20150319093930.5D5AE1C009D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76466:37b59327d474 Date: 2015-03-19 10:39 +0100 http://bitbucket.org/pypy/pypy/changeset/37b59327d474/ Log: More issues left... skip the test until we decide if we should address them diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py --- a/rpython/translator/c/test/test_newgc.py +++ b/rpython/translator/c/test/test_newgc.py @@ -1562,6 +1562,7 @@ def test_limited_memory_linux(self): if not sys.platform.startswith('linux'): py.test.skip("linux only") + py.test.skip('XXX fix me?') # import random # From noreply at buildbot.pypy.org Thu Mar 19 10:44:09 2015 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 19 Mar 2015 10:44:09 +0100 (CET) Subject: [pypy-commit] pypy vmprof: refactor the code to not do anything in case of lack of vmprof Message-ID: <20150319094409.A92921C0173@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76467:995ef117566b Date: 2015-03-19 11:44 +0200 http://bitbucket.org/pypy/pypy/changeset/995ef117566b/ Log: refactor the code to not do anything in case of lack of vmprof diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -666,30 +666,15 @@ assert ec is not None return ec - def _open_code_info_file(self): + def register_code_callback(self, callback): ec = self.getexecutioncontext() - try: - ec.code_info_file = os.tmpfile() - except: - ec.code_info_file_present = False # we failed to open it + ec._code_callback = callback def register_code_object(self, pycode): ec = self.getexecutioncontext() - if ec.code_info_file is None: - self._open_code_info_file() - if not ec.code_info_file_present: + if ec._code_callback is None: return - try: - rfile.write_int(ec.code_info_file, pycode._unique_id) - s = pycode._get_full_name() - rfile.write_int(ec.code_info_file, len(s)) - ec.code_info_file.write(s) - except OSError: - ec.code_info_file_present = False - try: - ec.code_info_file.close() - except: - pass # can fail, ignore + ec._code_callback(self, pycode) def _freeze_(self): return True diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -33,13 +33,12 @@ self.profilefunc = None self.w_profilefuncarg = None self.thread_disappeared = False # might be set to True after os.fork() - self.code_info_file = None - self.code_info_file_present = True if sys.maxint == 2147483647: self._code_unique_id = 0 # XXX this is wrong, it won't work on 32bit else: self._code_unique_id = 0x7000000000000000 + self._code_callback = None @staticmethod def _mark_thread_disappeared(space): diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py --- a/pypy/module/_vmprof/interp_vmprof.py +++ b/pypy/module/_vmprof/interp_vmprof.py @@ -3,13 +3,14 @@ from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.rtyper.annlowlevel import cast_instance_to_gcref, cast_base_ptr_to_instance from rpython.rlib.objectmodel import we_are_translated -from rpython.rlib import jit, rposix, entrypoint +from rpython.rlib import jit, rposix, entrypoint, rgc from rpython.rtyper.tool import rffi_platform as platform from rpython.rlib.rstring import StringBuilder from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.error import oefmt, wrap_oserror, OperationError from pypy.interpreter.gateway import unwrap_spec from pypy.interpreter.pyframe import PyFrame +from pypy.interpreter.pycode import PyCode ROOT = py.path.local(__file__).join('..') SRC = ROOT.join('src') @@ -120,6 +121,9 @@ b.append(chr((l >> 48) & 0xff)) b.append(chr((l >> 56) & 0xff)) +def try_cast_to_pycode(gcref): + return rgc.try_cast_gcref_to_instance(PyCode, gcref) + class VMProf(object): def __init__(self): self.is_enabled = False @@ -137,6 +141,8 @@ if we_are_translated(): pypy_vmprof_init() self.ever_enabled = True + self.gather_all_code_objs(space) + space.register_code_callback(self.register_code) if we_are_translated(): # does not work untranslated res = vmprof_enable(fileno, period, 0, @@ -147,6 +153,11 @@ raise wrap_oserror(space, OSError(rposix.get_saved_errno(), "_vmprof.enable")) + def gather_all_code_objs(self, space): + all_code_objs = rgc.do_get_objects(try_cast_to_pycode) + for code in all_code_objs: + self.register_code(space, code) + def write_header(self, fileno, period): if period == -1: period_usec = 1000000 / 100 # 100hz @@ -177,6 +188,7 @@ raise oefmt(space.w_ValueError, "_vmprof not enabled") self.is_enabled = False self.fileno = -1 + space.register_code_callback(None) if we_are_translated(): # does not work untranslated res = vmprof_disable() diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -695,7 +695,7 @@ @specialize.arg(0) def do_get_objects(callback): - """ Get all the objects that satisfy callback(gcref) -> True + """ Get all the objects that satisfy callback(gcref) -> obj """ roots = [gcref for gcref in get_rpy_roots() if gcref] pending = roots[:] From noreply at buildbot.pypy.org Thu Mar 19 11:04:13 2015 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 19 Mar 2015 11:04:13 +0100 (CET) Subject: [pypy-commit] pypy vmprof: kill all_code_info Message-ID: <20150319100413.30A2D1C1270@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76468:cf55b39d3d32 Date: 2015-03-19 12:03 +0200 http://bitbucket.org/pypy/pypy/changeset/cf55b39d3d32/ Log: kill all_code_info diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -70,7 +70,6 @@ 'debug_stop' : 'interp_debug.debug_stop', 'debug_print_once' : 'interp_debug.debug_print_once', 'debug_flush' : 'interp_debug.debug_flush', - 'all_code_info' : 'interp_magic.all_code_info', 'builtinify' : 'interp_magic.builtinify', 'lookup_special' : 'interp_magic.lookup_special', 'do_what_I_mean' : 'interp_magic.do_what_I_mean', diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -131,14 +131,3 @@ assert isinstance(w_frame, PyFrame) w_frame.locals2fast() -def all_code_info(space): - ec = space.getexecutioncontext() - if not ec.code_info_file_present or ec.code_info_file is None: - raise OperationError(space.w_RuntimeError, space.wrap( - "Info file not present, error writing it")) - ec.code_info_file.flush() - ec.code_info_file.seek(0, 0) - try: - return space.wrap(ec.code_info_file.read()) - finally: - ec.code_info_file.seek(0, 2) From noreply at buildbot.pypy.org Thu Mar 19 11:48:40 2015 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 19 Mar 2015 11:48:40 +0100 (CET) Subject: [pypy-commit] pypy vmprof: use the correct callback Message-ID: <20150319104840.0CD961C009D@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76469:bc67a7dfe373 Date: 2015-03-19 12:48 +0200 http://bitbucket.org/pypy/pypy/changeset/bc67a7dfe373/ Log: use the correct callback diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py --- a/pypy/module/_vmprof/interp_vmprof.py +++ b/pypy/module/_vmprof/interp_vmprof.py @@ -142,7 +142,7 @@ pypy_vmprof_init() self.ever_enabled = True self.gather_all_code_objs(space) - space.register_code_callback(self.register_code) + space.register_code_callback(vmprof_register_code) if we_are_translated(): # does not work untranslated res = vmprof_enable(fileno, period, 0, From noreply at buildbot.pypy.org Thu Mar 19 14:12:37 2015 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 19 Mar 2015 14:12:37 +0100 (CET) Subject: [pypy-commit] pypy vmprof: we no longer have entrypoints Message-ID: <20150319131237.BD6B41C019D@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76470:3e12cb44210e Date: 2015-03-19 15:02 +0200 http://bitbucket.org/pypy/pypy/changeset/3e12cb44210e/ Log: we no longer have entrypoints diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py --- a/pypy/module/_vmprof/interp_vmprof.py +++ b/pypy/module/_vmprof/interp_vmprof.py @@ -3,7 +3,7 @@ from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.rtyper.annlowlevel import cast_instance_to_gcref, cast_base_ptr_to_instance from rpython.rlib.objectmodel import we_are_translated -from rpython.rlib import jit, rposix, entrypoint, rgc +from rpython.rlib import jit, rposix, rgc from rpython.rtyper.tool import rffi_platform as platform from rpython.rlib.rstring import StringBuilder from pypy.interpreter.baseobjspace import W_Root From noreply at buildbot.pypy.org Thu Mar 19 14:12:38 2015 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 19 Mar 2015 14:12:38 +0100 (CET) Subject: [pypy-commit] pypy vmprof: remove tab Message-ID: <20150319131238.DDEC21C019D@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76471:93945881c594 Date: 2015-03-19 15:12 +0200 http://bitbucket.org/pypy/pypy/changeset/93945881c594/ Log: remove tab diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py --- a/rpython/translator/c/genc.py +++ b/rpython/translator/c/genc.py @@ -458,7 +458,7 @@ mk.rule('%.o %.gcmap', '%.asmgcc.s', [ '$(PYTHON) $(RPYDIR)/translator/c/gcc/trackgcroot.py ' '-t $*.asmgcc.s > $*.gctmp', - '$(CC) -o $*.o -c $*.asmgcc.lbl.s', + '$(CC) -o $*.o -c $*.asmgcc.lbl.s', 'mv $*.gctmp $*.gcmap', 'rm $*.asmgcc.lbl.s']) From noreply at buildbot.pypy.org Thu Mar 19 14:46:15 2015 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 19 Mar 2015 14:46:15 +0100 (CET) Subject: [pypy-commit] pypy vmprof: fix those tests Message-ID: <20150319134615.19F1F1C00FA@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76472:5dfab2a675c5 Date: 2015-03-19 15:13 +0200 http://bitbucket.org/pypy/pypy/changeset/5dfab2a675c5/ Log: fix those tests diff --git a/rpython/rlib/test/test_rbisect.py b/rpython/rlib/test/test_rbisect.py --- a/rpython/rlib/test/test_rbisect.py +++ b/rpython/rlib/test/test_rbisect.py @@ -44,7 +44,7 @@ ([1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 5, 10), ] for lst, elem, exp in cases: - assert bisect_left(lst, elem) == exp + assert bisect_left(lst, elem, len(lst)) == exp def test_bisect_right(): cases = [ @@ -90,4 +90,4 @@ ([1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 5, 10), ] for lst, elem, exp in cases: - assert bisect_right(lst, elem) == exp + assert bisect_right(lst, elem, len(lst)) == exp From noreply at buildbot.pypy.org Thu Mar 19 14:46:16 2015 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 19 Mar 2015 14:46:16 +0100 (CET) Subject: [pypy-commit] pypy vmprof: fix some tests Message-ID: <20150319134616.3C3021C00FA@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76473:e0be11b881be Date: 2015-03-19 15:46 +0200 http://bitbucket.org/pypy/pypy/changeset/e0be11b881be/ Log: fix some tests diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py --- a/pypy/module/pypyjit/interp_resop.py +++ b/pypy/module/pypyjit/interp_resop.py @@ -105,7 +105,7 @@ ofs = ops_offset.get(op, 0) if op.opnum == rop.DEBUG_MERGE_POINT: jd_sd = jitdrivers_sd[op.getarg(0).getint()] - greenkey = op.getarglist()[3:] + greenkey = op.getarglist()[4:] repr = jd_sd.warmstate.get_location_str(greenkey) w_greenkey = wrap_greenkey(space, jd_sd.jitdriver, greenkey, repr) l_w.append(DebugMergePoint(space, jit_hooks._cast_to_gcref(op), diff --git a/pypy/module/pypyjit/test/test_jit_hook.py b/pypy/module/pypyjit/test/test_jit_hook.py --- a/pypy/module/pypyjit/test/test_jit_hook.py +++ b/pypy/module/pypyjit/test/test_jit_hook.py @@ -55,7 +55,7 @@ oplist = parse(""" [i1, i2, p2] i3 = int_add(i1, i2) - debug_merge_point(0, 0, 0, 0, 0, ConstPtr(ptr0)) + debug_merge_point(0, 0, 0, 0, 0, 0, ConstPtr(ptr0)) guard_nonnull(p2) [] guard_true(i3) [] """, namespace={'ptr0': code_gcref}).operations From noreply at buildbot.pypy.org Thu Mar 19 17:04:56 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Thu, 19 Mar 2015 17:04:56 +0100 (CET) Subject: [pypy-commit] pypy exc-later: update tests Message-ID: <20150319160456.050A21C019D@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: exc-later Changeset: r76475:68638917e42b Date: 2015-03-19 01:01 +0000 http://bitbucket.org/pypy/pypy/changeset/68638917e42b/ Log: update tests diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py --- a/rpython/jit/codewriter/test/test_flatten.py +++ b/rpython/jit/codewriter/test/test_flatten.py @@ -485,11 +485,11 @@ --- L1: goto_if_exception_mismatch $<* struct object_vtable>, L2 - int_return $42 + int_return $-42 --- L2: goto_if_exception_mismatch $<* struct object_vtable>, L3 - int_return $-42 + int_return $42 --- L3: reraise @@ -503,9 +503,8 @@ return ovfcheck(i % j) except OverflowError: return 42 - # XXX so far, this really produces a int_mod_ovf_zer... self.encoding_test(f, [7, 2], """ - residual_call_ir_i $<* fn int_mod_ovf_zer>, I[%i0, %i1], R[], -> %i2 + residual_call_ir_i $<* fn int_mod_ovf>, I[%i0, %i1], R[], -> %i2 -live- catch_exception L1 int_return %i2 From noreply at buildbot.pypy.org Thu Mar 19 17:04:54 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Thu, 19 Mar 2015 17:04:54 +0100 (CET) Subject: [pypy-commit] pypy exc-later: take builtins_exceptions into account before adding the Exception case Message-ID: <20150319160454.B90F01C019D@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: exc-later Changeset: r76474:2de33decd42a Date: 2015-03-18 21:26 +0000 http://bitbucket.org/pypy/pypy/changeset/2de33decd42a/ Log: take builtins_exceptions into account before adding the Exception case diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py --- a/rpython/flowspace/test/test_objspace.py +++ b/rpython/flowspace/test/test_objspace.py @@ -336,7 +336,7 @@ simplify_graph(x) self.show(x) exc_cases = [exit.exitcase for exit in x.startblock.exits] - assert exc_cases == [None, OSError] + assert exc_cases == [None, OSError, Exception] #__________________________________________________________ def reraiseAnythingDicCase(dic): @@ -501,7 +501,7 @@ simplify_graph(graph) assert self.all_operations(graph) == {'simple_call': 1} exc_cases = [exit.exitcase for exit in graph.startblock.exits] - assert exc_cases == [None, IndexError, OSError] + assert exc_cases == [None, IndexError, OSError, Exception] #__________________________________________________________ def dellocal(): diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py --- a/rpython/translator/simplify.py +++ b/rpython/translator/simplify.py @@ -342,7 +342,15 @@ if not exits: block.exitswitch = None block.recloseblock(block.exits[0], *exits) - + else: + if block.exits[-1].exitcase is not Exception: + v_etype = Variable('last_exception') + v_exc = Variable('last_exc_value') + exit = Link([v_etype, v_exc], graph.exceptblock, Exception) + exit.extravars(v_etype, v_exc) + exits = list(block.exits) + exits.append(exit) + block.recloseblock(*exits) def remove_assertion_errors(graph): @@ -363,15 +371,6 @@ if block.canraise: if exit.exitcase is None: break - elif (exit.exitcase is Exception and - block.raising_op.opname - in ('simple_call', 'call_args')): - v_etype = Variable('last_exception') - v_exc = Variable('last_exc_value') - exit.args = [v_etype, v_exc] - exit.last_exception = v_etype - exit.last_exc_value = v_exc - continue if len(block.exits) == 2: # removing the last non-exceptional exit block.exitswitch = None From noreply at buildbot.pypy.org Thu Mar 19 17:33:49 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 19 Mar 2015 17:33:49 +0100 (CET) Subject: [pypy-commit] pypy default: In the incminimark GC, try to find all "raise MemoryError" that can't be Message-ID: <20150319163349.41DB51C1236@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76476:cc84b54369e9 Date: 2015-03-19 17:27 +0100 http://bitbucket.org/pypy/pypy/changeset/cc84b54369e9/ Log: In the incminimark GC, try to find all "raise MemoryError" that can't be guaranteed to leave the GC in a correct state, and turn them into clear fatal errors. diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -62,7 +62,8 @@ from rpython.rlib.rarithmetic import ovfcheck, LONG_BIT, intmask, r_uint from rpython.rlib.rarithmetic import LONG_BIT_SHIFT from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop -from rpython.rlib.objectmodel import specialize, we_are_translated +from rpython.rlib.objectmodel import specialize +from rpython.memory.gc.minimarkpage import out_of_memory # # Handles the objects in 2 generations: @@ -474,7 +475,7 @@ # anyway so it doesn't even counct. nursery = llarena.arena_malloc(self._nursery_memory_size(), 0) if not nursery: - raise MemoryError("cannot allocate nursery") + out_of_memory("cannot allocate nursery") return nursery def allocate_nursery(self): @@ -695,13 +696,9 @@ minor_collection_count = 0 while True: - if not we_are_translated(): - # debug: don't use 'nursery_free', but only if not translated; - # in real code we might get a MemoryError in minor_collection() - # and exit this function unexpectedly, but still catch the - # MemoryError somewhere and continue afterwards --- if we then - # see 'nursery_free == NULL', we segfault. - del self.nursery_free + self.nursery_free = llmemory.NULL # debug: don't use me + # note: no "raise MemoryError" between here and the next time + # we initialize nursery_free! if self.nursery_barriers.non_empty(): size_gc_header = self.gcheaderbuilder.size_gc_header @@ -1956,7 +1953,7 @@ # arena = llarena.arena_malloc(raw_malloc_usage(totalsize), False) if not arena: - raise MemoryError("cannot allocate object") + out_of_memory("out of memory: couldn't allocate a few KB more") llarena.arena_reserve(arena, totalsize) # size_gc_header = self.gcheaderbuilder.size_gc_header @@ -2154,9 +2151,9 @@ # even higher memory consumption. To prevent it, if it's # the second time we are here, then abort the program. if self.max_heap_size_already_raised: - llop.debug_fatalerror(lltype.Void, - "Using too much memory, aborting") + out_of_memory("using too much memory, aborting") self.max_heap_size_already_raised = True + self.gc_state = STATE_SCANNING raise MemoryError self.gc_state = STATE_FINALIZING diff --git a/rpython/memory/gc/minimarkpage.py b/rpython/memory/gc/minimarkpage.py --- a/rpython/memory/gc/minimarkpage.py +++ b/rpython/memory/gc/minimarkpage.py @@ -2,7 +2,7 @@ from rpython.rtyper.lltypesystem import lltype, llmemory, llarena, rffi from rpython.rlib.rarithmetic import LONG_BIT, r_uint from rpython.rlib.objectmodel import we_are_translated -from rpython.rlib.debug import ll_assert +from rpython.rlib.debug import ll_assert, fatalerror WORD = LONG_BIT // 8 NULL = llmemory.NULL @@ -294,7 +294,7 @@ # be a page-aligned address arena_base = llarena.arena_malloc(self.arena_size, False) if not arena_base: - raise MemoryError("couldn't allocate the next arena") + out_of_memory("out of memory: couldn't allocate the next arena") arena_end = arena_base + self.arena_size # # 'firstpage' points to the first unused page @@ -593,3 +593,10 @@ if isinstance(size, int): size = llmemory.sizeof(lltype.Char) * size return size + +def out_of_memory(errmsg): + """Signal a fatal out-of-memory error and abort. For situations where + it is hard to write and test code that would handle a MemoryError + exception gracefully. + """ + fatalerror(errmsg) diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py --- a/rpython/translator/c/test/test_newgc.py +++ b/rpython/translator/c/test/test_newgc.py @@ -1557,26 +1557,6 @@ res = self.run("limited_memory", -1, runner=myrunner) assert res == 42 - define_limited_memory_linux = define_limited_memory - - def test_limited_memory_linux(self): - if not sys.platform.startswith('linux'): - py.test.skip("linux only") - py.test.skip('XXX fix me?') - # - import random - # - def myrunner(args): - args1 = ['/bin/bash', '-c', 'ulimit -v %d && %s' % - (ulimitv, ' '.join(args),)] - return subprocess.check_output(args1) - # - for i in range(10): - ulimitv = random.randrange(50000, 100000) - print ulimitv - res = self.run("limited_memory_linux", -1, runner=myrunner) - assert res == 42 - class TestIncrementalMiniMarkGC(TestMiniMarkGC): gcpolicy = "incminimark" @@ -1630,6 +1610,29 @@ res = self.run("random_pin") assert res == 28495 + define_limited_memory_linux = TestMiniMarkGC.define_limited_memory.im_func + + def test_limited_memory_linux(self): + if not sys.platform.startswith('linux'): + py.test.skip("linux only") + # + import random + # + def myrunner(args): + args1 = ['/bin/bash', '-c', 'ulimit -v %d && %s' % + (ulimitv, ' '.join(args),)] + popen = subprocess.Popen(args1, stderr=subprocess.PIPE) + _, child_stderr = popen.communicate() + assert popen.wait() == 134 # aborted + assert 'out of memory:' in child_stderr + return '42' + # + for i in range(10): + ulimitv = random.randrange(50000, 100000) + print ulimitv + res = self.run("limited_memory_linux", -1, runner=myrunner) + assert res == 42 + # ____________________________________________________________________ From noreply at buildbot.pypy.org Thu Mar 19 18:46:47 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 19 Mar 2015 18:46:47 +0100 (CET) Subject: [pypy-commit] pypy default: Removed tag release-2.5.1 Message-ID: <20150319174647.F2B961C0173@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r76478:c75ffcefd8d1 Date: 2015-03-19 19:46 +0200 http://bitbucket.org/pypy/pypy/changeset/c75ffcefd8d1/ Log: Removed tag release-2.5.1 diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -12,3 +12,5 @@ 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1 10f1b29a2bd21f837090286174a9ca030b8680b2 release-2.5.0 8e24dac0b8e2db30d46d59f2c4daa3d4aaab7861 release-2.5.1 +8e24dac0b8e2db30d46d59f2c4daa3d4aaab7861 release-2.5.1 +0000000000000000000000000000000000000000 release-2.5.1 From noreply at buildbot.pypy.org Thu Mar 19 18:46:49 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 19 Mar 2015 18:46:49 +0100 (CET) Subject: [pypy-commit] pypy default: Added tag release-2.5.1 for changeset e3d046c43451 Message-ID: <20150319174649.27A451C0173@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r76479:6774bfcf9568 Date: 2015-03-19 19:47 +0200 http://bitbucket.org/pypy/pypy/changeset/6774bfcf9568/ Log: Added tag release-2.5.1 for changeset e3d046c43451 diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -14,3 +14,5 @@ 8e24dac0b8e2db30d46d59f2c4daa3d4aaab7861 release-2.5.1 8e24dac0b8e2db30d46d59f2c4daa3d4aaab7861 release-2.5.1 0000000000000000000000000000000000000000 release-2.5.1 +0000000000000000000000000000000000000000 release-2.5.1 +e3d046c43451403f5969580fc1c41d5df6c4082a release-2.5.1 From noreply at buildbot.pypy.org Thu Mar 19 18:46:46 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 19 Mar 2015 18:46:46 +0100 (CET) Subject: [pypy-commit] pypy object-dtype2: create a gc customtrace, improperly call the hook Message-ID: <20150319174646.C7FD81C0173@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: object-dtype2 Changeset: r76477:383a91bc4802 Date: 2015-03-19 19:46 +0200 http://bitbucket.org/pypy/pypy/changeset/383a91bc4802/ Log: create a gc customtrace, improperly call the hook diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py --- a/pypy/module/micronumpy/compile.py +++ b/pypy/module/micronumpy/compile.py @@ -511,8 +511,6 @@ dtype = get_dtype_cache(interp.space).w_int64dtype elif self.v == 'float': dtype = get_dtype_cache(interp.space).w_float64dtype - elif self.v == 'object': - dtype = get_dtype_cache(interp.space).w_objectdtype else: raise BadToken('unknown v to dtype "%s"' % self.v) return dtype diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py --- a/pypy/module/micronumpy/concrete.py +++ b/pypy/module/micronumpy/concrete.py @@ -4,8 +4,8 @@ from rpython.rlib.debug import make_sure_not_resized from rpython.rlib.rawstorage import alloc_raw_storage, free_raw_storage, \ raw_storage_getitem, raw_storage_setitem, RAW_STORAGE -from rpython.rtyper.lltypesystem import rffi, lltype -from pypy.module.micronumpy import support, loop +from rpython.rtyper.lltypesystem import rffi, lltype, llmemory +from pypy.module.micronumpy import support, loop, constants as NPY from pypy.module.micronumpy.base import convert_to_array, W_NDimArray, \ ArrayArgumentException from pypy.module.micronumpy.iterators import ArrayIter @@ -13,6 +13,8 @@ RecordChunk, calc_strides, calc_new_strides, shape_agreement, calculate_broadcast_strides, calc_backstrides) from rpython.rlib.objectmodel import keepalive_until_here +from rpython.rtyper.annlowlevel import cast_gcref_to_instance +from pypy.interpreter.baseobjspace import W_Root class BaseConcreteArray(object): @@ -333,6 +335,35 @@ loop.setslice(space, impl.get_shape(), impl, self) return impl +OBJECTSTORE = lltype.GcStruct('ObjectStore', + ('length', lltype.Signed), + ('step', lltype.Signed), + ('storage', llmemory.Address), + rtti=True) +offset_of_storage = llmemory.offsetof(OBJECTSTORE, 'storage') +offset_of_length = llmemory.offsetof(OBJECTSTORE, 'length') +offset_of_step = llmemory.offsetof(OBJECTSTORE, 'step') + +def customtrace(gc, obj, callback, arg): + print 'in customtrace w/obj',obj + length = rffi.cast(lltype.Signed, obj + offset_of_length) + step = rffi.cast(lltype.Signed, obj + offset_of_step) + storage = obj + offset_of_storage + print 'tracing', length, 'objects in ndarray.storage' + for i in range(length): + gcref = rffi.cast(llmemory.GCREF, storage) + w_obj = cast_gcref_to_instance(W_Root, gcref) + print w_obj + gc._trace_callback(callback, arg, storage) + storage += step + +lambda_customtrace = lambda: customtrace + +def _setup(): + print 'registering custom trace for OBJECTSTORE' + rgc.register_custom_trace_hook(OBJECTSTORE, lambda_customtrace) + + class ConcreteArrayNotOwning(BaseConcreteArray): def __init__(self, shape, dtype, order, strides, backstrides, storage, start=0): @@ -347,7 +378,7 @@ self.backstrides = backstrides self.storage = storage self.start = start - self.gcstruct = None + self.gcstruct = lltype.nullptr(OBJECTSTORE) def fill(self, space, box): self.dtype.itemtype.fill(self.storage, self.dtype.elsize, @@ -375,32 +406,25 @@ def base(self): return None -OBJECTSTORE = lltype.GcStruct('ObjectStore', - ('storage', llmemory.Address), - rtti=True) -def customtrace(gc, obj, callback, arg): - xxxx -lambda_customtrace = lambda: customtrace - - class ConcreteArray(ConcreteArrayNotOwning): def __init__(self, shape, dtype, order, strides, backstrides, storage=lltype.nullptr(RAW_STORAGE), zero=True): - - gcstruct = None + gcstruct = lltype.nullptr(OBJECTSTORE) if storage == lltype.nullptr(RAW_STORAGE): - storage = dtype.itemtype.malloc(support.product(shape) * - dtype.elsize, zero=zero) + length = support.product(shape) + storage = dtype.itemtype.malloc(length * dtype.elsize, zero=zero) if dtype.num == NPY.OBJECT: - rgc.register_custom_trace_hook(OBJECTSTORE, lambda_customtrace) + _setup() # make sure gc hook is registered gcstruct = lltype.malloc(OBJECTSTORE) - gcstruct.storage = storage + gcstruct.storage = llmemory.cast_ptr_to_adr(storage) + print 'create gcstruct',gcstruct,'with storage',storage,'as',gcstruct.storage + gcstruct.length = length + gcstruct.step = dtype.elsize ConcreteArrayNotOwning.__init__(self, shape, dtype, order, strides, backstrides, storage) self.gcstruct = gcstruct def __del__(self): - rgc. free_raw_storage(self.storage, track_allocation=False) diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -17,6 +17,7 @@ def __init__(self): self.base = self self.elsize = 1 + self.num = 0 def create_slice(space, a, chunks): From noreply at buildbot.pypy.org Thu Mar 19 22:09:47 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 19 Mar 2015 22:09:47 +0100 (CET) Subject: [pypy-commit] pypy default: partial fix for rpath with clang (darwin), broken by a5c8f6daa2f8 Message-ID: <20150319210947.A6AA51C019D@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r76480:4d66c739aa16 Date: 2015-03-19 23:11 +0200 http://bitbucket.org/pypy/pypy/changeset/4d66c739aa16/ Log: partial fix for rpath with clang (darwin), broken by a5c8f6daa2f8 diff --git a/rpython/translator/platform/darwin.py b/rpython/translator/platform/darwin.py --- a/rpython/translator/platform/darwin.py +++ b/rpython/translator/platform/darwin.py @@ -12,6 +12,11 @@ DEFAULT_CC = 'clang' rpath_flags = ['-Wl,-rpath', '-Wl, at executable_path/'] + def get_rpath_flags(self, rel_libdirs): + # needed for cross compiling on ARM, needs fixing if relevant for darwin + assert len(rel_libdirs) < 1 + return self.rpath_flags + def _args_for_shared(self, args): return (list(self.shared_only) + ['-dynamiclib', '-install_name', '@rpath/$(TARGET)', '-undefined', 'dynamic_lookup'] diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py --- a/rpython/translator/platform/posix.py +++ b/rpython/translator/platform/posix.py @@ -89,6 +89,11 @@ raise ValueError(msg) return result + def get_rpath_flags(self, rel_libdirs): + # needed for cross-compilation i.e. ARM + return self.rpath_flags + ['-Wl,-rpath-link=\'%s\'' % ldir + for ldir in rel_libdirs] + def get_shared_only_compile_flags(self): return tuple(self.shared_only) + ('-fvisibility=hidden',) @@ -167,8 +172,7 @@ ('CC', self.cc), ('CC_LINK', eci.use_cpp_linker and 'g++' or '$(CC)'), ('LINKFILES', eci.link_files), - ('RPATH_FLAGS', self.rpath_flags + ['-Wl,-rpath-link=\'%s\'' % ldir - for ldir in rel_libdirs]), + ('RPATH_FLAGS', self.get_rpath_flags(rel_libdirs)), ] for args in definitions: m.definition(*args) From noreply at buildbot.pypy.org Fri Mar 20 08:11:46 2015 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 20 Mar 2015 08:11:46 +0100 (CET) Subject: [pypy-commit] pypy default: 4d66c739aa16 was too strict, why? Message-ID: <20150320071146.2D31B1C00FA@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r76481:adc34b25572f Date: 2015-03-20 07:47 +0200 http://bitbucket.org/pypy/pypy/changeset/adc34b25572f/ Log: 4d66c739aa16 was too strict, why? diff --git a/rpython/translator/platform/darwin.py b/rpython/translator/platform/darwin.py --- a/rpython/translator/platform/darwin.py +++ b/rpython/translator/platform/darwin.py @@ -14,7 +14,8 @@ def get_rpath_flags(self, rel_libdirs): # needed for cross compiling on ARM, needs fixing if relevant for darwin - assert len(rel_libdirs) < 1 + if len(rel_libdirs) > 0: + print 'in get_rpath_flags, rel_libdirs is not fixed up',rel_libdirs return self.rpath_flags def _args_for_shared(self, args): From noreply at buildbot.pypy.org Fri Mar 20 18:11:41 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 20 Mar 2015 18:11:41 +0100 (CET) Subject: [pypy-commit] pypy default: Small rewrites Message-ID: <20150320171141.972CC1C00FA@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76482:81078d224b97 Date: 2015-03-20 18:11 +0100 http://bitbucket.org/pypy/pypy/changeset/81078d224b97/ Log: Small rewrites diff --git a/pypy/doc/stm.rst b/pypy/doc/stm.rst --- a/pypy/doc/stm.rst +++ b/pypy/doc/stm.rst @@ -174,6 +174,38 @@ User Guide ========== +How to write multithreaded programs: the 10'000-feet view +--------------------------------------------------------- + +PyPy-STM offers two ways to write multithreaded programs: + +* the traditional way, using the ``thread`` or ``threading`` modules, + described first__. + +* using ``TransactionQueue``, described next__, as a way to hide the + low-level notion of threads. + +.. __: `Drop-in replacement`_ +.. __: `transaction.TransactionQueue`_ + +The issue with low-level threads are well known (particularly in other +languages that don't have GIL-based interpreters): memory corruption, +deadlocks, livelocks, and so on. There are alternative approaches to +dealing directly with threads, like OpenMP_. These approaches +typically enforce some structure on your code. ``TransactionQueue`` +is in part similar: your program needs to have "some chances" of +parallelization before you can apply it. But I believe that the scope +of applicability is much larger with ``TransactionQueue`` than with +other approaches. It usually works without forcing a complete +reorganization of your existing code, and it works on any Python +program which has got *latent* and *imperfect* parallelism. Ideally, +it only requires that the end programmer identifies where this +parallelism is likely to be found, and communicates it to the system +using a simple API. + +.. _OpenMP: http://en.wikipedia.org/wiki/OpenMP + + Drop-in replacement ------------------- @@ -196,31 +228,6 @@ order. -How to write multithreaded programs: the 10'000-feet view ---------------------------------------------------------- - -PyPy-STM offers two ways to write multithreaded programs: - -* the traditional way, using the ``thread`` or ``threading`` modules. - -* using ``TransactionQueue``, described next__, as a way to hide the - low-level notion of threads. - -.. __: `transaction.TransactionQueue`_ - -``TransactionQueue`` hides the hard multithreading-related issues that -we typically encounter when using low-level threads. This is not the -first alternative approach to avoid dealing with low-level threads; -for example, OpenMP_ is one. However, it is one of the first ones -which does not require the code to be organized in a particular -fashion. Instead, it works on any Python program which has got -*latent* and *imperfect* parallelism. Ideally, it only requires that -the end programmer identifies where this parallelism is likely to be -found, and communicates it to the system using a simple API. - -.. _OpenMP: http://en.wikipedia.org/wiki/OpenMP - - transaction.TransactionQueue ---------------------------- @@ -256,8 +263,9 @@ behavior did not change because we are using ``TransactionQueue``. All the calls still *appear* to execute in some serial order. -However, the performance typically does not increase out of the box. -In fact, it is likely to be worse at first. Typically, this is +A typical usage of ``TransactionQueue`` goes like that: at first, +the performance does not increase. +In fact, it is likely to be worse. Typically, this is indicated by the total CPU usage, which remains low (closer to 1 than N cores). First note that it is expected that the CPU usage should not go much higher than 1 in the JIT warm-up phase: you must run a @@ -282,9 +290,9 @@ because of the reason shown in the two independent single-entry tracebacks: one thread ran the line ``someobj.stuff = 5``, whereas another thread concurrently ran the line ``someobj.other = 10`` on the -same object. Two writes to the same object cause a conflict, which -aborts one of the two transactions. In the example above this -occurred 12412 times. +same object. These two writes are done to the same object. This +causes a conflict, which aborts one of the two transactions. In the +example above this occurred 12412 times. The two other conflict sources are ``STM_CONTENTION_INEVITABLE``, which means that two transactions both tried to do an external @@ -303,7 +311,7 @@ each transaction starts with sending data to a log file. You should refactor this case so that it occurs either near the end of the transaction (which can then mostly run in non-inevitable mode), or - even delegate it to a separate thread. + delegate it to a separate transaction or even a separate thread. * Writing to a list or a dictionary conflicts with any read from the same list or dictionary, even one done with a different key. For @@ -322,7 +330,7 @@ results is fine, use ``transaction.time()`` or ``transaction.clock()``. -* ``transaction.threadlocalproperty`` can be used as class-level:: +* ``transaction.threadlocalproperty`` can be used at class-level:: class Foo(object): # must be a new-style class! x = transaction.threadlocalproperty() @@ -342,11 +350,11 @@ threads, each running the transactions one after the other; such thread-local properties will have the value last stored in them in the same thread,, which may come from a random previous transaction. - ``threadlocalproperty`` is still useful to avoid conflicts from - cache-like data structures. + This means that ``threadlocalproperty`` is useful mainly to avoid + conflicts from cache-like data structures. Note that Python is a complicated language; there are a number of less -common cases that may cause conflict (of any type) where we might not +common cases that may cause conflict (of any kind) where we might not expect it at priori. In many of these cases it could be fixed; please report any case that you don't understand. (For example, so far, creating a weakref to an object requires attaching an auxiliary @@ -395,8 +403,8 @@ it likely that such a piece of code will eventually block all other threads anyway. -Note that if you want to experiment with ``atomic``, you may have to add -manually a transaction break just before the atomic block. This is +Note that if you want to experiment with ``atomic``, you may have to +manually add a transaction break just before the atomic block. This is because the boundaries of the block are not guaranteed to be the boundaries of the transaction: the latter is at least as big as the block, but may be bigger. Therefore, if you run a big atomic block, it @@ -522,7 +530,7 @@ where, say, all threads try to increase the same global counter and do nothing else). -However, using if the program requires longer transactions, it comes +However, if the program requires longer transactions, it comes with less obvious rules. The exact details may vary from version to version, too, until they are a bit more stabilized. Here is an overview. From noreply at buildbot.pypy.org Fri Mar 20 19:21:54 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 20 Mar 2015 19:21:54 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Phew, this took quite a bit of guessing. See comments. Message-ID: <20150320182154.D9B5B1C0113@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76483:749349c85755 Date: 2015-03-20 19:22 +0100 http://bitbucket.org/pypy/pypy/changeset/749349c85755/ Log: Phew, this took quite a bit of guessing. See comments. diff --git a/lib_pypy/transaction.py b/lib_pypy/transaction.py --- a/lib_pypy/transaction.py +++ b/lib_pypy/transaction.py @@ -216,6 +216,12 @@ assert len(pending) == 0 while True: f, args, kwds = next_transaction + # The next hint_commit_soon() is essential: without it, the + # current transaction is short, so far, but includes everything + # after some lock.acquire() done recently. That means that + # anything we do in the atomic section will run with the lock + # still acquired. This prevents any parallelization. + hint_commit_soon() with atomic: if exception: break From noreply at buildbot.pypy.org Sat Mar 21 10:33:14 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 21 Mar 2015 10:33:14 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Disable the jitprof.Profiler class with stm, as an attempt to reduce conflicts. Message-ID: <20150321093314.A61081C15C5@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76484:3628a602564d Date: 2015-03-21 08:48 +0100 http://bitbucket.org/pypy/pypy/changeset/3628a602564d/ Log: Disable the jitprof.Profiler class with stm, as an attempt to reduce conflicts. diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py --- a/rpython/jit/metainterp/warmspot.py +++ b/rpython/jit/metainterp/warmspot.py @@ -38,6 +38,9 @@ ProfilerClass = Profiler # Always use Profiler here, which should have a very low impact. # Otherwise you can try with ProfilerClass = EmptyProfiler. + # We do that for stm to avoid conflicts. + if translator.config.translation.stm: + ProfilerClass = EmptyProfiler warmrunnerdesc = WarmRunnerDesc(translator, translate_support_code=True, listops=True, From noreply at buildbot.pypy.org Sat Mar 21 10:33:15 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 21 Mar 2015 10:33:15 +0100 (CET) Subject: [pypy-commit] pypy default: Test and fix for EmptyProfiler Message-ID: <20150321093315.D30381C15C8@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76485:8438bb05ecf2 Date: 2015-03-21 09:21 +0100 http://bitbucket.org/pypy/pypy/changeset/8438bb05ecf2/ Log: Test and fix for EmptyProfiler diff --git a/rpython/jit/metainterp/jitprof.py b/rpython/jit/metainterp/jitprof.py --- a/rpython/jit/metainterp/jitprof.py +++ b/rpython/jit/metainterp/jitprof.py @@ -44,7 +44,10 @@ pass def get_counter(self, num): - return -1.0 + return 0 + + def get_times(self, num): + return 0.0 class Profiler(BaseProfiler): initialized = False @@ -109,6 +112,9 @@ return self.cpu.tracker.total_freed_bridges return self.counters[num] + def get_times(self, num): + return self.times[num] + def count_ops(self, opnum, kind=Counters.OPS): from rpython.jit.metainterp.resoperation import rop self.counters[kind] += 1 diff --git a/rpython/jit/metainterp/test/test_jitiface.py b/rpython/jit/metainterp/test/test_jitiface.py --- a/rpython/jit/metainterp/test/test_jitiface.py +++ b/rpython/jit/metainterp/test/test_jitiface.py @@ -5,7 +5,7 @@ from rpython.jit.codewriter.policy import JitPolicy from rpython.jit.metainterp.resoperation import rop from rpython.rtyper.annlowlevel import hlstr -from rpython.jit.metainterp.jitprof import Profiler +from rpython.jit.metainterp.jitprof import Profiler, EmptyProfiler class JitHookInterfaceTests(object): @@ -178,6 +178,20 @@ self.meta_interp(main, [], ProfilerClass=Profiler) + def test_get_stats_empty(self): + driver = JitDriver(greens = [], reds = ['i']) + def loop(i): + while i > 0: + driver.jit_merge_point(i=i) + i -= 1 + def main(): + loop(30) + assert jit_hooks.stats_get_counter_value(None, + Counters.TOTAL_COMPILED_LOOPS) == 0 + assert jit_hooks.stats_get_times_value(None, Counters.TRACING) == 0 + self.meta_interp(main, [], ProfilerClass=EmptyProfiler) + + class LLJitHookInterfaceTests(JitHookInterfaceTests): # use this for any backend, instead of the super class diff --git a/rpython/rlib/jit_hooks.py b/rpython/rlib/jit_hooks.py --- a/rpython/rlib/jit_hooks.py +++ b/rpython/rlib/jit_hooks.py @@ -130,7 +130,7 @@ @register_helper(annmodel.SomeFloat()) def stats_get_times_value(warmrunnerdesc, no): - return warmrunnerdesc.metainterp_sd.profiler.times[no] + return warmrunnerdesc.metainterp_sd.profiler.get_times(no) LOOP_RUN_CONTAINER = lltype.GcArray(lltype.Struct('elem', ('type', lltype.Char), From noreply at buildbot.pypy.org Sat Mar 21 10:33:17 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 21 Mar 2015 10:33:17 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Test and fix for EmptyProfiler Message-ID: <20150321093317.1B0EA1C15FC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76486:fd630b2807a8 Date: 2015-03-21 09:21 +0100 http://bitbucket.org/pypy/pypy/changeset/fd630b2807a8/ Log: Test and fix for EmptyProfiler diff --git a/rpython/jit/metainterp/jitprof.py b/rpython/jit/metainterp/jitprof.py --- a/rpython/jit/metainterp/jitprof.py +++ b/rpython/jit/metainterp/jitprof.py @@ -44,7 +44,10 @@ pass def get_counter(self, num): - return -1.0 + return 0 + + def get_times(self, num): + return 0.0 class Profiler(BaseProfiler): initialized = False @@ -109,6 +112,9 @@ return self.cpu.tracker.total_freed_bridges return self.counters[num] + def get_times(self, num): + return self.times[num] + def count_ops(self, opnum, kind=Counters.OPS): from rpython.jit.metainterp.resoperation import rop self.counters[kind] += 1 diff --git a/rpython/jit/metainterp/test/test_jitiface.py b/rpython/jit/metainterp/test/test_jitiface.py --- a/rpython/jit/metainterp/test/test_jitiface.py +++ b/rpython/jit/metainterp/test/test_jitiface.py @@ -5,7 +5,7 @@ from rpython.jit.codewriter.policy import JitPolicy from rpython.jit.metainterp.resoperation import rop from rpython.rtyper.annlowlevel import hlstr -from rpython.jit.metainterp.jitprof import Profiler +from rpython.jit.metainterp.jitprof import Profiler, EmptyProfiler class JitHookInterfaceTests(object): @@ -178,6 +178,20 @@ self.meta_interp(main, [], ProfilerClass=Profiler) + def test_get_stats_empty(self): + driver = JitDriver(greens = [], reds = ['i']) + def loop(i): + while i > 0: + driver.jit_merge_point(i=i) + i -= 1 + def main(): + loop(30) + assert jit_hooks.stats_get_counter_value(None, + Counters.TOTAL_COMPILED_LOOPS) == 0 + assert jit_hooks.stats_get_times_value(None, Counters.TRACING) == 0 + self.meta_interp(main, [], ProfilerClass=EmptyProfiler) + + class LLJitHookInterfaceTests(JitHookInterfaceTests): # use this for any backend, instead of the super class diff --git a/rpython/rlib/jit_hooks.py b/rpython/rlib/jit_hooks.py --- a/rpython/rlib/jit_hooks.py +++ b/rpython/rlib/jit_hooks.py @@ -130,7 +130,7 @@ @register_helper(annmodel.SomeFloat()) def stats_get_times_value(warmrunnerdesc, no): - return warmrunnerdesc.metainterp_sd.profiler.times[no] + return warmrunnerdesc.metainterp_sd.profiler.get_times(no) LOOP_RUN_CONTAINER = lltype.GcArray(lltype.Struct('elem', ('type', lltype.Char), From noreply at buildbot.pypy.org Sat Mar 21 10:33:18 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 21 Mar 2015 10:33:18 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Add another stmdict to reduce conflicts Message-ID: <20150321093318.3B1F91C15FC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76487:3956536622d3 Date: 2015-03-21 10:33 +0100 http://bitbucket.org/pypy/pypy/changeset/3956536622d3/ Log: Add another stmdict to reduce conflicts diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py --- a/rpython/annotator/bookkeeper.py +++ b/rpython/annotator/bookkeeper.py @@ -23,6 +23,11 @@ from rpython.tool.algo.unionfind import UnionFind from rpython.rtyper import extregistry +try: + from pypystm import stmdict +except ImportError: + stmdict = {} + BUILTIN_ANALYZERS = {} @@ -54,7 +59,7 @@ def __init__(self, annotator): self.annotator = annotator self.bkTLS = TlsClass() - self.descs = {} # map Python objects to their XxxDesc wrappers + self.descs = stmdict() # map Python objects to their XxxDesc wrappers self.methoddescs = {} # map (funcdesc, classdef) to the MethodDesc self.classdefs = [] # list of all ClassDefs self.seen_mutable = {} From noreply at buildbot.pypy.org Sat Mar 21 12:03:11 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 21 Mar 2015 12:03:11 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Fix the simple weakrefs for STM: don't record them on the object for now. Message-ID: <20150321110311.82DE01C02BB@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76488:2f54cdf1fe3c Date: 2015-03-21 11:19 +0100 http://bitbucket.org/pypy/pypy/changeset/2f54cdf1fe3c/ Log: Fix the simple weakrefs for STM: don't record them on the object for now. diff --git a/pypy/module/_weakref/interp__weakref.py b/pypy/module/_weakref/interp__weakref.py --- a/pypy/module/_weakref/interp__weakref.py +++ b/pypy/module/_weakref/interp__weakref.py @@ -67,8 +67,7 @@ self.cached_weakref = weakref.ref(w_ref) else: # subclass: cannot cache - w_ref = space.allocate_instance(W_Weakref, w_subtype) - W_Weakref.__init__(w_ref, space, w_obj, None) + w_ref = W_Weakref.new(space, w_subtype, w_obj) self.append_wref_to(w_ref) return w_ref @@ -79,10 +78,7 @@ w_cached = self.cached_proxy() if w_cached is not None: return w_cached - if space.is_true(space.callable(w_obj)): - w_proxy = W_CallableProxy(space, w_obj, None) - else: - w_proxy = W_Proxy(space, w_obj, None) + w_proxy = W_Proxy.new(space, w_obj) self.cached_proxy = weakref.ref(w_proxy) return w_proxy @@ -128,18 +124,14 @@ @jit.dont_look_inside def make_weakref_with_callback(self, w_subtype, w_obj, w_callable): space = self.space - w_ref = space.allocate_instance(W_Weakref, w_subtype) - W_Weakref.__init__(w_ref, space, w_obj, w_callable) + w_ref = W_Weakref.new(space, w_subtype, w_obj, w_callable) self.append_wref_to(w_ref) return w_ref @jit.dont_look_inside def make_proxy_with_callback(self, w_obj, w_callable): space = self.space - if space.is_true(space.callable(w_obj)): - w_proxy = W_CallableProxy(space, w_obj, w_callable) - else: - w_proxy = W_Proxy(space, w_obj, w_callable) + w_proxy = W_Proxy.new(space, w_obj, w_callable) self.append_wref_to(w_proxy) return w_proxy @@ -185,6 +177,12 @@ W_WeakrefBase.__init__(w_self, space, w_obj, w_callable) w_self.w_hash = None + @staticmethod + def new(space, w_subtype, w_obj, w_callable=None): + w_ref = space.allocate_instance(W_Weakref, w_subtype) + W_Weakref.__init__(w_ref, space, w_obj, w_callable) + return w_ref + def descr__init__weakref(self, space, w_obj, w_callable=None, __args__=None): if __args__.arguments_w: @@ -238,6 +236,10 @@ def get_or_make_weakref(space, w_subtype, w_obj): + if space.config.translation.stm: + # With stm, return a non-cached, non-recorded weakref. + # Helpers like getweakrefcount() will fail; too bad for now. + return W_Weakref.new(space, w_subtype, w_obj) return getlifeline(space, w_obj).get_or_make_weakref(w_subtype, w_obj) @@ -305,6 +307,13 @@ raise OperationError(space.w_TypeError, space.wrap("unhashable type")) + @staticmethod + def new(space, w_obj, w_callable=None): + if space.is_true(space.callable(w_obj)): + return W_CallableProxy(space, w_obj, w_callable) + else: + return W_Proxy(space, w_obj, w_callable) + class W_CallableProxy(W_Proxy): def descr__call__(self, space, __args__): w_obj = force(space, self) @@ -312,6 +321,10 @@ def get_or_make_proxy(space, w_obj): + if space.config.translation.stm: + # With stm, return a non-cached, non-recorded proxy. + # Helpers like getweakrefcount() will fail; too bad for now. + return W_Proxy.new(space, w_obj) return getlifeline(space, w_obj).get_or_make_proxy(w_obj) From noreply at buildbot.pypy.org Sat Mar 21 12:03:12 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 21 Mar 2015 12:03:12 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: uh Message-ID: <20150321110312.A42381C02BB@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76489:652ec34956fa Date: 2015-03-21 11:21 +0100 http://bitbucket.org/pypy/pypy/changeset/652ec34956fa/ Log: uh diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py --- a/rpython/annotator/bookkeeper.py +++ b/rpython/annotator/bookkeeper.py @@ -26,7 +26,7 @@ try: from pypystm import stmdict except ImportError: - stmdict = {} + stmdict = dict BUILTIN_ANALYZERS = {} From noreply at buildbot.pypy.org Sat Mar 21 12:03:13 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 21 Mar 2015 12:03:13 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Fix the conflict with 'fixed_graphs' by moving this tiny part of the logic Message-ID: <20150321110313.C1F671C02BB@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76490:f390775085de Date: 2015-03-21 11:44 +0100 http://bitbucket.org/pypy/pypy/changeset/f390775085de/ Log: Fix the conflict with 'fixed_graphs' by moving this tiny part of the logic outside the transactions. diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py --- a/rpython/annotator/annrpython.py +++ b/rpython/annotator/annrpython.py @@ -40,7 +40,7 @@ self.annotated = {} # set of blocks already seen self.links_followed = {} # set of links that have ever been followed self.notify = {} # {block: {positions-to-reflow-from-when-done}} - self.fixed_graphs = stmset() # set of graphs not to annotate again + self.fixed_graphs = {} # set of graphs not to annotate again self.blocked_blocks = {} # set of {blocked_block: (graph, index)} # --- the following information is recorded for debugging --- self.blocked_graphs = {} # set of graphs that have blocked blocks diff --git a/rpython/rtyper/rtyper.py b/rpython/rtyper/rtyper.py --- a/rpython/rtyper/rtyper.py +++ b/rpython/rtyper/rtyper.py @@ -244,6 +244,7 @@ for block in pending: tracking(block) blockcount += 1 + self._processing_block(block) self.specialize_block(block) self.already_seen[block] = True # progress bar @@ -265,6 +266,7 @@ (len(pending),)) tq = transaction.TransactionQueue() for block in pending: + self._processing_block(block) tq.add(self.specialize_block, block) tq.run() self.log.event('left transactional mode') @@ -366,7 +368,7 @@ # and then other reprs' setup crash self.call_all_setups() - def _specialize_block(self, block): + def _processing_block(self, block): graph = self.annotator.annotated[block] if graph not in self.annotator.fixed_graphs: self.annotator.fixed_graphs.add(graph) @@ -374,6 +376,7 @@ # are concretetype'd self.setconcretetype(graph.getreturnvar()) + def _specialize_block(self, block): # give the best possible types to the input args try: self.setup_block_entry(block) From noreply at buildbot.pypy.org Sat Mar 21 12:03:14 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 21 Mar 2015 12:03:14 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Oups Message-ID: <20150321110314.E92FC1C02BB@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76491:1a696d049ba6 Date: 2015-03-21 11:51 +0100 http://bitbucket.org/pypy/pypy/changeset/1a696d049ba6/ Log: Oups diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py --- a/rpython/annotator/annrpython.py +++ b/rpython/annotator/annrpython.py @@ -17,11 +17,6 @@ log = py.log.Producer("annrpython") py.log.setconsumer("annrpython", ansi_log) -try: - from pypystm import stmset -except ImportError: - stmset = set - class RPythonAnnotator(object): """Block annotator for RPython. @@ -40,7 +35,7 @@ self.annotated = {} # set of blocks already seen self.links_followed = {} # set of links that have ever been followed self.notify = {} # {block: {positions-to-reflow-from-when-done}} - self.fixed_graphs = {} # set of graphs not to annotate again + self.fixed_graphs = set() # set of graphs not to annotate again self.blocked_blocks = {} # set of {blocked_block: (graph, index)} # --- the following information is recorded for debugging --- self.blocked_graphs = {} # set of graphs that have blocked blocks From noreply at buildbot.pypy.org Sat Mar 21 12:03:16 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 21 Mar 2015 12:03:16 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: fix: _processing_block may add new entries into must_call_setup Message-ID: <20150321110316.193851C02BB@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76492:40ed22359f76 Date: 2015-03-21 12:00 +0100 http://bitbucket.org/pypy/pypy/changeset/40ed22359f76/ Log: fix: _processing_block may add new entries into must_call_setup diff --git a/rpython/rtyper/rtyper.py b/rpython/rtyper/rtyper.py --- a/rpython/rtyper/rtyper.py +++ b/rpython/rtyper/rtyper.py @@ -234,6 +234,8 @@ else: tracking = lambda block: None + for block in pending: + self._processing_block(block) self.call_all_setups(all_threads=True) try: @@ -244,7 +246,6 @@ for block in pending: tracking(block) blockcount += 1 - self._processing_block(block) self.specialize_block(block) self.already_seen[block] = True # progress bar @@ -266,7 +267,6 @@ (len(pending),)) tq = transaction.TransactionQueue() for block in pending: - self._processing_block(block) tq.add(self.specialize_block, block) tq.run() self.log.event('left transactional mode') From noreply at buildbot.pypy.org Sat Mar 21 12:37:17 2015 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 21 Mar 2015 12:37:17 +0100 (CET) Subject: [pypy-commit] pypy vmprof: more test fixing Message-ID: <20150321113717.85CE11C02BB@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76493:c11d670ae976 Date: 2015-03-19 15:54 +0200 http://bitbucket.org/pypy/pypy/changeset/c11d670ae976/ Log: more test fixing diff --git a/rpython/jit/metainterp/test/test_warmstate.py b/rpython/jit/metainterp/test/test_warmstate.py --- a/rpython/jit/metainterp/test/test_warmstate.py +++ b/rpython/jit/metainterp/test/test_warmstate.py @@ -91,12 +91,14 @@ class FakeWarmRunnerDesc: cpu = None memory_manager = None + rtyper = None jitcounter = DeterministicJitCounter() class FakeJitDriverSD: jitdriver = None _green_args_spec = [lltype.Signed, lltype.Float] _get_printable_location_ptr = None _confirm_enter_jit_ptr = None + _get_unique_id_ptr = None _can_never_inline_ptr = None _should_unroll_one_iteration_ptr = None red_args_types = [] @@ -128,6 +130,7 @@ _get_printable_location_ptr = llhelper(GET_LOCATION, get_location) _confirm_enter_jit_ptr = None _can_never_inline_ptr = None + _get_unique_id_ptr = None _should_unroll_one_iteration_ptr = None red_args_types = [] state = WarmEnterState(FakeWarmRunnerDesc(), FakeJitDriverSD()) @@ -154,6 +157,7 @@ _get_printable_location_ptr = None _confirm_enter_jit_ptr = llhelper(ENTER_JIT, confirm_enter_jit) _can_never_inline_ptr = None + _get_unique_id_ptr = None _should_unroll_one_iteration_ptr = None red_args_types = [] @@ -179,6 +183,7 @@ _green_args_spec = [lltype.Signed, lltype.Float] _get_printable_location_ptr = None _confirm_enter_jit_ptr = None + _get_unique_id_ptr = None _can_never_inline_ptr = llhelper(CAN_NEVER_INLINE, can_never_inline) _should_unroll_one_iteration_ptr = None red_args_types = [] From noreply at buildbot.pypy.org Sat Mar 21 12:37:18 2015 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 21 Mar 2015 12:37:18 +0100 (CET) Subject: [pypy-commit] pypy vmprof: kill a test for a killed feature Message-ID: <20150321113718.A8A381C02BB@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76494:9f326dc014f4 Date: 2015-03-19 15:55 +0200 http://bitbucket.org/pypy/pypy/changeset/9f326dc014f4/ Log: kill a test for a killed feature diff --git a/pypy/module/__pypy__/test/test_special.py b/pypy/module/__pypy__/test/test_special.py --- a/pypy/module/__pypy__/test/test_special.py +++ b/pypy/module/__pypy__/test/test_special.py @@ -83,13 +83,6 @@ s = set([2, 3, 4]) assert strategy(s) == "IntegerSetStrategy" - def test_code_info_file(self): - from __pypy__ import all_code_info - - assert len(all_code_info()) > 0 # eh, what else? - exec """def function_of_a_relatively_unique_name(): - pass""" - assert 'function_of_a_relatively_unique_name' in all_code_info() class AppTestJitFeatures(object): spaceconfig = {"translation.jit": True} From noreply at buildbot.pypy.org Sat Mar 21 12:37:19 2015 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 21 Mar 2015 12:37:19 +0100 (CET) Subject: [pypy-commit] pypy vmprof: add rpython-vmprof, a small wrapper that runs rpython with vmprof on Message-ID: <20150321113719.D06BA1C02BB@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76495:027c22aa974a Date: 2015-03-21 13:37 +0200 http://bitbucket.org/pypy/pypy/changeset/027c22aa974a/ Log: add rpython-vmprof, a small wrapper that runs rpython with vmprof on diff --git a/rpython/bin/rpython-vmprof b/rpython/bin/rpython-vmprof new file mode 100755 --- /dev/null +++ b/rpython/bin/rpython-vmprof @@ -0,0 +1,26 @@ +#!/usr/bin/env pypy + +"""RPython translation usage: + +rpython target + +run with --help for more information +""" + +import sys, os +sys.path.insert(0, os.path.dirname(os.path.dirname( + os.path.dirname(os.path.realpath(__file__))))) +from rpython.translator.goal.translate import main + +# no implicit targets +if len(sys.argv) == 1: + print __doc__ + sys.exit(1) + +import _vmprof +x = open("vmprof.log", "w") +_vmprof.enable(x.fileno(), 1000) +try: + main() +finally: + _vmprof.disable() diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py --- a/rpython/jit/backend/test/runner_test.py +++ b/rpython/jit/backend/test/runner_test.py @@ -1792,7 +1792,7 @@ c_box = self.alloc_string("hi there").constbox() c_nest = ConstInt(0) c_id = ConstInt(0) - self.execute_operation(rop.DEBUG_MERGE_POINT, [c_box, c_nest, c_id], 'void') + self.execute_operation(rop.DEBUG_MERGE_POINT, [c_box, c_nest, c_id, c_nest], 'void') self.execute_operation(rop.JIT_DEBUG, [c_box, c_nest, c_nest, c_nest, c_nest], 'void') From noreply at buildbot.pypy.org Sat Mar 21 12:46:47 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 21 Mar 2015 12:46:47 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: fix some more conflicts Message-ID: <20150321114647.50D451C02CD@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76496:ea2e519614ab Date: 2015-03-21 12:34 +0100 http://bitbucket.org/pypy/pypy/changeset/ea2e519614ab/ Log: fix some more conflicts diff --git a/rpython/rtyper/rdict.py b/rpython/rtyper/rdict.py --- a/rpython/rtyper/rdict.py +++ b/rpython/rtyper/rdict.py @@ -25,8 +25,10 @@ custom_eq_hash, force_non_null) def rtyper_makekey(self): - self.dictdef.dictkey .dont_change_any_more = True - self.dictdef.dictvalue.dont_change_any_more = True + if not self.dictdef.dictkey.dont_change_any_more: + self.dictdef.dictkey.dont_change_any_more = True + if not self.dictdef.dictvalue.dont_change_any_more: + self.dictdef.dictvalue.dont_change_any_more = True return (self.__class__, self.dictdef.dictkey, self.dictdef.dictvalue) class __extend__(annmodel.SomeOrderedDict): diff --git a/rpython/rtyper/rtyper.py b/rpython/rtyper/rtyper.py --- a/rpython/rtyper/rtyper.py +++ b/rpython/rtyper/rtyper.py @@ -337,23 +337,19 @@ assert isinstance(v, Variable) v.concretetype = self.bindingrepr(v).lowleveltype - def setup_block_entry(self, block): + def get_block_entry(self, block): if block.operations == () and len(block.inputargs) == 2: # special case for exception blocks: force them to return an # exception type and value in a standardized format - v1, v2 = block.inputargs - v1.concretetype = self.exceptiondata.lltype_of_exception_type - v2.concretetype = self.exceptiondata.lltype_of_exception_value return [self.exceptiondata.r_exception_type, self.exceptiondata.r_exception_value] else: # normal path - result = [] - for a in block.inputargs: - r = self.bindingrepr(a) - a.concretetype = r.lowleveltype - result.append(r) - return result + return [self.bindingrepr(a) for a in block.inputargs] + + def setup_block_entry(self, block, entry_reprs): + for r, a in zip(entry_reprs, block.inputargs): + a.concretetype = r.lowleveltype def make_new_lloplist(self, block): return LowLevelOpList(self, block) @@ -379,7 +375,7 @@ def _specialize_block(self, block): # give the best possible types to the input args try: - self.setup_block_entry(block) + self.setup_block_entry(block, self.get_block_entry(block)) except TyperError, e: self.gottypererror(e, block, "block-entry", None) return # cannot continue this block @@ -469,7 +465,7 @@ can_insert_here = block.exitswitch is None and len(block.exits) == 1 for link in block.exits[skip:]: self._convert_link(block, link) - inputargs_reprs = self.setup_block_entry(link.target) + inputargs_reprs = self.get_block_entry(link.target) newops = self.make_new_lloplist(block) newlinkargs = {} for i in range(len(link.args)): From noreply at buildbot.pypy.org Sat Mar 21 17:10:43 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 21 Mar 2015 17:10:43 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: import stmgc/5da5ba86b2fa Message-ID: <20150321161043.E2E2C1C00FA@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76497:db478464880b Date: 2015-03-21 17:08 +0100 http://bitbucket.org/pypy/pypy/changeset/db478464880b/ Log: import stmgc/5da5ba86b2fa diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision --- a/rpython/translator/stm/src_stm/revision +++ b/rpython/translator/stm/src_stm/revision @@ -1,1 +1,1 @@ -7861b2a77263 +5da5ba86b2fa diff --git a/rpython/translator/stm/src_stm/stm/atomic.h b/rpython/translator/stm/src_stm/stm/atomic.h --- a/rpython/translator/stm/src_stm/stm/atomic.h +++ b/rpython/translator/stm/src_stm/stm/atomic.h @@ -27,11 +27,13 @@ # define HAVE_FULL_EXCHANGE_INSN static inline void spin_loop(void) { asm("pause" : : : "memory"); } static inline void write_fence(void) { asm("" : : : "memory"); } + static inline void read_fence(void) { asm("" : : : "memory"); } #else static inline void spin_loop(void) { asm("" : : : "memory"); } static inline void write_fence(void) { __sync_synchronize(); } + static inline void read_fence(void) { asm("" : : : "memory"); } #endif diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c --- a/rpython/translator/stm/src_stm/stm/core.c +++ b/rpython/translator/stm/src_stm/stm/core.c @@ -323,6 +323,7 @@ /* failed */ stm_fatalerror("reset_transaction_read_version: %m"); } + write_fence(); STM_SEGMENT->transaction_read_version = 1; } @@ -350,20 +351,22 @@ STM_PSEGMENT->shadowstack_at_start_of_transaction = tl->shadowstack; STM_PSEGMENT->threadlocal_at_start_of_transaction = tl->thread_local_obj; + uint8_t rv = STM_SEGMENT->transaction_read_version + 1; + STM_SEGMENT->transaction_read_version = rv; + enter_safe_point_if_requested(); dprintf(("start_transaction\n")); s_mutex_unlock(); - /* Now running the SP_RUNNING start. We can set our - 'transaction_read_version' after releasing the mutex, - because it is only read by a concurrent thread in - stm_commit_transaction(), which waits until SP_RUNNING - threads are paused. + /* Now running the SP_RUNNING start. We can reset the + transaction read version here, but we need to increment it + before, otherwise we can end up in enter_safe_point_if_requested(), + and another thread runs stm_commit_transaction(), and it may find + that we have read some object (even though we didn't do any read + so far). */ - uint8_t old_rv = STM_SEGMENT->transaction_read_version; - STM_SEGMENT->transaction_read_version = old_rv + 1; - if (UNLIKELY(old_rv == 0xff)) { + if (UNLIKELY(rv == 0xff)) { reset_transaction_read_version(); } diff --git a/rpython/translator/stm/src_stm/stm/core.h b/rpython/translator/stm/src_stm/stm/core.h --- a/rpython/translator/stm/src_stm/stm/core.h +++ b/rpython/translator/stm/src_stm/stm/core.h @@ -295,6 +295,7 @@ uint8_t other_transaction_read_version = ((struct stm_segment_info_s *)REAL_ADDRESS(base, STM_PSEGMENT)) ->transaction_read_version; + read_fence(); uint8_t rm = ((struct stm_read_marker_s *) (base + (((uintptr_t)obj) >> 4)))->rm; assert(rm <= other_transaction_read_version); From noreply at buildbot.pypy.org Sat Mar 21 17:10:45 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 21 Mar 2015 17:10:45 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: This test finally passes Message-ID: <20150321161045.0C78E1C00FA@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76498:9af0e8881e06 Date: 2015-03-21 17:10 +0100 http://bitbucket.org/pypy/pypy/changeset/9af0e8881e06/ Log: This test finally passes diff --git a/pypy/module/pypystm/test_pypy_c/test_no_conflict.py b/pypy/module/pypystm/test_pypy_c/test_no_conflict.py --- a/pypy/module/pypystm/test_pypy_c/test_no_conflict.py +++ b/pypy/module/pypystm/test_pypy_c/test_no_conflict.py @@ -68,7 +68,6 @@ self.check_almost_no_conflict(f) def test_weakrefs(self): - py.test.skip("next issue") def f(): import weakref From noreply at buildbot.pypy.org Sat Mar 21 17:11:10 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 21 Mar 2015 17:11:10 +0100 (CET) Subject: [pypy-commit] stmgc default: Issue fix: if thread A is in stm_start_transaction(), it may call Message-ID: <20150321161110.BF6301C00FA@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1738:5da5ba86b2fa Date: 2015-03-21 17:07 +0100 http://bitbucket.org/pypy/stmgc/changeset/5da5ba86b2fa/ Log: Issue fix: if thread A is in stm_start_transaction(), it may call enter_safe_point_if_requested(), and thread B might then run stm_commit_transaction(). If doing so, the thread B notion of "what objects have the read marker in thread A" is wrong. diff --git a/c7/stm/atomic.h b/c7/stm/atomic.h --- a/c7/stm/atomic.h +++ b/c7/stm/atomic.h @@ -27,11 +27,13 @@ # define HAVE_FULL_EXCHANGE_INSN static inline void spin_loop(void) { asm("pause" : : : "memory"); } static inline void write_fence(void) { asm("" : : : "memory"); } + static inline void read_fence(void) { asm("" : : : "memory"); } #else static inline void spin_loop(void) { asm("" : : : "memory"); } static inline void write_fence(void) { __sync_synchronize(); } + static inline void read_fence(void) { asm("" : : : "memory"); } #endif diff --git a/c7/stm/core.c b/c7/stm/core.c --- a/c7/stm/core.c +++ b/c7/stm/core.c @@ -323,6 +323,7 @@ /* failed */ stm_fatalerror("reset_transaction_read_version: %m"); } + write_fence(); STM_SEGMENT->transaction_read_version = 1; } @@ -350,20 +351,22 @@ STM_PSEGMENT->shadowstack_at_start_of_transaction = tl->shadowstack; STM_PSEGMENT->threadlocal_at_start_of_transaction = tl->thread_local_obj; + uint8_t rv = STM_SEGMENT->transaction_read_version + 1; + STM_SEGMENT->transaction_read_version = rv; + enter_safe_point_if_requested(); dprintf(("start_transaction\n")); s_mutex_unlock(); - /* Now running the SP_RUNNING start. We can set our - 'transaction_read_version' after releasing the mutex, - because it is only read by a concurrent thread in - stm_commit_transaction(), which waits until SP_RUNNING - threads are paused. + /* Now running the SP_RUNNING start. We can reset the + transaction read version here, but we need to increment it + before, otherwise we can end up in enter_safe_point_if_requested(), + and another thread runs stm_commit_transaction(), and it may find + that we have read some object (even though we didn't do any read + so far). */ - uint8_t old_rv = STM_SEGMENT->transaction_read_version; - STM_SEGMENT->transaction_read_version = old_rv + 1; - if (UNLIKELY(old_rv == 0xff)) { + if (UNLIKELY(rv == 0xff)) { reset_transaction_read_version(); } diff --git a/c7/stm/core.h b/c7/stm/core.h --- a/c7/stm/core.h +++ b/c7/stm/core.h @@ -295,6 +295,7 @@ uint8_t other_transaction_read_version = ((struct stm_segment_info_s *)REAL_ADDRESS(base, STM_PSEGMENT)) ->transaction_read_version; + read_fence(); uint8_t rm = ((struct stm_read_marker_s *) (base + (((uintptr_t)obj) >> 4)))->rm; assert(rm <= other_transaction_read_version); From noreply at buildbot.pypy.org Sat Mar 21 19:13:12 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 21 Mar 2015 19:13:12 +0100 (CET) Subject: [pypy-commit] cffi default: Some more classifiers Message-ID: <20150321181312.B763B1C02CD@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1674:1c7b1e870144 Date: 2015-03-21 19:13 +0100 http://bitbucket.org/cffi/cffi/changeset/1c7b1e870144/ Log: Some more classifiers diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -162,5 +162,8 @@ 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.2', 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy', ], ) From noreply at buildbot.pypy.org Sat Mar 21 22:23:48 2015 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 21 Mar 2015 22:23:48 +0100 (CET) Subject: [pypy-commit] pypy vmprof: (fijal, arigo) fix an obscure bug by saving/restoring errno, this is sooo obscure Message-ID: <20150321212348.928B01C009F@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76499:f5cbfdb05c32 Date: 2015-03-21 23:23 +0200 http://bitbucket.org/pypy/pypy/changeset/f5cbfdb05c32/ Log: (fijal, arigo) fix an obscure bug by saving/restoring errno, this is sooo obscure diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c --- a/pypy/module/_vmprof/src/vmprof.c +++ b/pypy/module/_vmprof/src/vmprof.c @@ -25,6 +25,7 @@ #include #include #include +#include #define UNW_LOCAL_ONLY #include @@ -200,10 +201,12 @@ static void sigprof_handler(int sig_nr, siginfo_t* info, void *ucontext) { void* stack[MAX_STACK_DEPTH]; + int saved_errno = errno; stack[0] = GetPC((ucontext_t*)ucontext); int depth = frame_forcer(get_stack_trace(stack+1, MAX_STACK_DEPTH-1, ucontext)); depth++; // To account for pc value in stack[0]; prof_write_stacktrace(profile_file, stack, depth, 1); + errno = saved_errno; } /* ************************************************************* From noreply at buildbot.pypy.org Sun Mar 22 09:10:51 2015 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 22 Mar 2015 09:10:51 +0100 (CET) Subject: [pypy-commit] pypy vmprof: improve the situation of writing to a file - dont use buffering for now Message-ID: <20150322081051.52CB51C009F@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76500:71c7b2e97542 Date: 2015-03-22 10:10 +0200 http://bitbucket.org/pypy/pypy/changeset/71c7b2e97542/ Log: improve the situation of writing to a file - dont use buffering for now diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c --- a/pypy/module/_vmprof/src/vmprof.c +++ b/pypy/module/_vmprof/src/vmprof.c @@ -37,7 +37,10 @@ #define MAX_FUNC_NAME 128 #define MAX_STACK_DEPTH 64 -static FILE* profile_file = NULL; + +static int profile_file = 0; +static char profile_write_buffer[4096]; +static int profile_buffer_position = 0; void* vmprof_mainloop_func; static ptrdiff_t mainloop_sp_offset; static vmprof_get_virtual_ip_t mainloop_get_virtual_ip; @@ -52,27 +55,29 @@ #define MARKER_VIRTUAL_IP '\x02' #define MARKER_TRAILER '\x03' -static void prof_word(FILE* f, long x) { - fwrite(&x, sizeof(x), 1, f); +static void prof_word(long x) { + ((long*)(profile_write_buffer + profile_buffer_position))[0] = x; + profile_buffer_position += sizeof(long); } -static void prof_header(FILE* f, long period_usec) { - prof_word(f, 0); - prof_word(f, 3); - prof_word(f, 0); - prof_word(f, period_usec); - prof_word(f, 0); +static void prof_header(long period_usec) { + prof_word(0); + prof_word(3); + prof_word(0); + prof_word(period_usec); + prof_word(0); } -static void prof_write_stacktrace(FILE* f, void** stack, int depth, int count) { +static void prof_write_stacktrace(void** stack, int depth, int count) { int i; char marker = MARKER_STACKTRACE; - fwrite(&marker, 1, 1, f); - prof_word(f, count); - prof_word(f, depth); + profile_write_buffer[profile_buffer_position++] = MARKER_STACKTRACE; + prof_word(count); + prof_word(depth); for(i=0; i Author: Maciej Fijalkowski Branch: vmprof Changeset: r76501:f361fc756693 Date: 2015-03-22 10:50 +0200 http://bitbucket.org/pypy/pypy/changeset/f361fc756693/ Log: make code obj ids more unique diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -2,6 +2,7 @@ from pypy.interpreter.error import OperationError, get_cleared_operation_error from rpython.rlib.unroll import unrolling_iterable from rpython.rlib import jit +from rpython.rlib.objectmodel import we_are_translated TICK_COUNTER_STEP = 100 @@ -37,7 +38,11 @@ if sys.maxint == 2147483647: self._code_unique_id = 0 # XXX this is wrong, it won't work on 32bit else: - self._code_unique_id = 0x7000000000000000 + if we_are_translated(): + self._code_unique_id = 0x7000000000000000 + else: + self._code_unique_id = 0x7700000000000000 + # should be enough code objects self._code_callback = None @staticmethod From noreply at buildbot.pypy.org Sun Mar 22 17:38:13 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 22 Mar 2015 17:38:13 +0100 (CET) Subject: [pypy-commit] stmgc default: a rare (maybe impossible) case Message-ID: <20150322163813.CBA5A1C02BB@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1739:a0b5046a7bea Date: 2015-03-22 08:38 +0100 http://bitbucket.org/pypy/stmgc/changeset/a0b5046a7bea/ Log: a rare (maybe impossible) case diff --git a/c7/stm/core.c b/c7/stm/core.c --- a/c7/stm/core.c +++ b/c7/stm/core.c @@ -351,7 +351,9 @@ STM_PSEGMENT->shadowstack_at_start_of_transaction = tl->shadowstack; STM_PSEGMENT->threadlocal_at_start_of_transaction = tl->thread_local_obj; - uint8_t rv = STM_SEGMENT->transaction_read_version + 1; + uint8_t rv = STM_SEGMENT->transaction_read_version; + if (rv < 0xff) /* else, rare (maybe impossible?) case: we did already */ + rv++; /* incr it but enter_safe_point_if_requested() aborted */ STM_SEGMENT->transaction_read_version = rv; enter_safe_point_if_requested(); From noreply at buildbot.pypy.org Sun Mar 22 17:39:16 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 22 Mar 2015 17:39:16 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: import stmgc/a0b5046a7bea Message-ID: <20150322163916.967A91C024F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76502:19a70527aa8a Date: 2015-03-22 08:38 +0100 http://bitbucket.org/pypy/pypy/changeset/19a70527aa8a/ Log: import stmgc/a0b5046a7bea diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision --- a/rpython/translator/stm/src_stm/revision +++ b/rpython/translator/stm/src_stm/revision @@ -1,1 +1,1 @@ -5da5ba86b2fa +a0b5046a7bea diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c --- a/rpython/translator/stm/src_stm/stm/core.c +++ b/rpython/translator/stm/src_stm/stm/core.c @@ -351,7 +351,9 @@ STM_PSEGMENT->shadowstack_at_start_of_transaction = tl->shadowstack; STM_PSEGMENT->threadlocal_at_start_of_transaction = tl->thread_local_obj; - uint8_t rv = STM_SEGMENT->transaction_read_version + 1; + uint8_t rv = STM_SEGMENT->transaction_read_version; + if (rv < 0xff) /* else, rare (maybe impossible?) case: we did already */ + rv++; /* incr it but enter_safe_point_if_requested() aborted */ STM_SEGMENT->transaction_read_version = rv; enter_safe_point_if_requested(); From noreply at buildbot.pypy.org Sun Mar 22 17:39:17 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 22 Mar 2015 17:39:17 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: merge heads Message-ID: <20150322163917.DE4D51C024F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76503:1f0c0b00b785 Date: 2015-03-22 17:39 +0100 http://bitbucket.org/pypy/pypy/changeset/1f0c0b00b785/ Log: merge heads diff --git a/pypy/module/pypystm/test_pypy_c/test_no_conflict.py b/pypy/module/pypystm/test_pypy_c/test_no_conflict.py --- a/pypy/module/pypystm/test_pypy_c/test_no_conflict.py +++ b/pypy/module/pypystm/test_pypy_c/test_no_conflict.py @@ -68,7 +68,6 @@ self.check_almost_no_conflict(f) def test_weakrefs(self): - py.test.skip("next issue") def f(): import weakref From noreply at buildbot.pypy.org Sun Mar 22 18:12:55 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 22 Mar 2015 18:12:55 +0100 (CET) Subject: [pypy-commit] pypy default: Don't freeze in the pypy binary the version numbers of the libexpat1 Message-ID: <20150322171255.A49051C02BB@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76504:96c123f0695a Date: 2015-03-22 18:12 +0100 http://bitbucket.org/pypy/pypy/changeset/96c123f0695a/ Log: Don't freeze in the pypy binary the version numbers of the libexpat1 library as found during translation. diff --git a/pypy/module/pyexpat/__init__.py b/pypy/module/pyexpat/__init__.py --- a/pypy/module/pyexpat/__init__.py +++ b/pypy/module/pyexpat/__init__.py @@ -39,8 +39,6 @@ 'error': 'space.fromcache(interp_pyexpat.Cache).w_error', '__version__': 'space.wrap("85819")', - 'EXPAT_VERSION': 'interp_pyexpat.get_expat_version(space)', - 'version_info': 'interp_pyexpat.get_expat_version_info(space)', } submodules = { @@ -53,3 +51,10 @@ 'XML_PARAM_ENTITY_PARSING_ALWAYS']: interpleveldefs[name] = 'space.wrap(interp_pyexpat.%s)' % (name,) + def setup_after_space_initialization(self): + from pypy.module.pyexpat import interp_pyexpat + space = self.space + w_ver = interp_pyexpat.get_expat_version(space) + space.setattr(self, space.wrap("EXPAT_VERSION"), w_ver) + w_ver = interp_pyexpat.get_expat_version_info(space) + space.setattr(self, space.wrap("version_info"), w_ver) From noreply at buildbot.pypy.org Sun Mar 22 18:15:34 2015 From: noreply at buildbot.pypy.org (bencord0) Date: Sun, 22 Mar 2015 18:15:34 +0100 (CET) Subject: [pypy-commit] cffi bencord0/also-look-for-ffih-under-usrlocalinclude-1426863787737: Also look for ffi.h under /usr/local/include Message-ID: <20150322171534.9C6AC1C11F4@cobra.cs.uni-duesseldorf.de> Author: Ben Cordero Branch: bencord0/also-look-for-ffih-under-usrlocalinclude-1426863787737 Changeset: r1675:8dc344226542 Date: 2015-03-20 15:03 +0000 http://bitbucket.org/cffi/cffi/changeset/8dc344226542/ Log: Also look for ffi.h under /usr/local/include diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -6,7 +6,8 @@ sources = ['c/_cffi_backend.c'] libraries = ['ffi'] include_dirs = ['/usr/include/ffi', - '/usr/include/libffi'] # may be changed by pkg-config + '/usr/include/libffi', + '/usr/local/include'] # may be changed by pkg-config define_macros = [] library_dirs = [] extra_compile_args = [] From noreply at buildbot.pypy.org Sun Mar 22 18:15:35 2015 From: noreply at buildbot.pypy.org (bencord0) Date: Sun, 22 Mar 2015 18:15:35 +0100 (CET) Subject: [pypy-commit] cffi bencord0/also-look-for-ffih-under-usrlocalinclude-1426863787737: Only add '/usr/local/include' to include_dirs on FreeBSD Message-ID: <20150322171535.9FFEA1C11F4@cobra.cs.uni-duesseldorf.de> Author: Ben Cordero Branch: bencord0/also-look-for-ffih-under-usrlocalinclude-1426863787737 Changeset: r1676:7b9b67e7086d Date: 2015-03-22 07:03 +0000 http://bitbucket.org/cffi/cffi/changeset/7b9b67e7086d/ Log: Only add '/usr/local/include' to include_dirs on FreeBSD diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -6,8 +6,7 @@ sources = ['c/_cffi_backend.c'] libraries = ['ffi'] include_dirs = ['/usr/include/ffi', - '/usr/include/libffi', - '/usr/local/include'] # may be changed by pkg-config + '/usr/include/libffi'] # may be changed by pkg-config define_macros = [] library_dirs = [] extra_compile_args = [] @@ -110,6 +109,9 @@ use_pkg_config() ask_supports_thread() +if 'freebsd' in sys.platform: + include_dirs.append('/usr/local/include') + if __name__ == '__main__': from setuptools import setup, Extension From noreply at buildbot.pypy.org Sun Mar 22 18:15:36 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 22 Mar 2015 18:15:36 +0100 (CET) Subject: [pypy-commit] cffi default: Merged in bencord0/cffi/bencord0/also-look-for-ffih-under-usrlocalinclude-1426863787737 (pull request #57) Message-ID: <20150322171536.A270A1C11F4@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1677:ba891f2e5892 Date: 2015-03-22 18:16 +0100 http://bitbucket.org/cffi/cffi/changeset/ba891f2e5892/ Log: Merged in bencord0/cffi/bencord0/also-look-for-ffih-under- usrlocalinclude-1426863787737 (pull request #57) Also look for ffi.h under /usr/local/include diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -109,6 +109,9 @@ use_pkg_config() ask_supports_thread() +if 'freebsd' in sys.platform: + include_dirs.append('/usr/local/include') + if __name__ == '__main__': from setuptools import setup, Extension From noreply at buildbot.pypy.org Sun Mar 22 22:58:25 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 22 Mar 2015 22:58:25 +0100 (CET) Subject: [pypy-commit] pypy default: Bah, wrong method. It's startup() that is called at runtime. Message-ID: <20150322215825.A35351C10AE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76505:9470a81396fa Date: 2015-03-22 22:58 +0100 http://bitbucket.org/pypy/pypy/changeset/9470a81396fa/ Log: Bah, wrong method. It's startup() that is called at runtime. diff --git a/pypy/module/pyexpat/__init__.py b/pypy/module/pyexpat/__init__.py --- a/pypy/module/pyexpat/__init__.py +++ b/pypy/module/pyexpat/__init__.py @@ -51,9 +51,8 @@ 'XML_PARAM_ENTITY_PARSING_ALWAYS']: interpleveldefs[name] = 'space.wrap(interp_pyexpat.%s)' % (name,) - def setup_after_space_initialization(self): + def startup(self, space): from pypy.module.pyexpat import interp_pyexpat - space = self.space w_ver = interp_pyexpat.get_expat_version(space) space.setattr(self, space.wrap("EXPAT_VERSION"), w_ver) w_ver = interp_pyexpat.get_expat_version_info(space) From noreply at buildbot.pypy.org Mon Mar 23 04:16:00 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Mon, 23 Mar 2015 04:16:00 +0100 (CET) Subject: [pypy-commit] pypy exc-later: add a failing test Message-ID: <20150323031600.5E10A1C01F3@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: exc-later Changeset: r76506:7da3860806b3 Date: 2015-03-23 03:00 +0000 http://bitbucket.org/pypy/pypy/changeset/7da3860806b3/ Log: add a failing test diff --git a/rpython/translator/c/test/test_exception.py b/rpython/translator/c/test/test_exception.py --- a/rpython/translator/c/test/test_exception.py +++ b/rpython/translator/c/test/test_exception.py @@ -28,9 +28,9 @@ b = raise_(i) + 12 c = raise_(i) + 13 return a+b+c - except TestException: + except TestException: return 7 - except MyException: + except MyException: return 123 except: return 22 @@ -173,3 +173,25 @@ f1 = getcompiledopt(fn, [int]) res = f1(100) assert res == 42 + +def test_getitem_custom_exception(): + class MyError(Exception): + pass + class BadContainer(object): + def __getitem__(self, n): + raise MyError + def f(): + d = BadContainer() + try: + return d[0] + except KeyError: + return 1 + def g(): + try: + return f() + except MyError: + return -1 + + assert g() == -1 + compiled = getcompiled(g, []) + assert compiled() == -1 From noreply at buildbot.pypy.org Mon Mar 23 04:16:01 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Mon, 23 Mar 2015 04:16:01 +0100 (CET) Subject: [pypy-commit] pypy exc-later: fix rpython/translator/c/test/test_exception.py::test_getitem_custom_exception Message-ID: <20150323031601.861F91C01F3@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: exc-later Changeset: r76507:ecfb2b754b12 Date: 2015-03-23 03:13 +0000 http://bitbucket.org/pypy/pypy/changeset/ecfb2b754b12/ Log: fix rpython/translator/c/test/test_exception.py::test_getitem_custom _exception diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py --- a/rpython/translator/simplify.py +++ b/rpython/translator/simplify.py @@ -342,7 +342,7 @@ if not exits: block.exitswitch = None block.recloseblock(block.exits[0], *exits) - else: + if Exception in op.canraise: if block.exits[-1].exitcase is not Exception: v_etype = Variable('last_exception') v_exc = Variable('last_exc_value') From noreply at buildbot.pypy.org Mon Mar 23 04:44:24 2015 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Mon, 23 Mar 2015 04:44:24 +0100 (CET) Subject: [pypy-commit] pypy default: Not sure why this code is indenteted 2x Message-ID: <20150323034424.E244F1C024F@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r76508:7a496fc36cd0 Date: 2015-03-22 23:44 -0400 http://bitbucket.org/pypy/pypy/changeset/7a496fc36cd0/ Log: Not sure why this code is indenteted 2x diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py --- a/rpython/translator/simplify.py +++ b/rpython/translator/simplify.py @@ -334,25 +334,25 @@ flowcontext.py). """ for block in list(graph.iterblocks()): - for i in range(len(block.exits)-1, -1, -1): - exit = block.exits[i] - if not (exit.target is graph.exceptblock and - exit.args[0] == Constant(AssertionError)): - continue - # can we remove this exit without breaking the graph? - if len(block.exits) < 2: + for i in range(len(block.exits)-1, -1, -1): + exit = block.exits[i] + if not (exit.target is graph.exceptblock and + exit.args[0] == Constant(AssertionError)): + continue + # can we remove this exit without breaking the graph? + if len(block.exits) < 2: + break + if block.canraise: + if exit.exitcase is None: break - if block.canraise: - if exit.exitcase is None: - break - if len(block.exits) == 2: - # removing the last non-exceptional exit - block.exitswitch = None - exit.exitcase = None - # remove this exit - lst = list(block.exits) - del lst[i] - block.recloseblock(*lst) + if len(block.exits) == 2: + # removing the last non-exceptional exit + block.exitswitch = None + exit.exitcase = None + # remove this exit + lst = list(block.exits) + del lst[i] + block.recloseblock(*lst) # _____________________________________________________________________ From noreply at buildbot.pypy.org Mon Mar 23 09:19:39 2015 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 23 Mar 2015 09:19:39 +0100 (CET) Subject: [pypy-commit] pypy vmprof: increase the stack size Message-ID: <20150323081939.CED9A1C02BB@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76509:83151495f346 Date: 2015-03-23 10:19 +0200 http://bitbucket.org/pypy/pypy/changeset/83151495f346/ Log: increase the stack size diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c --- a/pypy/module/_vmprof/src/vmprof.c +++ b/pypy/module/_vmprof/src/vmprof.c @@ -35,7 +35,7 @@ #define _unused(x) ((void)x) #define MAX_FUNC_NAME 128 -#define MAX_STACK_DEPTH 64 +#define MAX_STACK_DEPTH 1024 static int profile_file = 0; From noreply at buildbot.pypy.org Mon Mar 23 11:49:16 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Mon, 23 Mar 2015 11:49:16 +0100 (CET) Subject: [pypy-commit] pypy default: Remove references to long-dead PyObject from the docs Message-ID: <20150323104916.A124D1C0F98@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r76510:fd7ffb782207 Date: 2015-03-23 10:48 +0000 http://bitbucket.org/pypy/pypy/changeset/fd7ffb782207/ Log: Remove references to long-dead PyObject from the docs diff --git a/rpython/doc/rtyper.rst b/rpython/doc/rtyper.rst --- a/rpython/doc/rtyper.rst +++ b/rpython/doc/rtyper.rst @@ -118,8 +118,7 @@ given this representation. The RTyper also computes a ``concretetype`` for Constants, to match the way they are used in the low-level operations (for example, ``int_add(x, 1)`` requires a ``Constant(1)`` with -``concretetype=Signed``, but an untyped ``add(x, 1)`` works with a -``Constant(1)`` that must actually be a PyObject at run-time). +``concretetype=Signed``). In addition to ``lowleveltype``, each Repr subclass provides a set of methods called ``rtype_op_xxx()`` which define how each high-level operation ``op_xxx`` @@ -306,14 +305,14 @@ ~~~~~~~~~~~~~ As in C, pointers provide the indirection needed to make a reference modifiable -or sharable. Pointers can only point to a structure, an array, a function -(see below) or a PyObject (see below). Pointers to primitive types, if needed, -must be done by pointing to a structure with a single field of the required -type. Pointer types are declared by:: +or sharable. Pointers can only point to a structure, an array or a function +(see below). Pointers to primitive types, if needed, must be done by pointing +to a structure with a single field of the required type. Pointer types are +declared by:: Ptr(TYPE) -At run-time, pointers to GC structures (GcStruct, GcArray and PyObject) hold a +At run-time, pointers to GC structures (GcStruct, GcArray) hold a reference to what they are pointing to. Pointers to non-GC structures that can go away when their container is deallocated (Struct, Array) must be handled with care: the bigger structure of which they are part of could be freed while @@ -356,22 +355,6 @@ :graph: the flow graph of the function. -The PyObject Type -~~~~~~~~~~~~~~~~~ - -This is a special type, for compatibility with CPython: it stands for a -structure compatible with PyObject. This is also a "container" type (thinking -about C, this is ``PyObject``, not ``PyObject*``), so it is usually manipulated -via a Ptr. A typed graph can still contain generic space operations (add, -getitem, etc.) provided they are applied on objects whose low-level type is -``Ptr(PyObject)``. In fact, code generators that support this should consider -that the default type of a variable, if none is specified, is ``Ptr(PyObject)``. -In this way, they can generate the correct code for fully-untyped flow graphs. - -The testing implementation allows you to "create" PyObjects by calling -``pyobjectptr(obj)``. - - Opaque Types ~~~~~~~~~~~~ From noreply at buildbot.pypy.org Mon Mar 23 13:30:09 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Mon, 23 Mar 2015 13:30:09 +0100 (CET) Subject: [pypy-commit] pypy exc-later: Ensure (hackishly) that OverflowError is correctly propagated even when it's not caught explicitly Message-ID: <20150323123009.6B1B71C02BB@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: exc-later Changeset: r76511:1e06bd97b03a Date: 2015-03-23 12:30 +0000 http://bitbucket.org/pypy/pypy/changeset/1e06bd97b03a/ Log: Ensure (hackishly) that OverflowError is correctly propagated even when it's not caught explicitly diff --git a/rpython/translator/c/test/test_exception.py b/rpython/translator/c/test/test_exception.py --- a/rpython/translator/c/test/test_exception.py +++ b/rpython/translator/c/test/test_exception.py @@ -3,6 +3,7 @@ from rpython.translator.c.test import test_typed from rpython.translator.c.test import test_backendoptimized from rpython.rtyper.lltypesystem import lltype +from rpython.rlib.rarithmetic import ovfcheck getcompiled = test_typed.TestTypedTestCase().getcompiled getcompiledopt = test_backendoptimized.TestTypedOptimizedTestCase().getcompiled @@ -195,3 +196,19 @@ assert g() == -1 compiled = getcompiled(g, []) assert compiled() == -1 + +def test_ovf_propagation(): + def div(a, b): + try: + return ovfcheck(a//b) + except ZeroDivisionError: + raise + def f(): + div(4, 2) + try: + return div(-sys.maxint-1, -1) + except OverflowError: + return 0 + assert f() == 0 + compiled = getcompiled(f, []) + assert compiled() == 0 diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py --- a/rpython/translator/simplify.py +++ b/rpython/translator/simplify.py @@ -342,6 +342,16 @@ if not exits: block.exitswitch = None block.recloseblock(block.exits[0], *exits) + if OverflowError in op.canraise: + if not any(issubclass(OverflowError, exit.exitcase) + for exit in block.exits[1:]): + v_etype = const(OverflowError) + v_exc = Variable('last_exc_value') + exit = Link([v_etype, v_exc], graph.exceptblock, OverflowError) + exit.extravars(v_etype, v_exc) + exits = list(block.exits) + exits.append(exit) + block.recloseblock(*exits) if Exception in op.canraise: if block.exits[-1].exitcase is not Exception: v_etype = Variable('last_exception') From noreply at buildbot.pypy.org Mon Mar 23 14:16:24 2015 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 23 Mar 2015 14:16:24 +0100 (CET) Subject: [pypy-commit] pypy vmprof: use a bigger buffer Message-ID: <20150323131624.2AB4E1C0F0E@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76512:88804f9f0357 Date: 2015-03-23 15:16 +0200 http://bitbucket.org/pypy/pypy/changeset/88804f9f0357/ Log: use a bigger buffer diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c --- a/pypy/module/_vmprof/src/vmprof.c +++ b/pypy/module/_vmprof/src/vmprof.c @@ -39,7 +39,7 @@ static int profile_file = 0; -static char profile_write_buffer[4096]; +static char profile_write_buffer[10000]; static int profile_buffer_position = 0; void* vmprof_mainloop_func; static ptrdiff_t mainloop_sp_offset; From noreply at buildbot.pypy.org Mon Mar 23 14:21:43 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 23 Mar 2015 14:21:43 +0100 (CET) Subject: [pypy-commit] pypy default: Make sure this sanity check is not optimized in non-lldebug modes Message-ID: <20150323132143.6BA591C0835@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76513:ebff7c96679e Date: 2015-03-23 14:21 +0100 http://bitbucket.org/pypy/pypy/changeset/ebff7c96679e/ Log: Make sure this sanity check is not optimized in non-lldebug modes diff --git a/rpython/memory/gctransform/asmgcroot.py b/rpython/memory/gctransform/asmgcroot.py --- a/rpython/memory/gctransform/asmgcroot.py +++ b/rpython/memory/gctransform/asmgcroot.py @@ -519,17 +519,15 @@ from rpython.jit.backend.llsupport.jitframe import STACK_DEPTH_OFS tid = self.gc.get_possibly_forwarded_type_id(ebp_in_caller) - ll_assert(rffi.cast(lltype.Signed, tid) == - rffi.cast(lltype.Signed, self.frame_tid), - "found a stack frame that does not belong " - "anywhere I know, bug in asmgcc") - # fish the depth - extra_stack_depth = (ebp_in_caller + STACK_DEPTH_OFS).signed[0] - ll_assert((extra_stack_depth & (rffi.sizeof(lltype.Signed) - 1)) - == 0, "asmgcc: misaligned extra_stack_depth") - extra_stack_depth //= rffi.sizeof(lltype.Signed) - self._shape_decompressor.setjitframe(extra_stack_depth) - return + if (rffi.cast(lltype.Signed, tid) == + rffi.cast(lltype.Signed, self.frame_tid)): + # fish the depth + extra_stack_depth = (ebp_in_caller + STACK_DEPTH_OFS).signed[0] + ll_assert((extra_stack_depth & (rffi.sizeof(lltype.Signed) - 1)) + == 0, "asmgcc: misaligned extra_stack_depth") + extra_stack_depth //= rffi.sizeof(lltype.Signed) + self._shape_decompressor.setjitframe(extra_stack_depth) + return llop.debug_fatalerror(lltype.Void, "cannot find gc roots!") def getlocation(self, callee, ebp_in_caller, location): From noreply at buildbot.pypy.org Mon Mar 23 14:21:44 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 23 Mar 2015 14:21:44 +0100 (CET) Subject: [pypy-commit] pypy default: Issue #2004: fix Message-ID: <20150323132144.8165F1C0835@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76514:58547af971fb Date: 2015-03-23 14:21 +0100 http://bitbucket.org/pypy/pypy/changeset/58547af971fb/ Log: Issue #2004: fix diff --git a/rpython/memory/gctransform/asmgcroot.py b/rpython/memory/gctransform/asmgcroot.py --- a/rpython/memory/gctransform/asmgcroot.py +++ b/rpython/memory/gctransform/asmgcroot.py @@ -368,6 +368,13 @@ if rpy_fastgil != 1: ll_assert(rpy_fastgil != 0, "walk_stack_from doesn't have the GIL") initialframedata = rffi.cast(llmemory.Address, rpy_fastgil) + # + # very rare issue: initialframedata.address[0] is uninitialized + # in this case, but "retaddr = callee.frame_address.address[0]" + # reads it. If it happens to be exactly a valid return address + # inside the C code, then bad things occur. + initialframedata.address[0] = llmemory.NULL + # self.walk_frames(curframe, otherframe, initialframedata) stackscount += 1 # From noreply at buildbot.pypy.org Mon Mar 23 14:21:45 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 23 Mar 2015 14:21:45 +0100 (CET) Subject: [pypy-commit] pypy default: merge heads Message-ID: <20150323132145.A65681C0835@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76515:3fb15976e0c2 Date: 2015-03-23 14:21 +0100 http://bitbucket.org/pypy/pypy/changeset/3fb15976e0c2/ Log: merge heads diff --git a/rpython/doc/rtyper.rst b/rpython/doc/rtyper.rst --- a/rpython/doc/rtyper.rst +++ b/rpython/doc/rtyper.rst @@ -118,8 +118,7 @@ given this representation. The RTyper also computes a ``concretetype`` for Constants, to match the way they are used in the low-level operations (for example, ``int_add(x, 1)`` requires a ``Constant(1)`` with -``concretetype=Signed``, but an untyped ``add(x, 1)`` works with a -``Constant(1)`` that must actually be a PyObject at run-time). +``concretetype=Signed``). In addition to ``lowleveltype``, each Repr subclass provides a set of methods called ``rtype_op_xxx()`` which define how each high-level operation ``op_xxx`` @@ -306,14 +305,14 @@ ~~~~~~~~~~~~~ As in C, pointers provide the indirection needed to make a reference modifiable -or sharable. Pointers can only point to a structure, an array, a function -(see below) or a PyObject (see below). Pointers to primitive types, if needed, -must be done by pointing to a structure with a single field of the required -type. Pointer types are declared by:: +or sharable. Pointers can only point to a structure, an array or a function +(see below). Pointers to primitive types, if needed, must be done by pointing +to a structure with a single field of the required type. Pointer types are +declared by:: Ptr(TYPE) -At run-time, pointers to GC structures (GcStruct, GcArray and PyObject) hold a +At run-time, pointers to GC structures (GcStruct, GcArray) hold a reference to what they are pointing to. Pointers to non-GC structures that can go away when their container is deallocated (Struct, Array) must be handled with care: the bigger structure of which they are part of could be freed while @@ -356,22 +355,6 @@ :graph: the flow graph of the function. -The PyObject Type -~~~~~~~~~~~~~~~~~ - -This is a special type, for compatibility with CPython: it stands for a -structure compatible with PyObject. This is also a "container" type (thinking -about C, this is ``PyObject``, not ``PyObject*``), so it is usually manipulated -via a Ptr. A typed graph can still contain generic space operations (add, -getitem, etc.) provided they are applied on objects whose low-level type is -``Ptr(PyObject)``. In fact, code generators that support this should consider -that the default type of a variable, if none is specified, is ``Ptr(PyObject)``. -In this way, they can generate the correct code for fully-untyped flow graphs. - -The testing implementation allows you to "create" PyObjects by calling -``pyobjectptr(obj)``. - - Opaque Types ~~~~~~~~~~~~ diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py --- a/rpython/translator/simplify.py +++ b/rpython/translator/simplify.py @@ -334,25 +334,25 @@ flowcontext.py). """ for block in list(graph.iterblocks()): - for i in range(len(block.exits)-1, -1, -1): - exit = block.exits[i] - if not (exit.target is graph.exceptblock and - exit.args[0] == Constant(AssertionError)): - continue - # can we remove this exit without breaking the graph? - if len(block.exits) < 2: + for i in range(len(block.exits)-1, -1, -1): + exit = block.exits[i] + if not (exit.target is graph.exceptblock and + exit.args[0] == Constant(AssertionError)): + continue + # can we remove this exit without breaking the graph? + if len(block.exits) < 2: + break + if block.canraise: + if exit.exitcase is None: break - if block.canraise: - if exit.exitcase is None: - break - if len(block.exits) == 2: - # removing the last non-exceptional exit - block.exitswitch = None - exit.exitcase = None - # remove this exit - lst = list(block.exits) - del lst[i] - block.recloseblock(*lst) + if len(block.exits) == 2: + # removing the last non-exceptional exit + block.exitswitch = None + exit.exitcase = None + # remove this exit + lst = list(block.exits) + del lst[i] + block.recloseblock(*lst) # _____________________________________________________________________ From noreply at buildbot.pypy.org Mon Mar 23 14:24:31 2015 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 23 Mar 2015 14:24:31 +0100 (CET) Subject: [pypy-commit] pypy vmprof: make this buffer even bigger, we can have more stuff in there Message-ID: <20150323132431.16BB91C0F0E@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76516:4025e049f4fb Date: 2015-03-23 15:21 +0200 http://bitbucket.org/pypy/pypy/changeset/4025e049f4fb/ Log: make this buffer even bigger, we can have more stuff in there diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c --- a/pypy/module/_vmprof/src/vmprof.c +++ b/pypy/module/_vmprof/src/vmprof.c @@ -39,7 +39,7 @@ static int profile_file = 0; -static char profile_write_buffer[10000]; +static char profile_write_buffer[100000]; static int profile_buffer_position = 0; void* vmprof_mainloop_func; static ptrdiff_t mainloop_sp_offset; From noreply at buildbot.pypy.org Mon Mar 23 14:28:15 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Mon, 23 Mar 2015 14:28:15 +0100 (CET) Subject: [pypy-commit] pypy exc-later: fixes Message-ID: <20150323132815.577CE1C0835@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: exc-later Changeset: r76517:5e9c2ef07cb3 Date: 2015-03-23 13:28 +0000 http://bitbucket.org/pypy/pypy/changeset/5e9c2ef07cb3/ Log: fixes diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py --- a/rpython/translator/simplify.py +++ b/rpython/translator/simplify.py @@ -339,9 +339,9 @@ continue if has_generic_case: exits += exc_exits - if not exits: - block.exitswitch = None - block.recloseblock(block.exits[0], *exits) + exits.insert(0, block.exits[0]) + else: + exits = list(block.exits) if OverflowError in op.canraise: if not any(issubclass(OverflowError, exit.exitcase) for exit in block.exits[1:]): @@ -349,18 +349,17 @@ v_exc = Variable('last_exc_value') exit = Link([v_etype, v_exc], graph.exceptblock, OverflowError) exit.extravars(v_etype, v_exc) - exits = list(block.exits) exits.append(exit) - block.recloseblock(*exits) if Exception in op.canraise: if block.exits[-1].exitcase is not Exception: v_etype = Variable('last_exception') v_exc = Variable('last_exc_value') exit = Link([v_etype, v_exc], graph.exceptblock, Exception) exit.extravars(v_etype, v_exc) - exits = list(block.exits) exits.append(exit) - block.recloseblock(*exits) + block.recloseblock(*exits) + if len(exits) == 1: + block.exitswitch = None def remove_assertion_errors(graph): diff --git a/rpython/translator/test/test_simplify.py b/rpython/translator/test/test_simplify.py --- a/rpython/translator/test/test_simplify.py +++ b/rpython/translator/test/test_simplify.py @@ -71,7 +71,7 @@ graph, _ = translate(f, [int, int]) assert len(graph.startblock.operations) == 1 assert graph.startblock.operations[0].opname == 'int_floordiv_ovf_zer' - assert len(graph.startblock.exits) == 2 + assert len(graph.startblock.exits) == 3 assert graph.startblock.exits[1].target is graph.returnblock def test_remove_direct_call_without_side_effects(): From noreply at buildbot.pypy.org Mon Mar 23 14:28:58 2015 From: noreply at buildbot.pypy.org (groggi) Date: Mon, 23 Mar 2015 14:28:58 +0100 (CET) Subject: [pypy-commit] pypy gc-incminimark-pinning-improve: branch to improve current object pinning implementation for incminimark. Message-ID: <20150323132858.0D4901C0835@cobra.cs.uni-duesseldorf.de> Author: Gregor Wegberg Branch: gc-incminimark-pinning-improve Changeset: r76518:71b312e97a78 Date: 2014-12-11 18:29 +0100 http://bitbucket.org/pypy/pypy/changeset/71b312e97a78/ Log: branch to improve current object pinning implementation for incminimark. From noreply at buildbot.pypy.org Mon Mar 23 14:28:59 2015 From: noreply at buildbot.pypy.org (groggi) Date: Mon, 23 Mar 2015 14:28:59 +0100 (CET) Subject: [pypy-commit] pypy gc-incminimark-pinning-improve: added comments to collect_and_reserve Message-ID: <20150323132859.348151C0835@cobra.cs.uni-duesseldorf.de> Author: Gregor Wegberg Branch: gc-incminimark-pinning-improve Changeset: r76519:1fdd47a9602c Date: 2014-12-11 20:43 +0100 http://bitbucket.org/pypy/pypy/changeset/1fdd47a9602c/ Log: added comments to collect_and_reserve diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -682,7 +682,7 @@ def collect_and_reserve(self, totalsize): """To call when nursery_free overflows nursery_top. First check if the nursery_top is the real top, otherwise we - can just move the top of one cleanup and continue + can just move the top of one cleanup and continue. Do a minor collection, and possibly also a major collection, and finally reserve 'totalsize' bytes at the start of the @@ -694,10 +694,22 @@ self.nursery_free = llmemory.NULL # debug: don't use me if self.nursery_barriers.non_empty(): + # Pinned object in front of nursery_top. Try by jumping into the + # next area inside the nursery. 'Next area' is in this case the + # pinned object after the next pinned one and the one after the + # next or the end of the nursery. Graphically explained: + # + # |- allocating totalsize failed in this are + # v v- next pinned object, jump over + # +---------+--------+--------+--------+-----------+ + # | unknown | pinned | empty | pinned | empty | <- nursery + # +---------+--------+--------+--------+-----------+ + # ^- try allocating totalsize in here next + # size_gc_header = self.gcheaderbuilder.size_gc_header pinned_obj_size = size_gc_header + self.get_size( self.nursery_top + size_gc_header) - + # self.nursery_free = self.nursery_top + pinned_obj_size self.nursery_top = self.nursery_barriers.popleft() else: @@ -725,6 +737,8 @@ "Seeing minor_collection() at least twice." "Too many pinned objects?") # + # GC tried to do something about nursery_free overflowing + # nursery_top before this point. Try to allocate totalsize now. result = self.nursery_free if self.nursery_free + totalsize <= self.nursery_top: self.nursery_free = result + totalsize From noreply at buildbot.pypy.org Mon Mar 23 14:29:00 2015 From: noreply at buildbot.pypy.org (groggi) Date: Mon, 23 Mar 2015 14:29:00 +0100 (CET) Subject: [pypy-commit] pypy gc-incminimark-pinning-improve: XXX: should try merging old_objects_pointing_to_pinned into old_objects_pointing_to_young Message-ID: <20150323132900.5CC2A1C0835@cobra.cs.uni-duesseldorf.de> Author: Gregor Wegberg Branch: gc-incminimark-pinning-improve Changeset: r76520:7bb618bd9e31 Date: 2014-12-11 20:49 +0100 http://bitbucket.org/pypy/pypy/changeset/7bb618bd9e31/ Log: XXX: should try merging old_objects_pointing_to_pinned into old_objects_pointing_to_young diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -52,6 +52,9 @@ # XXX total addressable size. Maybe by keeping some minimarkpage arenas # XXX pre-reserved, enough for a few nursery collections? What about # XXX raw-malloced memory? + +# XXX try merging old_objects_pointing_to_pinned into +# XXX old_objects_pointing_to_young (IRC 2014-10-22, fijal and gregor_w) import sys from rpython.rtyper.lltypesystem import lltype, llmemory, llarena, llgroup from rpython.rtyper.lltypesystem.lloperation import llop From noreply at buildbot.pypy.org Mon Mar 23 14:29:01 2015 From: noreply at buildbot.pypy.org (groggi) Date: Mon, 23 Mar 2015 14:29:01 +0100 (CET) Subject: [pypy-commit] pypy gc-incminimark-pinning-improve: fix typo Message-ID: <20150323132901.788B81C0835@cobra.cs.uni-duesseldorf.de> Author: Gregor Wegberg Branch: gc-incminimark-pinning-improve Changeset: r76521:96b6aa76b5d0 Date: 2014-12-11 20:50 +0100 http://bitbucket.org/pypy/pypy/changeset/96b6aa76b5d0/ Log: fix typo diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -702,7 +702,7 @@ # pinned object after the next pinned one and the one after the # next or the end of the nursery. Graphically explained: # - # |- allocating totalsize failed in this are + # |- allocating totalsize failed in this area # v v- next pinned object, jump over # +---------+--------+--------+--------+-----------+ # | unknown | pinned | empty | pinned | empty | <- nursery From noreply at buildbot.pypy.org Mon Mar 23 14:29:02 2015 From: noreply at buildbot.pypy.org (groggi) Date: Mon, 23 Mar 2015 14:29:02 +0100 (CET) Subject: [pypy-commit] pypy gc-incminimark-pinning-improve: try again to make collect_and_reserve clearer Message-ID: <20150323132902.90B5A1C0835@cobra.cs.uni-duesseldorf.de> Author: Gregor Wegberg Branch: gc-incminimark-pinning-improve Changeset: r76522:ffabb60fdaf5 Date: 2014-12-12 17:12 +0100 http://bitbucket.org/pypy/pypy/changeset/ffabb60fdaf5/ Log: try again to make collect_and_reserve clearer diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -684,12 +684,10 @@ def collect_and_reserve(self, totalsize): """To call when nursery_free overflows nursery_top. - First check if the nursery_top is the real top, otherwise we - can just move the top of one cleanup and continue. - - Do a minor collection, and possibly also a major collection, - and finally reserve 'totalsize' bytes at the start of the - now-empty nursery. + First check if pinned objects are in front of nursery_top. If so, + jump over the pinned object and try again to reserve totalsize. + Otherwise do a minor collection, and possibly a major collection, and + finally reserve totalsize bytes. """ minor_collection_count = 0 @@ -697,17 +695,18 @@ self.nursery_free = llmemory.NULL # debug: don't use me if self.nursery_barriers.non_empty(): - # Pinned object in front of nursery_top. Try by jumping into the - # next area inside the nursery. 'Next area' is in this case the - # pinned object after the next pinned one and the one after the - # next or the end of the nursery. Graphically explained: + # Pinned object in front of nursery_top. Try reserving totalsize + # by jumping into the next, yet unused, area inside the + # nursery. 'Next area' is in this case the space inside the + # nursery between the next pinned object and the one after that + # or the end of the nursery. Graphically explained: # # |- allocating totalsize failed in this area - # v v- next pinned object, jump over + # v v- next pinned object, jump over this one # +---------+--------+--------+--------+-----------+ # | unknown | pinned | empty | pinned | empty | <- nursery # +---------+--------+--------+--------+-----------+ - # ^- try allocating totalsize in here next + # ^- try reserving totalsize in here next # size_gc_header = self.gcheaderbuilder.size_gc_header pinned_obj_size = size_gc_header + self.get_size( @@ -740,8 +739,9 @@ "Seeing minor_collection() at least twice." "Too many pinned objects?") # - # GC tried to do something about nursery_free overflowing - # nursery_top before this point. Try to allocate totalsize now. + # Tried to do something about nursery_free overflowing + # nursery_top before this point. Try to reserve totalsize now. + # If this succeeds break out of loop. result = self.nursery_free if self.nursery_free + totalsize <= self.nursery_top: self.nursery_free = result + totalsize From noreply at buildbot.pypy.org Mon Mar 23 14:29:18 2015 From: noreply at buildbot.pypy.org (groggi) Date: Mon, 23 Mar 2015 14:29:18 +0100 (CET) Subject: [pypy-commit] pypy gc-incminimark-pinning-improve: Merge default into gc-incminimark-pinning-improve Message-ID: <20150323132918.16E1E1C0835@cobra.cs.uni-duesseldorf.de> Author: Gregor Wegberg Branch: gc-incminimark-pinning-improve Changeset: r76523:1c5f5f6406cf Date: 2015-03-21 10:41 +0100 http://bitbucket.org/pypy/pypy/changeset/1c5f5f6406cf/ Log: Merge default into gc-incminimark-pinning-improve diff too long, truncating to 2000 out of 67422 lines diff --git a/.gitignore b/.gitignore --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,10 @@ bin/pypy-c include/*.h +include/numpy/ lib_pypy/ctypes_config_cache/_[^_]*_*.py +libpypy-c.* +pypy-c pypy/_cache pypy/doc/*.html pypy/doc/config/*.html @@ -18,4 +21,5 @@ pypy/translator/c/src/dtoa.o pypy/translator/goal/pypy-c pypy/translator/goal/target*-c -release/ \ No newline at end of file +release/ +rpython/_cache/ diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -7,10 +7,12 @@ 9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm ab0dd631c22015ed88e583d9fdd4c43eebf0be21 pypy-2.1-beta1-arm 20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0 -20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0 -0000000000000000000000000000000000000000 release-2.3.0 394146e9bb673514c61f0150ab2013ccf78e8de7 release-2.3 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1 -32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1 -0000000000000000000000000000000000000000 release-2.2=3.1 +10f1b29a2bd21f837090286174a9ca030b8680b2 release-2.5.0 +8e24dac0b8e2db30d46d59f2c4daa3d4aaab7861 release-2.5.1 +8e24dac0b8e2db30d46d59f2c4daa3d4aaab7861 release-2.5.1 +0000000000000000000000000000000000000000 release-2.5.1 +0000000000000000000000000000000000000000 release-2.5.1 +e3d046c43451403f5969580fc1c41d5df6c4082a release-2.5.1 diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -28,7 +28,7 @@ DEALINGS IN THE SOFTWARE. -PyPy Copyright holders 2003-2014 +PyPy Copyright holders 2003-2015 ----------------------------------- Except when otherwise stated (look for LICENSE files or information at @@ -42,19 +42,19 @@ Amaury Forgeot d'Arc Samuele Pedroni Alex Gaynor + Brian Kearns + Matti Picus + Philip Jenvey Michael Hudson David Schneider - Matti Picus - Brian Kearns - Philip Jenvey Holger Krekel Christian Tismer Hakan Ardo Benjamin Peterson Manuel Jacob + Ronan Lamy Anders Chrigstrom Eric van Riet Paap - Ronan Lamy Wim Lavrijsen Richard Emslie Alexander Schremmer @@ -68,9 +68,9 @@ Camillo Bruni Laura Creighton Toon Verwaest + Romain Guillebert Leonardo Santagada Seo Sanghyeon - Romain Guillebert Justin Peel Ronny Pfannschmidt David Edelsohn @@ -91,15 +91,16 @@ Michal Bendowski Jan de Mooij stian + Tyler Wade Michael Foord Stephan Diehl - Tyler Wade Stefan Schwarzer Valentino Volonghi Tomek Meka Patrick Maupin Bob Ippolito Bruno Gola + David Malcolm Jean-Paul Calderone Timo Paulssen Squeaky @@ -108,18 +109,19 @@ Marius Gedminas Martin Matusiak Konstantin Lopuhin + Wenzhu Man John Witulski - Wenzhu Man + Laurence Tratt + Ivan Sichmann Freitas Greg Price Dario Bertini Mark Pearse Simon Cross - Ivan Sichmann Freitas Andreas Stührk + Stefano Rivera Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov - Stefano Rivera Paweł Piotr Przeradowski Paul deGrandis Ilya Osadchiy @@ -129,7 +131,6 @@ tav Taavi Burns Georg Brandl - Laurence Tratt Bert Freudenberg Stian Andreassen Wanja Saatkamp @@ -141,13 +142,12 @@ Jeremy Thurgood Rami Chowdhury Tobias Pape - David Malcolm Eugene Oden Henry Mason Vasily Kuznetsov Preston Timmons + David Ripton Jeff Terrace - David Ripton Dusty Phillips Lukas Renggli Guenter Jantzen @@ -166,13 +166,16 @@ Gintautas Miliauskas Michael Twomey Lucian Branescu Mihaila + Yichao Yu Gabriel Lavoie Olivier Dormond Jared Grubb Karl Bartel + Wouter van Heyst Brian Dorsey Victor Stinner Andrews Medina + anatoly techtonik Stuart Williams Jasper Schulz Christian Hudon @@ -182,12 +185,11 @@ Michael Cheng Justas Sadzevicius Gasper Zejn - anatoly techtonik Neil Shepperd + Stanislaw Halik Mikael Schönenberg Elmo M?ntynen Jonathan David Riehl - Stanislaw Halik Anders Qvist Corbin Simpson Chirag Jadwani @@ -196,10 +198,13 @@ Vincent Legoll Alan McIntyre Alexander Sedov + Attila Gobi Christopher Pope Christian Tismer Marc Abramowitz Dan Stromberg + Arjun Naik + Valentina Mukhamedzhanova Stefano Parmesan Alexis Daboville Jens-Uwe Mager @@ -213,8 +218,6 @@ Sylvain Thenault Nathan Taylor Vladimir Kryachko - Arjun Naik - Attila Gobi Jacek Generowicz Alejandro J. Cura Jacob Oscarson @@ -222,22 +225,23 @@ Ryan Gonzalez Ian Foote Kristjan Valur Jonsson + David Lievens Neil Blakey-Milner Lutz Paelike Lucio Torre Lars Wassermann - Valentina Mukhamedzhanova Henrik Vendelbo Dan Buch Miguel de Val Borro Artur Lisiecki Sergey Kishchenko - Yichao Yu Ignas Mikalajunas Christoph Gerum Martin Blais Lene Wagner Tomo Cocoa + Toni Mattis + Lucas Stadler roberto at goyle Yury V. Zaytsev Anna Katrina Dominguez @@ -265,23 +269,30 @@ Stephan Busemann Rafał Gałczyński Christian Muirhead + Berker Peksag James Lan shoma hosaka - Daniel Neuh?user - Matthew Miller + Daniel Neuhäuser + Ben Mather + halgari + Boglarka Vezer + Chris Pressey Buck Golemon Konrad Delong Dinu Gherman Chris Lambacher coolbutuseless at gmail.com + Jim Baker Rodrigo Araújo - Jim Baker + Nikolaos-Digenis Karagiannis James Robert Armin Ronacher Brett Cannon + Donald Stufft yrttyr aliceinwire OlivierBlanvillain + Dan Sanders Zooko Wilcox-O Hearn Tomer Chachamu Christopher Groskopf @@ -295,6 +306,7 @@ Markus Unterwaditzer Even Wiik Thomassen jbs + squeaky soareschen Kurt Griffiths Mike Bayer @@ -306,6 +318,7 @@ Anna Ravencroft Dan Crosta Julien Phalip + Roman Podoliaka Dan Loewenherz Heinrich-Heine University, Germany diff --git a/lib-python/2.7/CGIHTTPServer.py b/lib-python/2.7/CGIHTTPServer.py --- a/lib-python/2.7/CGIHTTPServer.py +++ b/lib-python/2.7/CGIHTTPServer.py @@ -106,16 +106,16 @@ def run_cgi(self): """Execute a CGI script.""" dir, rest = self.cgi_info - - i = rest.find('/') + path = dir + '/' + rest + i = path.find('/', len(dir)+1) while i >= 0: - nextdir = rest[:i] - nextrest = rest[i+1:] + nextdir = path[:i] + nextrest = path[i+1:] scriptdir = self.translate_path(nextdir) if os.path.isdir(scriptdir): dir, rest = nextdir, nextrest - i = rest.find('/') + i = path.find('/', len(dir)+1) else: break diff --git a/lib-python/2.7/Cookie.py b/lib-python/2.7/Cookie.py --- a/lib-python/2.7/Cookie.py +++ b/lib-python/2.7/Cookie.py @@ -56,7 +56,7 @@ >>> C = Cookie.SmartCookie() [Note: Long-time users of Cookie.py will remember using -Cookie.Cookie() to create an Cookie object. Although deprecated, it +Cookie.Cookie() to create a Cookie object. Although deprecated, it is still supported by the code. See the Backward Compatibility notes for more information.] @@ -426,6 +426,8 @@ "version" : "Version", } + _flags = {'secure', 'httponly'} + def __init__(self): # Set defaults self.key = self.value = self.coded_value = None @@ -529,9 +531,11 @@ _LegalCharsPatt = r"[\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=]" _CookiePattern = re.compile( r"(?x)" # This is a Verbose pattern + r"\s*" # Optional whitespace at start of cookie r"(?P" # Start of group 'key' ""+ _LegalCharsPatt +"+?" # Any word of at least one letter, nongreedy r")" # End of group 'key' + r"(" # Optional group: there may not be a value. r"\s*=\s*" # Equal Sign r"(?P" # Start of group 'val' r'"(?:[^\\"]|\\.)*"' # Any doublequoted string @@ -540,7 +544,9 @@ r"|" # or ""+ _LegalCharsPatt +"*" # Any word or empty string r")" # End of group 'val' - r"\s*;?" # Probably ending in a semi-colon + r")?" # End of optional value group + r"\s*" # Any number of spaces. + r"(\s+|;|$)" # Ending either at space, semicolon, or EOS. ) @@ -585,8 +591,12 @@ def __setitem__(self, key, value): """Dictionary style assignment.""" - rval, cval = self.value_encode(value) - self.__set(key, rval, cval) + if isinstance(value, Morsel): + # allow assignment of constructed Morsels (e.g. for pickling) + dict.__setitem__(self, key, value) + else: + rval, cval = self.value_encode(value) + self.__set(key, rval, cval) # end __setitem__ def output(self, attrs=None, header="Set-Cookie:", sep="\015\012"): @@ -641,7 +651,7 @@ while 0 <= i < n: # Start looking for a cookie - match = patt.search(str, i) + match = patt.match(str, i) if not match: break # No more cookies K,V = match.group("key"), match.group("val") @@ -656,8 +666,12 @@ M[ K[1:] ] = V elif K.lower() in Morsel._reserved: if M: - M[ K ] = _unquote(V) - else: + if V is None: + if K.lower() in Morsel._flags: + M[K] = True + else: + M[K] = _unquote(V) + elif V is not None: rval, cval = self.value_decode(V) self.__set(K, rval, cval) M = self[K] diff --git a/lib-python/2.7/SocketServer.py b/lib-python/2.7/SocketServer.py --- a/lib-python/2.7/SocketServer.py +++ b/lib-python/2.7/SocketServer.py @@ -416,8 +416,12 @@ self.socket = socket.socket(self.address_family, self.socket_type) if bind_and_activate: - self.server_bind() - self.server_activate() + try: + self.server_bind() + self.server_activate() + except: + self.server_close() + raise def server_bind(self): """Called by constructor to bind the socket. diff --git a/lib-python/2.7/_abcoll.py b/lib-python/2.7/_abcoll.py --- a/lib-python/2.7/_abcoll.py +++ b/lib-python/2.7/_abcoll.py @@ -143,7 +143,7 @@ methods except for __contains__, __iter__ and __len__. To override the comparisons (presumably for speed, as the - semantics are fixed), all you have to do is redefine __le__ and + semantics are fixed), redefine __le__ and __ge__, then the other operations will automatically follow suit. """ diff --git a/lib-python/2.7/argparse.py b/lib-python/2.7/argparse.py --- a/lib-python/2.7/argparse.py +++ b/lib-python/2.7/argparse.py @@ -1089,7 +1089,14 @@ # parse all the remaining options into the namespace # store any unrecognized options on the object, so that the top # level parser can decide what to do with them - namespace, arg_strings = parser.parse_known_args(arg_strings, namespace) + + # In case this subparser defines new defaults, we parse them + # in a new namespace object and then update the original + # namespace for the relevant parts. + subnamespace, arg_strings = parser.parse_known_args(arg_strings, None) + for key, value in vars(subnamespace).items(): + setattr(namespace, key, value) + if arg_strings: vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, []) getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings) diff --git a/lib-python/2.7/asynchat.py b/lib-python/2.7/asynchat.py --- a/lib-python/2.7/asynchat.py +++ b/lib-python/2.7/asynchat.py @@ -46,12 +46,17 @@ you - by calling your self.found_terminator() method. """ +import asyncore +import errno import socket -import asyncore from collections import deque from sys import py3kwarning from warnings import filterwarnings, catch_warnings +_BLOCKING_IO_ERRORS = (errno.EAGAIN, errno.EALREADY, errno.EINPROGRESS, + errno.EWOULDBLOCK) + + class async_chat (asyncore.dispatcher): """This is an abstract class. You must derive from this class, and add the two methods collect_incoming_data() and found_terminator()""" @@ -109,6 +114,8 @@ try: data = self.recv (self.ac_in_buffer_size) except socket.error, why: + if why.args[0] in _BLOCKING_IO_ERRORS: + return self.handle_error() return diff --git a/lib-python/2.7/bsddb/test/test_queue.py b/lib-python/2.7/bsddb/test/test_queue.py --- a/lib-python/2.7/bsddb/test/test_queue.py +++ b/lib-python/2.7/bsddb/test/test_queue.py @@ -10,6 +10,7 @@ #---------------------------------------------------------------------- + at unittest.skip("fails on Windows; see issue 22943") class SimpleQueueTestCase(unittest.TestCase): def setUp(self): self.filename = get_new_database_path() diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py --- a/lib-python/2.7/collections.py +++ b/lib-python/2.7/collections.py @@ -17,6 +17,10 @@ except ImportError: assert '__pypy__' not in _sys.builtin_module_names newdict = lambda _ : {} +try: + from __pypy__ import reversed_dict +except ImportError: + reversed_dict = lambda d: reversed(d.keys()) try: from thread import get_ident as _get_ident @@ -29,142 +33,35 @@ ################################################################################ class OrderedDict(dict): - 'Dictionary that remembers insertion order' - # An inherited dict maps keys to values. - # The inherited dict provides __getitem__, __len__, __contains__, and get. - # The remaining methods are order-aware. - # Big-O running times for all methods are the same as regular dictionaries. + '''Dictionary that remembers insertion order. - # The internal self.__map dict maps keys to links in a doubly linked list. - # The circular doubly linked list starts and ends with a sentinel element. - # The sentinel element never gets deleted (this simplifies the algorithm). - # Each link is stored as a list of length three: [PREV, NEXT, KEY]. + In PyPy all dicts are ordered anyway. This is mostly useful as a + placeholder to mean "this dict must be ordered even on CPython". - def __init__(self, *args, **kwds): - '''Initialize an ordered dictionary. The signature is the same as - regular dictionaries, but keyword arguments are not recommended because - their insertion order is arbitrary. - - ''' - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - try: - self.__root - except AttributeError: - self.__root = root = [] # sentinel node - root[:] = [root, root, None] - self.__map = {} - self.__update(*args, **kwds) - - def __setitem__(self, key, value, dict_setitem=dict.__setitem__): - 'od.__setitem__(i, y) <==> od[i]=y' - # Setting a new item creates a new link at the end of the linked list, - # and the inherited dictionary is updated with the new key/value pair. - if key not in self: - root = self.__root - last = root[0] - last[1] = root[0] = self.__map[key] = [last, root, key] - return dict_setitem(self, key, value) - - def __delitem__(self, key, dict_delitem=dict.__delitem__): - 'od.__delitem__(y) <==> del od[y]' - # Deleting an existing item uses self.__map to find the link which gets - # removed by updating the links in the predecessor and successor nodes. - dict_delitem(self, key) - link_prev, link_next, _ = self.__map.pop(key) - link_prev[1] = link_next # update link_prev[NEXT] - link_next[0] = link_prev # update link_next[PREV] - - def __iter__(self): - 'od.__iter__() <==> iter(od)' - # Traverse the linked list in order. - root = self.__root - curr = root[1] # start at the first node - while curr is not root: - yield curr[2] # yield the curr[KEY] - curr = curr[1] # move to next node + Known difference: iterating over an OrderedDict which is being + concurrently modified raises RuntimeError in PyPy. In CPython + instead we get some behavior that appears reasonable in some + cases but is nonsensical in other cases. This is officially + forbidden by the CPython docs, so we forbid it explicitly for now. + ''' def __reversed__(self): - 'od.__reversed__() <==> reversed(od)' - # Traverse the linked list in reverse order. - root = self.__root - curr = root[0] # start at the last node - while curr is not root: - yield curr[2] # yield the curr[KEY] - curr = curr[0] # move to previous node - - def clear(self): - 'od.clear() -> None. Remove all items from od.' - root = self.__root - root[:] = [root, root, None] - self.__map.clear() - dict.clear(self) - - # -- the following methods do not depend on the internal structure -- - - def keys(self): - 'od.keys() -> list of keys in od' - return list(self) - - def values(self): - 'od.values() -> list of values in od' - return [self[key] for key in self] - - def items(self): - 'od.items() -> list of (key, value) pairs in od' - return [(key, self[key]) for key in self] - - def iterkeys(self): - 'od.iterkeys() -> an iterator over the keys in od' - return iter(self) - - def itervalues(self): - 'od.itervalues -> an iterator over the values in od' - for k in self: - yield self[k] - - def iteritems(self): - 'od.iteritems -> an iterator over the (key, value) pairs in od' - for k in self: - yield (k, self[k]) - - update = MutableMapping.update - - __update = update # let subclasses override update without breaking __init__ - - __marker = object() - - def pop(self, key, default=__marker): - '''od.pop(k[,d]) -> v, remove specified key and return the corresponding - value. If key is not found, d is returned if given, otherwise KeyError - is raised. - - ''' - if key in self: - result = self[key] - del self[key] - return result - if default is self.__marker: - raise KeyError(key) - return default - - def setdefault(self, key, default=None): - 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od' - if key in self: - return self[key] - self[key] = default - return default + return reversed_dict(self) def popitem(self, last=True): '''od.popitem() -> (k, v), return and remove a (key, value) pair. Pairs are returned in LIFO order if last is true or FIFO order if false. ''' - if not self: - raise KeyError('dictionary is empty') - key = next(reversed(self) if last else iter(self)) - value = self.pop(key) - return key, value + if last: + return dict.popitem(self) + else: + it = dict.__iter__(self) + try: + k = it.next() + except StopIteration: + raise KeyError('dictionary is empty') + return (k, self.pop(k)) def __repr__(self, _repr_running={}): 'od.__repr__() <==> repr(od)' @@ -183,8 +80,6 @@ 'Return state information for pickling' items = [[k, self[k]] for k in self] inst_dict = vars(self).copy() - for k in vars(OrderedDict()): - inst_dict.pop(k, None) if inst_dict: return (self.__class__, (items,), inst_dict) return self.__class__, (items,) @@ -193,17 +88,6 @@ 'od.copy() -> a shallow copy of od' return self.__class__(self) - @classmethod - def fromkeys(cls, iterable, value=None): - '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S. - If not specified, the value defaults to None. - - ''' - self = cls() - for key in iterable: - self[key] = value - return self - def __eq__(self, other): '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive while comparison to a regular mapping is order-insensitive. diff --git a/lib-python/2.7/cookielib.py b/lib-python/2.7/cookielib.py --- a/lib-python/2.7/cookielib.py +++ b/lib-python/2.7/cookielib.py @@ -1719,12 +1719,12 @@ def __repr__(self): r = [] for cookie in self: r.append(repr(cookie)) - return "<%s[%s]>" % (self.__class__, ", ".join(r)) + return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r)) def __str__(self): r = [] for cookie in self: r.append(str(cookie)) - return "<%s[%s]>" % (self.__class__, ", ".join(r)) + return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r)) # derives from IOError for backwards-compatibility with Python 2.4.0 diff --git a/lib-python/2.7/ctypes/test/test_frombuffer.py b/lib-python/2.7/ctypes/test/test_frombuffer.py --- a/lib-python/2.7/ctypes/test/test_frombuffer.py +++ b/lib-python/2.7/ctypes/test/test_frombuffer.py @@ -2,7 +2,6 @@ import array import gc import unittest -from ctypes.test import xfail class X(Structure): _fields_ = [("c_int", c_int)] @@ -11,7 +10,6 @@ self._init_called = True class Test(unittest.TestCase): - @xfail def test_fom_buffer(self): a = array.array("i", range(16)) x = (c_int * 16).from_buffer(a) @@ -34,10 +32,9 @@ del a; gc.collect(); gc.collect(); gc.collect() self.assertEqual(x[:], expected) - self.assertRaises(TypeError, + self.assertRaises((TypeError, ValueError), (c_char * 16).from_buffer, "a" * 16) - @xfail def test_fom_buffer_with_offset(self): a = array.array("i", range(16)) x = (c_int * 15).from_buffer(a, sizeof(c_int)) @@ -46,7 +43,6 @@ self.assertRaises(ValueError, lambda: (c_int * 16).from_buffer(a, sizeof(c_int))) self.assertRaises(ValueError, lambda: (c_int * 1).from_buffer(a, 16 * sizeof(c_int))) - @xfail def test_from_buffer_copy(self): a = array.array("i", range(16)) x = (c_int * 16).from_buffer_copy(a) @@ -71,7 +67,6 @@ x = (c_char * 16).from_buffer_copy("a" * 16) self.assertEqual(x[:], "a" * 16) - @xfail def test_fom_buffer_copy_with_offset(self): a = array.array("i", range(16)) x = (c_int * 15).from_buffer_copy(a, sizeof(c_int)) diff --git a/lib-python/2.7/ctypes/test/test_pointers.py b/lib-python/2.7/ctypes/test/test_pointers.py --- a/lib-python/2.7/ctypes/test/test_pointers.py +++ b/lib-python/2.7/ctypes/test/test_pointers.py @@ -7,6 +7,8 @@ c_long, c_ulong, c_longlong, c_ulonglong, c_double, c_float] python_types = [int, int, int, int, int, long, int, long, long, long, float, float] +LargeNamedType = type('T' * 2 ** 25, (Structure,), {}) +large_string = 'T' * 2 ** 25 class PointersTestCase(unittest.TestCase): @@ -188,5 +190,11 @@ mth = WINFUNCTYPE(None)(42, "name", (), None) self.assertEqual(bool(mth), True) + def test_pointer_type_name(self): + self.assertTrue(POINTER(LargeNamedType)) + + def test_pointer_type_str_name(self): + self.assertTrue(POINTER(large_string)) + if __name__ == '__main__': unittest.main() diff --git a/lib-python/2.7/ctypes/test/test_python_api.py b/lib-python/2.7/ctypes/test/test_python_api.py --- a/lib-python/2.7/ctypes/test/test_python_api.py +++ b/lib-python/2.7/ctypes/test/test_python_api.py @@ -46,8 +46,8 @@ # This test is unreliable, because it is possible that code in # unittest changes the refcount of the '42' integer. So, it # is disabled by default. - @requires("refcount") def test_PyInt_Long(self): + requires("refcount") ref42 = grc(42) pythonapi.PyInt_FromLong.restype = py_object self.assertEqual(pythonapi.PyInt_FromLong(42), 42) diff --git a/lib-python/2.7/ctypes/test/test_win32.py b/lib-python/2.7/ctypes/test/test_win32.py --- a/lib-python/2.7/ctypes/test/test_win32.py +++ b/lib-python/2.7/ctypes/test/test_win32.py @@ -38,8 +38,11 @@ @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') class FunctionCallTestCase(unittest.TestCase): - @requires("SEH") + @unittest.skipUnless('MSC' in sys.version, "SEH only supported by MSC") + @unittest.skipIf(sys.executable.endswith('_d.exe'), + "SEH not enabled in debug builds") def test_SEH(self): + requires("SEH") # Call functions with invalid arguments, and make sure # that access violations are trapped and raise an # exception. @@ -87,9 +90,29 @@ dll = CDLL(_ctypes_test.__file__) - pt = POINT(10, 10) - rect = RECT(0, 0, 20, 20) - self.assertEqual(1, dll.PointInRect(byref(rect), pt)) + pt = POINT(15, 25) + left = c_long.in_dll(dll, 'left') + top = c_long.in_dll(dll, 'top') + right = c_long.in_dll(dll, 'right') + bottom = c_long.in_dll(dll, 'bottom') + rect = RECT(left, top, right, bottom) + PointInRect = dll.PointInRect + PointInRect.argtypes = [POINTER(RECT), POINT] + self.assertEqual(1, PointInRect(byref(rect), pt)) + + ReturnRect = dll.ReturnRect + ReturnRect.argtypes = [c_int, RECT, POINTER(RECT), POINT, RECT, + POINTER(RECT), POINT, RECT] + ReturnRect.restype = RECT + for i in range(4): + ret = ReturnRect(i, rect, pointer(rect), pt, rect, + byref(rect), pt, rect) + # the c function will check and modify ret if something is + # passed in improperly + self.assertEqual(ret.left, left.value) + self.assertEqual(ret.right, right.value) + self.assertEqual(ret.top, top.value) + self.assertEqual(ret.bottom, bottom.value) if __name__ == '__main__': unittest.main() diff --git a/lib-python/2.7/decimal.py b/lib-python/2.7/decimal.py --- a/lib-python/2.7/decimal.py +++ b/lib-python/2.7/decimal.py @@ -136,7 +136,6 @@ __version__ = '1.70' # Highest version of the spec this complies with -import copy as _copy import math as _math import numbers as _numbers @@ -3665,6 +3664,8 @@ if self._is_special: sign = _format_sign(self._sign, spec) body = str(self.copy_abs()) + if spec['type'] == '%': + body += '%' return _format_align(sign, body, spec) # a type of None defaults to 'g' or 'G', depending on context @@ -6033,7 +6034,10 @@ format_dict['decimal_point'] = '.' # record whether return type should be str or unicode - format_dict['unicode'] = isinstance(format_spec, unicode) + try: + format_dict['unicode'] = isinstance(format_spec, unicode) + except NameError: + format_dict['unicode'] = False return format_dict diff --git a/lib-python/2.7/distutils/__init__.py b/lib-python/2.7/distutils/__init__.py --- a/lib-python/2.7/distutils/__init__.py +++ b/lib-python/2.7/distutils/__init__.py @@ -15,5 +15,5 @@ # Updated automatically by the Python release process. # #--start constants-- -__version__ = "2.7.8" +__version__ = "2.7.9" #--end constants-- diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py --- a/lib-python/2.7/distutils/command/build_ext.py +++ b/lib-python/2.7/distutils/command/build_ext.py @@ -245,7 +245,7 @@ # Python's library directory must be appended to library_dirs # See Issues: #1600860, #4366 if (sysconfig.get_config_var('Py_ENABLE_SHARED')): - if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")): + if not sysconfig.python_build: # building third party extensions self.library_dirs.append(sysconfig.get_config_var('LIBDIR')) else: diff --git a/lib-python/2.7/distutils/command/upload.py b/lib-python/2.7/distutils/command/upload.py --- a/lib-python/2.7/distutils/command/upload.py +++ b/lib-python/2.7/distutils/command/upload.py @@ -136,8 +136,8 @@ # Build up the MIME payload for the POST data boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254' - sep_boundary = '\n--' + boundary - end_boundary = sep_boundary + '--' + sep_boundary = '\r\n--' + boundary + end_boundary = sep_boundary + '--\r\n' body = StringIO.StringIO() for key, value in data.items(): # handle multiple entries for the same name @@ -151,14 +151,13 @@ fn = "" body.write(sep_boundary) - body.write('\nContent-Disposition: form-data; name="%s"'%key) + body.write('\r\nContent-Disposition: form-data; name="%s"' % key) body.write(fn) - body.write("\n\n") + body.write("\r\n\r\n") body.write(value) if value and value[-1] == '\r': body.write('\n') # write an extra newline (lurve Macs) body.write(end_boundary) - body.write("\n") body = body.getvalue() self.announce("Submitting %s to %s" % (filename, self.repository), log.INFO) diff --git a/lib-python/2.7/distutils/file_util.py b/lib-python/2.7/distutils/file_util.py --- a/lib-python/2.7/distutils/file_util.py +++ b/lib-python/2.7/distutils/file_util.py @@ -85,7 +85,8 @@ (os.symlink) instead of copying: set it to "hard" or "sym"; if it is None (the default), files are copied. Don't set 'link' on systems that don't support it: 'copy_file()' doesn't check if hard or symbolic - linking is available. + linking is available. If hardlink fails, falls back to + _copy_file_contents(). Under Mac OS, uses the native file copy function in macostools; on other systems, uses '_copy_file_contents()' to copy file contents. @@ -137,24 +138,31 @@ # (Unix only, of course, but that's the caller's responsibility) if link == 'hard': if not (os.path.exists(dst) and os.path.samefile(src, dst)): - os.link(src, dst) + try: + os.link(src, dst) + return (dst, 1) + except OSError: + # If hard linking fails, fall back on copying file + # (some special filesystems don't support hard linking + # even under Unix, see issue #8876). + pass elif link == 'sym': if not (os.path.exists(dst) and os.path.samefile(src, dst)): os.symlink(src, dst) + return (dst, 1) # Otherwise (non-Mac, not linking), copy the file contents and # (optionally) copy the times and mode. - else: - _copy_file_contents(src, dst) - if preserve_mode or preserve_times: - st = os.stat(src) + _copy_file_contents(src, dst) + if preserve_mode or preserve_times: + st = os.stat(src) - # According to David Ascher , utime() should be done - # before chmod() (at least under NT). - if preserve_times: - os.utime(dst, (st[ST_ATIME], st[ST_MTIME])) - if preserve_mode: - os.chmod(dst, S_IMODE(st[ST_MODE])) + # According to David Ascher , utime() should be done + # before chmod() (at least under NT). + if preserve_times: + os.utime(dst, (st[ST_ATIME], st[ST_MTIME])) + if preserve_mode: + os.chmod(dst, S_IMODE(st[ST_MODE])) return (dst, 1) diff --git a/lib-python/2.7/distutils/sysconfig_cpython.py b/lib-python/2.7/distutils/sysconfig_cpython.py --- a/lib-python/2.7/distutils/sysconfig_cpython.py +++ b/lib-python/2.7/distutils/sysconfig_cpython.py @@ -165,7 +165,8 @@ # version and build tools may not support the same set # of CPU architectures for universal builds. global _config_vars - if not _config_vars.get('CUSTOMIZED_OSX_COMPILER', ''): + # Use get_config_var() to ensure _config_vars is initialized. + if not get_config_var('CUSTOMIZED_OSX_COMPILER'): import _osx_support _osx_support.customize_compiler(_config_vars) _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True' diff --git a/lib-python/2.7/distutils/tests/test_bdist_rpm.py b/lib-python/2.7/distutils/tests/test_bdist_rpm.py --- a/lib-python/2.7/distutils/tests/test_bdist_rpm.py +++ b/lib-python/2.7/distutils/tests/test_bdist_rpm.py @@ -25,6 +25,7 @@ """ class BuildRpmTestCase(support.TempdirManager, + support.EnvironGuard, support.LoggingSilencer, unittest.TestCase): @@ -50,6 +51,7 @@ def test_quiet(self): # let's create a package tmp_dir = self.mkdtemp() + os.environ['HOME'] = tmp_dir # to confine dir '.rpmdb' creation pkg_dir = os.path.join(tmp_dir, 'foo') os.mkdir(pkg_dir) self.write_file((pkg_dir, 'setup.py'), SETUP_PY) @@ -92,6 +94,7 @@ def test_no_optimize_flag(self): # let's create a package that brakes bdist_rpm tmp_dir = self.mkdtemp() + os.environ['HOME'] = tmp_dir # to confine dir '.rpmdb' creation pkg_dir = os.path.join(tmp_dir, 'foo') os.mkdir(pkg_dir) self.write_file((pkg_dir, 'setup.py'), SETUP_PY) diff --git a/lib-python/2.7/distutils/tests/test_dist.py b/lib-python/2.7/distutils/tests/test_dist.py --- a/lib-python/2.7/distutils/tests/test_dist.py +++ b/lib-python/2.7/distutils/tests/test_dist.py @@ -11,7 +11,7 @@ from distutils.dist import Distribution, fix_help_options from distutils.cmd import Command import distutils.dist -from test.test_support import TESTFN, captured_stdout, run_unittest +from test.test_support import TESTFN, captured_stdout, run_unittest, unlink from distutils.tests import support @@ -64,6 +64,7 @@ with open(TESTFN, "w") as f: f.write("[global]\n") f.write("command_packages = foo.bar, splat") + self.addCleanup(unlink, TESTFN) files = [TESTFN] sys.argv.append("build") diff --git a/lib-python/2.7/distutils/tests/test_file_util.py b/lib-python/2.7/distutils/tests/test_file_util.py --- a/lib-python/2.7/distutils/tests/test_file_util.py +++ b/lib-python/2.7/distutils/tests/test_file_util.py @@ -8,6 +8,11 @@ from distutils.tests import support from test.test_support import run_unittest + +requires_os_link = unittest.skipUnless(hasattr(os, "link"), + "test requires os.link()") + + class FileUtilTestCase(support.TempdirManager, unittest.TestCase): def _log(self, msg, *args): @@ -74,6 +79,44 @@ copy_file(foo, dst_dir) self.assertTrue(os.path.exists(os.path.join(dst_dir, 'foo'))) + @requires_os_link + def test_copy_file_hard_link(self): + with open(self.source, 'w') as f: + f.write('some content') + st = os.stat(self.source) + copy_file(self.source, self.target, link='hard') + st2 = os.stat(self.source) + st3 = os.stat(self.target) + self.assertTrue(os.path.samestat(st, st2), (st, st2)) + self.assertTrue(os.path.samestat(st2, st3), (st2, st3)) + with open(self.source, 'r') as f: + self.assertEqual(f.read(), 'some content') + + @requires_os_link + def test_copy_file_hard_link_failure(self): + # If hard linking fails, copy_file() falls back on copying file + # (some special filesystems don't support hard linking even under + # Unix, see issue #8876). + with open(self.source, 'w') as f: + f.write('some content') + st = os.stat(self.source) + def _os_link(*args): + raise OSError(0, "linking unsupported") + old_link = os.link + os.link = _os_link + try: + copy_file(self.source, self.target, link='hard') + finally: + os.link = old_link + st2 = os.stat(self.source) + st3 = os.stat(self.target) + self.assertTrue(os.path.samestat(st, st2), (st, st2)) + self.assertFalse(os.path.samestat(st2, st3), (st2, st3)) + for fn in (self.source, self.target): + with open(fn, 'r') as f: + self.assertEqual(f.read(), 'some content') + + def test_suite(): return unittest.makeSuite(FileUtilTestCase) diff --git a/lib-python/2.7/distutils/tests/test_sysconfig.py b/lib-python/2.7/distutils/tests/test_sysconfig.py --- a/lib-python/2.7/distutils/tests/test_sysconfig.py +++ b/lib-python/2.7/distutils/tests/test_sysconfig.py @@ -3,6 +3,9 @@ import test import unittest import shutil +import subprocess +import sys +import textwrap from distutils import sysconfig from distutils.tests import support @@ -99,6 +102,24 @@ self.assertEqual(global_sysconfig.get_config_var('LDSHARED'), sysconfig.get_config_var('LDSHARED')) self.assertEqual(global_sysconfig.get_config_var('CC'), sysconfig.get_config_var('CC')) + def test_customize_compiler_before_get_config_vars(self): + # Issue #21923: test that a Distribution compiler + # instance can be called without an explicit call to + # get_config_vars(). + with open(TESTFN, 'w') as f: + f.writelines(textwrap.dedent('''\ + from distutils.core import Distribution + config = Distribution().get_command_obj('config') + # try_compile may pass or it may fail if no compiler + # is found but it should not raise an exception. + rc = config.try_compile('int x;') + ''')) + p = subprocess.Popen([str(sys.executable), TESTFN], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True) + outs, errs = p.communicate() + self.assertEqual(0, p.returncode, "Subprocess failed: " + outs) def test_suite(): diff --git a/lib-python/2.7/distutils/tests/test_upload.py b/lib-python/2.7/distutils/tests/test_upload.py --- a/lib-python/2.7/distutils/tests/test_upload.py +++ b/lib-python/2.7/distutils/tests/test_upload.py @@ -119,7 +119,7 @@ # what did we send ? self.assertIn('dédé', self.last_open.req.data) headers = dict(self.last_open.req.headers) - self.assertEqual(headers['Content-length'], '2085') + self.assertEqual(headers['Content-length'], '2159') self.assertTrue(headers['Content-type'].startswith('multipart/form-data')) self.assertEqual(self.last_open.req.get_method(), 'POST') self.assertEqual(self.last_open.req.get_full_url(), diff --git a/lib-python/2.7/distutils/unixccompiler.py b/lib-python/2.7/distutils/unixccompiler.py --- a/lib-python/2.7/distutils/unixccompiler.py +++ b/lib-python/2.7/distutils/unixccompiler.py @@ -58,7 +58,7 @@ executables = {'preprocessor' : None, 'compiler' : ["cc"], 'compiler_so' : ["cc"], - 'compiler_cxx' : ["cc"], + 'compiler_cxx' : ["c++"], # pypy: changed, 'cc' is bogus 'linker_so' : ["cc", "-shared"], 'linker_exe' : ["cc"], 'archiver' : ["ar", "-cr"], diff --git a/lib-python/2.7/doctest.py b/lib-python/2.7/doctest.py --- a/lib-python/2.7/doctest.py +++ b/lib-python/2.7/doctest.py @@ -216,7 +216,7 @@ # get_data() opens files as 'rb', so one must do the equivalent # conversion as universal newlines would do. return file_contents.replace(os.linesep, '\n'), filename - with open(filename) as f: + with open(filename, 'U') as f: return f.read(), filename # Use sys.stdout encoding for ouput. diff --git a/lib-python/2.7/email/feedparser.py b/lib-python/2.7/email/feedparser.py --- a/lib-python/2.7/email/feedparser.py +++ b/lib-python/2.7/email/feedparser.py @@ -49,8 +49,8 @@ simple abstraction -- it parses until EOF closes the current message. """ def __init__(self): - # The last partial line pushed into this object. - self._partial = '' + # Chunks of the last partial line pushed into this object. + self._partial = [] # The list of full, pushed lines, in reverse order self._lines = [] # The stack of false-EOF checking predicates. @@ -66,8 +66,8 @@ def close(self): # Don't forget any trailing partial line. - self._lines.append(self._partial) - self._partial = '' + self.pushlines(''.join(self._partial).splitlines(True)) + self._partial = [] self._closed = True def readline(self): @@ -95,8 +95,29 @@ def push(self, data): """Push some new data into this object.""" - # Handle any previous leftovers - data, self._partial = self._partial + data, '' + # Crack into lines, but preserve the linesep characters on the end of each + parts = data.splitlines(True) + + if not parts or not parts[0].endswith(('\n', '\r')): + # No new complete lines, so just accumulate partials + self._partial += parts + return + + if self._partial: + # If there are previous leftovers, complete them now + self._partial.append(parts[0]) + parts[0:1] = ''.join(self._partial).splitlines(True) + del self._partial[:] + + # If the last element of the list does not end in a newline, then treat + # it as a partial line. We only check for '\n' here because a line + # ending with '\r' might be a line that was split in the middle of a + # '\r\n' sequence (see bugs 1555570 and 1721862). + if not parts[-1].endswith('\n'): + self._partial = [parts.pop()] + self.pushlines(parts) + + def pushlines(self, lines): # Crack into lines, but preserve the newlines on the end of each parts = NLCRE_crack.split(data) # The *ahem* interesting behaviour of re.split when supplied grouping diff --git a/lib-python/2.7/email/mime/nonmultipart.py b/lib-python/2.7/email/mime/nonmultipart.py --- a/lib-python/2.7/email/mime/nonmultipart.py +++ b/lib-python/2.7/email/mime/nonmultipart.py @@ -12,7 +12,7 @@ class MIMENonMultipart(MIMEBase): - """Base class for MIME multipart/* type messages.""" + """Base class for MIME non-multipart type messages.""" def attach(self, payload): # The public API prohibits attaching multiple subparts to MIMEBase diff --git a/lib-python/2.7/email/test/test_email.py b/lib-python/2.7/email/test/test_email.py --- a/lib-python/2.7/email/test/test_email.py +++ b/lib-python/2.7/email/test/test_email.py @@ -11,6 +11,7 @@ import warnings import textwrap from cStringIO import StringIO +from random import choice import email @@ -2578,16 +2579,64 @@ bsf.push(il) nt += n n1 = 0 - while True: - ol = bsf.readline() - if ol == NeedMoreData: - break + for ol in iter(bsf.readline, NeedMoreData): om.append(ol) n1 += 1 self.assertEqual(n, n1) self.assertEqual(len(om), nt) self.assertEqual(''.join([il for il, n in imt]), ''.join(om)) + def test_push_random(self): + from email.feedparser import BufferedSubFile, NeedMoreData + + n = 10000 + chunksize = 5 + chars = 'abcd \t\r\n' + + s = ''.join(choice(chars) for i in range(n)) + '\n' + target = s.splitlines(True) + + bsf = BufferedSubFile() + lines = [] + for i in range(0, len(s), chunksize): + chunk = s[i:i+chunksize] + bsf.push(chunk) + lines.extend(iter(bsf.readline, NeedMoreData)) + self.assertEqual(lines, target) + + +class TestFeedParsers(TestEmailBase): + + def parse(self, chunks): + from email.feedparser import FeedParser + feedparser = FeedParser() + for chunk in chunks: + feedparser.feed(chunk) + return feedparser.close() + + def test_newlines(self): + m = self.parse(['a:\nb:\rc:\r\nd:\n']) + self.assertEqual(m.keys(), ['a', 'b', 'c', 'd']) + m = self.parse(['a:\nb:\rc:\r\nd:']) + self.assertEqual(m.keys(), ['a', 'b', 'c', 'd']) + m = self.parse(['a:\rb', 'c:\n']) + self.assertEqual(m.keys(), ['a', 'bc']) + m = self.parse(['a:\r', 'b:\n']) + self.assertEqual(m.keys(), ['a', 'b']) + m = self.parse(['a:\r', '\nb:\n']) + self.assertEqual(m.keys(), ['a', 'b']) + + def test_long_lines(self): + # Expected peak memory use on 32-bit platform: 4*N*M bytes. + M, N = 1000, 20000 + m = self.parse(['a:b\n\n'] + ['x'*M] * N) + self.assertEqual(m.items(), [('a', 'b')]) + self.assertEqual(m.get_payload(), 'x'*M*N) + m = self.parse(['a:b\r\r'] + ['x'*M] * N) + self.assertEqual(m.items(), [('a', 'b')]) + self.assertEqual(m.get_payload(), 'x'*M*N) + m = self.parse(['a:\r', 'b: '] + ['x'*M] * N) + self.assertEqual(m.items(), [('a', ''), ('b', 'x'*M*N)]) class TestParsers(TestEmailBase): @@ -3180,7 +3229,6 @@ self.assertEqual(res, '=?iso-8859-2?q?abc?=') self.assertIsInstance(res, str) - # Test RFC 2231 header parameters (en/de)coding class TestRFC2231(TestEmailBase): def test_get_param(self): diff --git a/lib-python/2.7/ensurepip/__init__.py b/lib-python/2.7/ensurepip/__init__.py new file mode 100644 --- /dev/null +++ b/lib-python/2.7/ensurepip/__init__.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python2 +from __future__ import print_function + +import os +import os.path +import pkgutil +import shutil +import sys +import tempfile + + +__all__ = ["version", "bootstrap"] + + +_SETUPTOOLS_VERSION = "7.0" + +_PIP_VERSION = "1.5.6" + +# pip currently requires ssl support, so we try to provide a nicer +# error message when that is missing (http://bugs.python.org/issue19744) +_MISSING_SSL_MESSAGE = ("pip {} requires SSL/TLS".format(_PIP_VERSION)) +try: + import ssl +except ImportError: + ssl = None + + def _require_ssl_for_pip(): + raise RuntimeError(_MISSING_SSL_MESSAGE) +else: + def _require_ssl_for_pip(): + pass + +_PROJECTS = [ + ("setuptools", _SETUPTOOLS_VERSION), + ("pip", _PIP_VERSION), +] + + +def _run_pip(args, additional_paths=None): + # Add our bundled software to the sys.path so we can import it + if additional_paths is not None: + sys.path = additional_paths + sys.path + + # Install the bundled software + import pip + pip.main(args) + + +def version(): + """ + Returns a string specifying the bundled version of pip. + """ + return _PIP_VERSION + + +def _disable_pip_configuration_settings(): + # We deliberately ignore all pip environment variables + # when invoking pip + # See http://bugs.python.org/issue19734 for details + keys_to_remove = [k for k in os.environ if k.startswith("PIP_")] + for k in keys_to_remove: + del os.environ[k] + # We also ignore the settings in the default pip configuration file + # See http://bugs.python.org/issue20053 for details + os.environ['PIP_CONFIG_FILE'] = os.devnull + + +def bootstrap(root=None, upgrade=False, user=False, + altinstall=False, default_pip=True, + verbosity=0): + """ + Bootstrap pip into the current Python installation (or the given root + directory). + + Note that calling this function will alter both sys.path and os.environ. + """ + if altinstall and default_pip: + raise ValueError("Cannot use altinstall and default_pip together") + + _require_ssl_for_pip() + _disable_pip_configuration_settings() + + # By default, installing pip and setuptools installs all of the + # following scripts (X.Y == running Python version): + # + # pip, pipX, pipX.Y, easy_install, easy_install-X.Y + # + # pip 1.5+ allows ensurepip to request that some of those be left out + if altinstall: + # omit pip, pipX and easy_install + os.environ["ENSUREPIP_OPTIONS"] = "altinstall" + elif not default_pip: + # omit pip and easy_install + os.environ["ENSUREPIP_OPTIONS"] = "install" + + tmpdir = tempfile.mkdtemp() + try: + # Put our bundled wheels into a temporary directory and construct the + # additional paths that need added to sys.path + additional_paths = [] + for project, version in _PROJECTS: + wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version) + whl = pkgutil.get_data( + "ensurepip", + "_bundled/{}".format(wheel_name), + ) + with open(os.path.join(tmpdir, wheel_name), "wb") as fp: + fp.write(whl) + + additional_paths.append(os.path.join(tmpdir, wheel_name)) + + # Construct the arguments to be passed to the pip command + args = ["install", "--no-index", "--find-links", tmpdir] + if root: + args += ["--root", root] + if upgrade: + args += ["--upgrade"] + if user: + args += ["--user"] + if verbosity: + args += ["-" + "v" * verbosity] + + _run_pip(args + [p[0] for p in _PROJECTS], additional_paths) + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + +def _uninstall_helper(verbosity=0): + """Helper to support a clean default uninstall process on Windows + + Note that calling this function may alter os.environ. + """ + # Nothing to do if pip was never installed, or has been removed + try: + import pip + except ImportError: + return + + # If the pip version doesn't match the bundled one, leave it alone + if pip.__version__ != _PIP_VERSION: + msg = ("ensurepip will only uninstall a matching version " + "({!r} installed, {!r} bundled)") + print(msg.format(pip.__version__, _PIP_VERSION), file=sys.stderr) + return + + _require_ssl_for_pip() + _disable_pip_configuration_settings() + + # Construct the arguments to be passed to the pip command + args = ["uninstall", "-y"] + if verbosity: + args += ["-" + "v" * verbosity] + + _run_pip(args + [p[0] for p in reversed(_PROJECTS)]) + + +def _main(argv=None): + if ssl is None: + print("Ignoring ensurepip failure: {}".format(_MISSING_SSL_MESSAGE), + file=sys.stderr) + return + + import argparse + parser = argparse.ArgumentParser(prog="python -m ensurepip") + parser.add_argument( + "--version", + action="version", + version="pip {}".format(version()), + help="Show the version of pip that is bundled with this Python.", + ) + parser.add_argument( + "-v", "--verbose", + action="count", + default=0, + dest="verbosity", + help=("Give more output. Option is additive, and can be used up to 3 " + "times."), + ) + parser.add_argument( + "-U", "--upgrade", + action="store_true", + default=False, + help="Upgrade pip and dependencies, even if already installed.", + ) + parser.add_argument( + "--user", + action="store_true", + default=False, + help="Install using the user scheme.", + ) + parser.add_argument( + "--root", + default=None, + help="Install everything relative to this alternate root directory.", + ) + parser.add_argument( + "--altinstall", + action="store_true", + default=False, + help=("Make an alternate install, installing only the X.Y versioned" + "scripts (Default: pipX, pipX.Y, easy_install-X.Y)"), + ) + parser.add_argument( + "--default-pip", + action="store_true", + default=True, + dest="default_pip", + help=argparse.SUPPRESS, + ) + parser.add_argument( + "--no-default-pip", + action="store_false", + dest="default_pip", + help=("Make a non default install, installing only the X and X.Y " + "versioned scripts."), + ) + + args = parser.parse_args(argv) + + bootstrap( + root=args.root, + upgrade=args.upgrade, + user=args.user, + verbosity=args.verbosity, + altinstall=args.altinstall, + default_pip=args.default_pip, + ) diff --git a/lib-python/2.7/ensurepip/__main__.py b/lib-python/2.7/ensurepip/__main__.py new file mode 100644 --- /dev/null +++ b/lib-python/2.7/ensurepip/__main__.py @@ -0,0 +1,4 @@ +import ensurepip + +if __name__ == "__main__": + ensurepip._main() diff --git a/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..097ab43430d4c1302b0be353a8c16407c370693b GIT binary patch [cut] diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..fa1d1054da1dab98f8906555d31a9fda271b3a85 GIT binary patch [cut] diff --git a/lib-python/2.7/ensurepip/_uninstall.py b/lib-python/2.7/ensurepip/_uninstall.py new file mode 100644 --- /dev/null +++ b/lib-python/2.7/ensurepip/_uninstall.py @@ -0,0 +1,30 @@ +"""Basic pip uninstallation support, helper for the Windows uninstaller""" + +import argparse +import ensurepip + + +def _main(argv=None): + parser = argparse.ArgumentParser(prog="python -m ensurepip._uninstall") + parser.add_argument( + "--version", + action="version", + version="pip {}".format(ensurepip.version()), + help="Show the version of pip this will attempt to uninstall.", + ) + parser.add_argument( + "-v", "--verbose", + action="count", + default=0, + dest="verbosity", + help=("Give more output. Option is additive, and can be used up to 3 " + "times."), + ) + + args = parser.parse_args(argv) + + ensurepip._uninstall_helper(verbosity=args.verbosity) + + +if __name__ == "__main__": + _main() diff --git a/lib-python/2.7/glob.py b/lib-python/2.7/glob.py --- a/lib-python/2.7/glob.py +++ b/lib-python/2.7/glob.py @@ -35,11 +35,16 @@ patterns. """ + dirname, basename = os.path.split(pathname) if not has_magic(pathname): - if os.path.lexists(pathname): - yield pathname + if basename: + if os.path.lexists(pathname): + yield pathname + else: + # Patterns ending with a slash should match only directories + if os.path.isdir(dirname): + yield pathname return - dirname, basename = os.path.split(pathname) if not dirname: for name in glob1(os.curdir, basename): yield name diff --git a/lib-python/2.7/gzip.py b/lib-python/2.7/gzip.py --- a/lib-python/2.7/gzip.py +++ b/lib-python/2.7/gzip.py @@ -164,9 +164,16 @@ def _write_gzip_header(self): self.fileobj.write('\037\213') # magic header self.fileobj.write('\010') # compression method - fname = os.path.basename(self.name) - if fname.endswith(".gz"): - fname = fname[:-3] + try: + # RFC 1952 requires the FNAME field to be Latin-1. Do not + # include filenames that cannot be represented that way. + fname = os.path.basename(self.name) + if not isinstance(fname, str): + fname = fname.encode('latin-1') + if fname.endswith('.gz'): + fname = fname[:-3] + except UnicodeEncodeError: + fname = '' flags = 0 if fname: flags = FNAME diff --git a/lib-python/2.7/hashlib.py b/lib-python/2.7/hashlib.py --- a/lib-python/2.7/hashlib.py +++ b/lib-python/2.7/hashlib.py @@ -15,8 +15,9 @@ md5(), sha1(), sha224(), sha256(), sha384(), and sha512() -More algorithms may be available on your platform but the above are -guaranteed to exist. +More algorithms may be available on your platform but the above are guaranteed +to exist. See the algorithms_guaranteed and algorithms_available attributes +to find out what algorithm names can be passed to new(). NOTE: If you want the adler32 or crc32 hash functions they are available in the zlib module. @@ -58,9 +59,14 @@ # always available algorithm is added. __always_supported = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512') +algorithms_guaranteed = set(__always_supported) +algorithms_available = set(__always_supported) + algorithms = __always_supported -__all__ = __always_supported + ('new', 'algorithms', 'pbkdf2_hmac') +__all__ = __always_supported + ('new', 'algorithms_guaranteed', + 'algorithms_available', 'algorithms', + 'pbkdf2_hmac') def __get_builtin_constructor(name): @@ -128,6 +134,8 @@ import _hashlib new = __hash_new __get_hash = __get_openssl_constructor + algorithms_available = algorithms_available.union( + _hashlib.openssl_md_meth_names) except ImportError: new = __py_new __get_hash = __get_builtin_constructor diff --git a/lib-python/2.7/httplib.py b/lib-python/2.7/httplib.py --- a/lib-python/2.7/httplib.py +++ b/lib-python/2.7/httplib.py @@ -215,6 +215,10 @@ # maximal line length when calling readline(). _MAXLINE = 65536 +# maximum amount of headers accepted +_MAXHEADERS = 100 + + class HTTPMessage(mimetools.Message): def addheader(self, key, value): @@ -271,6 +275,8 @@ elif self.seekable: tell = self.fp.tell while True: + if len(hlist) > _MAXHEADERS: + raise HTTPException("got more than %d headers" % _MAXHEADERS) if tell: try: startofline = tell() @@ -1185,21 +1191,29 @@ def __init__(self, host, port=None, key_file=None, cert_file=None, strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - source_address=None): + source_address=None, context=None): HTTPConnection.__init__(self, host, port, strict, timeout, source_address) self.key_file = key_file self.cert_file = cert_file + if context is None: + context = ssl._create_default_https_context() + if key_file or cert_file: + context.load_cert_chain(cert_file, key_file) + self._context = context def connect(self): "Connect to a host on a given (SSL) port." - sock = self._create_connection((self.host, self.port), - self.timeout, self.source_address) + HTTPConnection.connect(self) + if self._tunnel_host: - self.sock = sock - self._tunnel() - self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file) + server_hostname = self._tunnel_host + else: + server_hostname = self.host + + self.sock = self._context.wrap_socket(self.sock, + server_hostname=server_hostname) __all__.append("HTTPSConnection") @@ -1214,14 +1228,15 @@ _connection_class = HTTPSConnection def __init__(self, host='', port=None, key_file=None, cert_file=None, - strict=None): + strict=None, context=None): # provide a default host, pass the X509 cert info # urf. compensate for bad input. if port == 0: port = None self._setup(self._connection_class(host, port, key_file, - cert_file, strict)) + cert_file, strict, + context=context)) # we never actually use these for anything, but we keep them # here for compatibility with post-1.5.2 CVS. diff --git a/lib-python/2.7/idlelib/Bindings.py b/lib-python/2.7/idlelib/Bindings.py --- a/lib-python/2.7/idlelib/Bindings.py +++ b/lib-python/2.7/idlelib/Bindings.py @@ -75,7 +75,8 @@ ('!_Auto-open Stack Viewer', '<>'), ]), ('options', [ - ('_Configure IDLE...', '<>'), + ('Configure _IDLE', '<>'), + ('Configure _Extensions', '<>'), None, ]), ('help', [ diff --git a/lib-python/2.7/idlelib/CallTipWindow.py b/lib-python/2.7/idlelib/CallTipWindow.py --- a/lib-python/2.7/idlelib/CallTipWindow.py +++ b/lib-python/2.7/idlelib/CallTipWindow.py @@ -2,9 +2,8 @@ After ToolTip.py, which uses ideas gleaned from PySol Used by the CallTips IDLE extension. - """ -from Tkinter import * +from Tkinter import Toplevel, Label, LEFT, SOLID, TclError HIDE_VIRTUAL_EVENT_NAME = "<>" HIDE_SEQUENCES = ("", "") @@ -133,35 +132,28 @@ return bool(self.tipwindow) -def _calltip_window(parent): - root = Tk() - root.title("Test calltips") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) +def _calltip_window(parent): # htest # + from Tkinter import Toplevel, Text, LEFT, BOTH - class MyEditWin: # comparenceptually an editor_window - def __init__(self): - text = self.text = Text(root) - text.pack(side=LEFT, fill=BOTH, expand=1) - text.insert("insert", "string.split") - root.update() - self.calltip = CallTip(text) + top = Toplevel(parent) + top.title("Test calltips") + top.geometry("200x100+%d+%d" % (parent.winfo_rootx() + 200, + parent.winfo_rooty() + 150)) + text = Text(top) + text.pack(side=LEFT, fill=BOTH, expand=1) + text.insert("insert", "string.split") + top.update() + calltip = CallTip(text) - text.event_add("<>", "(") - text.event_add("<>", ")") - text.bind("<>", self.calltip_show) - text.bind("<>", self.calltip_hide) - - text.focus_set() - root.mainloop() - - def calltip_show(self, event): - self.calltip.showtip("Hello world", "insert", "end") - - def calltip_hide(self, event): - self.calltip.hidetip() - - editwin = MyEditWin() + def calltip_show(event): + calltip.showtip("(s=Hello world)", "insert", "end") + def calltip_hide(event): + calltip.hidetip() + text.event_add("<>", "(") + text.event_add("<>", ")") + text.bind("<>", calltip_show) + text.bind("<>", calltip_hide) + text.focus_set() if __name__=='__main__': from idlelib.idle_test.htest import run diff --git a/lib-python/2.7/idlelib/ClassBrowser.py b/lib-python/2.7/idlelib/ClassBrowser.py --- a/lib-python/2.7/idlelib/ClassBrowser.py +++ b/lib-python/2.7/idlelib/ClassBrowser.py @@ -19,6 +19,9 @@ from idlelib.TreeWidget import TreeNode, TreeItem, ScrolledCanvas from idlelib.configHandler import idleConf +file_open = None # Method...Item and Class...Item use this. +# Normally PyShell.flist.open, but there is no PyShell.flist for htest. + class ClassBrowser: def __init__(self, flist, name, path, _htest=False): @@ -27,6 +30,9 @@ """ _htest - bool, change box when location running htest. """ + global file_open + if not _htest: + file_open = PyShell.flist.open self.name = name self.file = os.path.join(path[0], self.name + ".py") self._htest = _htest @@ -101,7 +107,7 @@ return [] try: dict = pyclbr.readmodule_ex(name, [dir] + sys.path) - except ImportError, msg: + except ImportError: return [] items = [] self.classes = {} @@ -170,7 +176,7 @@ def OnDoubleClick(self): if not os.path.exists(self.file): return - edit = PyShell.flist.open(self.file) + edit = file_open(self.file) if hasattr(self.cl, 'lineno'): lineno = self.cl.lineno edit.gotoline(lineno) @@ -206,7 +212,7 @@ def OnDoubleClick(self): if not os.path.exists(self.file): return - edit = PyShell.flist.open(self.file) + edit = file_open(self.file) edit.gotoline(self.cl.methods[self.name]) def _class_browser(parent): #Wrapper for htest @@ -221,8 +227,9 @@ dir, file = os.path.split(file) name = os.path.splitext(file)[0] flist = PyShell.PyShellFileList(parent) + global file_open + file_open = flist.open ClassBrowser(flist, name, [dir], _htest=True) - parent.mainloop() if __name__ == "__main__": from idlelib.idle_test.htest import run diff --git a/lib-python/2.7/idlelib/ColorDelegator.py b/lib-python/2.7/idlelib/ColorDelegator.py --- a/lib-python/2.7/idlelib/ColorDelegator.py +++ b/lib-python/2.7/idlelib/ColorDelegator.py @@ -2,7 +2,6 @@ import re import keyword import __builtin__ -from Tkinter import * from idlelib.Delegator import Delegator from idlelib.configHandler import idleConf @@ -34,7 +33,6 @@ prog = re.compile(make_pat(), re.S) idprog = re.compile(r"\s+(\w+)", re.S) -asprog = re.compile(r".*?\b(as)\b") class ColorDelegator(Delegator): @@ -42,7 +40,6 @@ Delegator.__init__(self) self.prog = prog self.idprog = idprog - self.asprog = asprog self.LoadTagDefs() def setdelegate(self, delegate): @@ -74,7 +71,6 @@ "DEFINITION": idleConf.GetHighlight(theme, "definition"), "SYNC": {'background':None,'foreground':None}, "TODO": {'background':None,'foreground':None}, - "BREAK": idleConf.GetHighlight(theme, "break"), "ERROR": idleConf.GetHighlight(theme, "error"), # The following is used by ReplaceDialog: "hit": idleConf.GetHighlight(theme, "hit"), @@ -216,22 +212,6 @@ self.tag_add("DEFINITION", head + "+%dc" % a, head + "+%dc" % b) - elif value == "import": - # color all the "as" words on same line, except - # if in a comment; cheap approximation to the - # truth - if '#' in chars: - endpos = chars.index('#') - else: - endpos = len(chars) - while True: - m1 = self.asprog.match(chars, b, endpos) - if not m1: - break - a, b = m1.span(1) - self.tag_add("KEYWORD", - head + "+%dc" % a, - head + "+%dc" % b) m = self.prog.search(chars, m.end()) if "SYNC" in self.tag_names(next + "-1c"): head = next @@ -255,20 +235,23 @@ for tag in self.tagdefs.keys(): self.tag_remove(tag, "1.0", "end") -def _color_delegator(parent): +def _color_delegator(parent): # htest # + from Tkinter import Toplevel, Text from idlelib.Percolator import Percolator - root = Tk() - root.title("Test ColorDelegator") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) - source = "if somename: x = 'abc' # comment\nprint" - text = Text(root, background="white") + + top = Toplevel(parent) + top.title("Test ColorDelegator") + top.geometry("200x100+%d+%d" % (parent.winfo_rootx() + 200, From noreply at buildbot.pypy.org Mon Mar 23 14:29:19 2015 From: noreply at buildbot.pypy.org (groggi) Date: Mon, 23 Mar 2015 14:29:19 +0100 (CET) Subject: [pypy-commit] pypy gc-incminimark-pinning-improve: fix typo Message-ID: <20150323132919.54E0F1C0835@cobra.cs.uni-duesseldorf.de> Author: Gregor Wegberg Branch: gc-incminimark-pinning-improve Changeset: r76524:01630ad8c431 Date: 2015-03-22 18:36 +0100 http://bitbucket.org/pypy/pypy/changeset/01630ad8c431/ Log: fix typo diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -475,7 +475,7 @@ # the start of the nursery: we actually allocate a bit more for # the nursery than really needed, to simplify pointer arithmetic # in malloc_fixedsize(). The few extra pages are never used - # anyway so it doesn't even counct. + # anyway so it doesn't even count. nursery = llarena.arena_malloc(self._nursery_memory_size(), 0) if not nursery: out_of_memory("cannot allocate nursery") From noreply at buildbot.pypy.org Mon Mar 23 14:29:20 2015 From: noreply at buildbot.pypy.org (groggi) Date: Mon, 23 Mar 2015 14:29:20 +0100 (CET) Subject: [pypy-commit] pypy gc-incminimark-pinning-improve: improve comment a bit more Message-ID: <20150323132920.F42191C0835@cobra.cs.uni-duesseldorf.de> Author: Gregor Wegberg Branch: gc-incminimark-pinning-improve Changeset: r76525:2f78e998eec4 Date: 2015-03-23 11:21 +0100 http://bitbucket.org/pypy/pypy/changeset/2f78e998eec4/ Log: improve comment a bit more diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -704,21 +704,33 @@ if self.nursery_barriers.non_empty(): # Pinned object in front of nursery_top. Try reserving totalsize # by jumping into the next, yet unused, area inside the - # nursery. 'Next area' is in this case the space inside the - # nursery between the next pinned object and the one after that - # or the end of the nursery. Graphically explained: + # nursery. "Next area" in this case is the space between the + # pinned object in front of nusery_top and the pinned object + # after that. Graphically explained: # # |- allocating totalsize failed in this area - # v v- next pinned object, jump over this one - # +---------+--------+--------+--------+-----------+ - # | unknown | pinned | empty | pinned | empty | <- nursery - # +---------+--------+--------+--------+-----------+ - # ^- try reserving totalsize in here next + # | |- nursery_top + # | | |- pinned object in front of nursery_top, + # v v v jump over this + # +---------+--------+--------+--------+-----------+ } + # | used | pinned | empty | pinned | empty | }- nursery + # +---------+--------+--------+--------+-----------+ } + # ^- try reserving totalsize in here next # + # All pinned objects are represented by entries in + # nursery_barriers (see minor_collection). The last entry is + # always the end of the nursery. Therefore if nursery_barriers + # contains only one element, we jump over a pinned object and + # the "next area" (the space where we will try to allocate + # totalsize) starts at the end of the pinned object and ends at + # nursery's end. + # + # find the size of the pinned object after nursery_top size_gc_header = self.gcheaderbuilder.size_gc_header pinned_obj_size = size_gc_header + self.get_size( self.nursery_top + size_gc_header) # + # update used nursery space to allocate objects self.nursery_free = self.nursery_top + pinned_obj_size self.nursery_top = self.nursery_barriers.popleft() else: @@ -1645,7 +1657,9 @@ else: llarena.arena_reset(prev, self.nursery + self.nursery_size - prev, 0) # + # always add the end of the nursery to the list nursery_barriers.append(self.nursery + self.nursery_size) + # self.nursery_barriers = nursery_barriers self.surviving_pinned_objects.delete() # From noreply at buildbot.pypy.org Mon Mar 23 14:29:22 2015 From: noreply at buildbot.pypy.org (groggi) Date: Mon, 23 Mar 2015 14:29:22 +0100 (CET) Subject: [pypy-commit] pypy gc-incminimark-pinning-improve: fix typos Message-ID: <20150323132922.23CFF1C0835@cobra.cs.uni-duesseldorf.de> Author: Gregor Wegberg Branch: gc-incminimark-pinning-improve Changeset: r76526:13651a7e10d4 Date: 2015-03-23 11:31 +0100 http://bitbucket.org/pypy/pypy/changeset/13651a7e10d4/ Log: fix typos diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -1523,7 +1523,7 @@ # being moved, not from being collected if it is not reachable anymore. self.surviving_pinned_objects = self.AddressStack() # The following counter keeps track of alive and pinned young objects - # inside the nursery. We reset it here and increace it in + # inside the nursery. We reset it here and increase it in # '_trace_drag_out()'. any_pinned_object_from_earlier = self.any_pinned_object_kept self.pinned_objects_in_nursery = 0 @@ -2092,7 +2092,7 @@ # XXX A simplifying assumption that should be checked, # finalizers/weak references are rare and short which means that - # they do not need a seperate state and do not need to be + # they do not need a separate state and do not need to be # made incremental. if (not self.objects_to_trace.non_empty() and not self.more_objects_to_trace.non_empty()): From noreply at buildbot.pypy.org Mon Mar 23 14:29:23 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 23 Mar 2015 14:29:23 +0100 (CET) Subject: [pypy-commit] pypy default: Merged in groggi/pypy/gc-incminimark-pinning-improve (pull request #311) Message-ID: <20150323132923.3E5011C0835@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76527:e4bbecd90180 Date: 2015-03-23 14:29 +0100 http://bitbucket.org/pypy/pypy/changeset/e4bbecd90180/ Log: Merged in groggi/pypy/gc-incminimark-pinning-improve (pull request #311) Improve GC Pinning Documentation diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -52,6 +52,9 @@ # XXX total addressable size. Maybe by keeping some minimarkpage arenas # XXX pre-reserved, enough for a few nursery collections? What about # XXX raw-malloced memory? + +# XXX try merging old_objects_pointing_to_pinned into +# XXX old_objects_pointing_to_young (IRC 2014-10-22, fijal and gregor_w) import sys from rpython.rtyper.lltypesystem import lltype, llmemory, llarena, llgroup from rpython.rtyper.lltypesystem.lloperation import llop @@ -472,7 +475,7 @@ # the start of the nursery: we actually allocate a bit more for # the nursery than really needed, to simplify pointer arithmetic # in malloc_fixedsize(). The few extra pages are never used - # anyway so it doesn't even counct. + # anyway so it doesn't even count. nursery = llarena.arena_malloc(self._nursery_memory_size(), 0) if not nursery: out_of_memory("cannot allocate nursery") @@ -686,12 +689,10 @@ def collect_and_reserve(self, totalsize): """To call when nursery_free overflows nursery_top. - First check if the nursery_top is the real top, otherwise we - can just move the top of one cleanup and continue - - Do a minor collection, and possibly also a major collection, - and finally reserve 'totalsize' bytes at the start of the - now-empty nursery. + First check if pinned objects are in front of nursery_top. If so, + jump over the pinned object and try again to reserve totalsize. + Otherwise do a minor collection, and possibly a major collection, and + finally reserve totalsize bytes. """ minor_collection_count = 0 @@ -701,10 +702,35 @@ # we initialize nursery_free! if self.nursery_barriers.non_empty(): + # Pinned object in front of nursery_top. Try reserving totalsize + # by jumping into the next, yet unused, area inside the + # nursery. "Next area" in this case is the space between the + # pinned object in front of nusery_top and the pinned object + # after that. Graphically explained: + # + # |- allocating totalsize failed in this area + # | |- nursery_top + # | | |- pinned object in front of nursery_top, + # v v v jump over this + # +---------+--------+--------+--------+-----------+ } + # | used | pinned | empty | pinned | empty | }- nursery + # +---------+--------+--------+--------+-----------+ } + # ^- try reserving totalsize in here next + # + # All pinned objects are represented by entries in + # nursery_barriers (see minor_collection). The last entry is + # always the end of the nursery. Therefore if nursery_barriers + # contains only one element, we jump over a pinned object and + # the "next area" (the space where we will try to allocate + # totalsize) starts at the end of the pinned object and ends at + # nursery's end. + # + # find the size of the pinned object after nursery_top size_gc_header = self.gcheaderbuilder.size_gc_header pinned_obj_size = size_gc_header + self.get_size( self.nursery_top + size_gc_header) - + # + # update used nursery space to allocate objects self.nursery_free = self.nursery_top + pinned_obj_size self.nursery_top = self.nursery_barriers.popleft() else: @@ -732,6 +758,9 @@ "Seeing minor_collection() at least twice." "Too many pinned objects?") # + # Tried to do something about nursery_free overflowing + # nursery_top before this point. Try to reserve totalsize now. + # If this succeeds break out of loop. result = self.nursery_free if self.nursery_free + totalsize <= self.nursery_top: self.nursery_free = result + totalsize @@ -1494,7 +1523,7 @@ # being moved, not from being collected if it is not reachable anymore. self.surviving_pinned_objects = self.AddressStack() # The following counter keeps track of alive and pinned young objects - # inside the nursery. We reset it here and increace it in + # inside the nursery. We reset it here and increase it in # '_trace_drag_out()'. any_pinned_object_from_earlier = self.any_pinned_object_kept self.pinned_objects_in_nursery = 0 @@ -1628,7 +1657,9 @@ else: llarena.arena_reset(prev, self.nursery + self.nursery_size - prev, 0) # + # always add the end of the nursery to the list nursery_barriers.append(self.nursery + self.nursery_size) + # self.nursery_barriers = nursery_barriers self.surviving_pinned_objects.delete() # @@ -2061,7 +2092,7 @@ # XXX A simplifying assumption that should be checked, # finalizers/weak references are rare and short which means that - # they do not need a seperate state and do not need to be + # they do not need a separate state and do not need to be # made incremental. if (not self.objects_to_trace.non_empty() and not self.more_objects_to_trace.non_empty()): From noreply at buildbot.pypy.org Mon Mar 23 15:22:37 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 23 Mar 2015 15:22:37 +0100 (CET) Subject: [pypy-commit] pypy.org extradoc: update the values Message-ID: <20150323142237.7F94A1C0835@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r583:977e01af539d Date: 2015-03-23 15:23 +0100 http://bitbucket.org/pypy/pypy.org/changeset/977e01af539d/ Log: update the values diff --git a/don1.html b/don1.html --- a/don1.html +++ b/don1.html @@ -9,13 +9,13 @@ - $58935 of $105000 (56.1%) + $59002 of $105000 (56.2%)
diff --git a/don3.html b/don3.html --- a/don3.html +++ b/don3.html @@ -9,13 +9,13 @@ - $51326 of $60000 (85.5%) + $51350 of $60000 (85.6%)
From noreply at buildbot.pypy.org Mon Mar 23 15:24:08 2015 From: noreply at buildbot.pypy.org (groggi) Date: Mon, 23 Mar 2015 15:24:08 +0100 (CET) Subject: [pypy-commit] pypy gc-incminimark-pinning-improve: add environment variable to set the maximum number of pinned objects Message-ID: <20150323142408.717341C0F98@cobra.cs.uni-duesseldorf.de> Author: Gregor Wegberg Branch: gc-incminimark-pinning-improve Changeset: r76528:a3b1ea41b1f9 Date: 2015-03-23 15:23 +0100 http://bitbucket.org/pypy/pypy/changeset/a3b1ea41b1f9/ Log: add environment variable to set the maximum number of pinned objects If set to zero no object can be pinned. This can be useful to quickly check if a problem is because of pinning or not. diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -56,6 +56,7 @@ # XXX try merging old_objects_pointing_to_pinned into # XXX old_objects_pointing_to_young (IRC 2014-10-22, fijal and gregor_w) import sys +import os from rpython.rtyper.lltypesystem import lltype, llmemory, llarena, llgroup from rpython.rtyper.lltypesystem.lloperation import llop from rpython.rtyper.lltypesystem.llmemory import raw_malloc_usage @@ -463,9 +464,19 @@ self.nursery_size = newsize self.allocate_nursery() # - # Estimate this number conservatively - bigobj = self.nonlarge_max + 1 - self.max_number_of_pinned_objects = self.nursery_size / (bigobj * 2) + max_number_of_pinned_objects = os.environ.get('PYPY_GC_MAX_PINNED') + if max_number_of_pinned_objects: + try: + max_number_of_pinned_objects = int(max_number_of_pinned_objects) + except ValueError: + max_number_of_pinned_objects = 0 + # + if max_number_of_pinned_objects >= 0: # 0 allows to disable pinning completely + self.max_number_of_pinned_objects = max_number_of_pinned_objects + else: + # Estimate this number conservatively + bigobj = self.nonlarge_max + 1 + self.max_number_of_pinned_objects = self.nursery_size / (bigobj * 2) def _nursery_memory_size(self): extra = self.nonlarge_max + 1 From noreply at buildbot.pypy.org Mon Mar 23 15:24:09 2015 From: noreply at buildbot.pypy.org (groggi) Date: Mon, 23 Mar 2015 15:24:09 +0100 (CET) Subject: [pypy-commit] pypy gc-incminimark-pinning-improve: use pinning on input buffer in rzlib Message-ID: <20150323142409.A75801C0F98@cobra.cs.uni-duesseldorf.de> Author: Gregor Wegberg Branch: gc-incminimark-pinning-improve Changeset: r76529:5fda4d8f0436 Date: 2015-03-23 15:23 +0100 http://bitbucket.org/pypy/pypy/changeset/5fda4d8f0436/ Log: use pinning on input buffer in rzlib This removes one malloc for each compress()/decompress() diff --git a/rpython/rlib/rzlib.py b/rpython/rlib/rzlib.py --- a/rpython/rlib/rzlib.py +++ b/rpython/rlib/rzlib.py @@ -365,10 +365,7 @@ """Common code for compress() and decompress(). """ # Prepare the input buffer for the stream - with lltype.scoped_alloc(rffi.CCHARP.TO, len(data)) as inbuf: - # XXX (groggi) should be possible to improve this with pinning by - # not performing the 'copy_string_to_raw' if non-movable/pinned - copy_string_to_raw(llstr(data), inbuf, 0, len(data)) + with rffi.scoped_nonmovingbuffer(data) as inbuf: stream.c_next_in = rffi.cast(Bytefp, inbuf) rffi.setintfield(stream, 'c_avail_in', len(data)) From noreply at buildbot.pypy.org Mon Mar 23 15:40:20 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 23 Mar 2015 15:40:20 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Remove this stmgc-c4 test entirely for now. Message-ID: <20150323144020.2E1481C02BB@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76530:8f1796b78ec0 Date: 2015-03-23 15:33 +0100 http://bitbucket.org/pypy/pypy/changeset/8f1796b78ec0/ Log: Remove this stmgc-c4 test entirely for now. diff --git a/rpython/jit/backend/x86/test/test_stm_integration.py b/rpython/jit/backend/x86/test/test_stm_integration.py deleted file mode 100644 --- a/rpython/jit/backend/x86/test/test_stm_integration.py +++ /dev/null @@ -1,1128 +0,0 @@ -import py -from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rstr -from rpython.rtyper import rclass -from rpython.jit.metainterp.history import ResOperation, TargetToken,\ - JitCellToken -from rpython.jit.metainterp.history import (BoxInt, BoxPtr, ConstInt, - ConstPtr, Box, Const, - BasicFailDescr, BasicFinalDescr) -from rpython.jit.backend.detect_cpu import getcpuclass -from rpython.jit.backend.x86.arch import WORD -from rpython.jit.backend.x86.rx86 import fits_in_32bits -from rpython.jit.backend.llsupport import symbolic -from rpython.jit.metainterp.resoperation import rop -from rpython.jit.metainterp.executor import execute -from rpython.jit.backend.test.runner_test import LLtypeBackendTest -from rpython.jit.tool.oparser import parse -from rpython.rtyper.annlowlevel import llhelper -from rpython.jit.backend.llsupport.gc import BarrierDescr -from rpython.jit.backend.llsupport.test.test_gc_integration import ( - GCDescrShadowstackDirect, BaseTestRegalloc, JitFrameDescrs) -from rpython.jit.backend.llsupport import jitframe -from rpython.memory.gc.stmgc import StmGC -from rpython.jit.metainterp import history -from rpython.jit.codewriter.effectinfo import EffectInfo -from rpython.rlib import rgc -from rpython.rtyper.llinterp import LLException -import itertools, sys -import ctypes - -def cast_to_int(obj): - if isinstance(obj, rgc._GcRef): - return rgc.cast_gcref_to_int(obj) - else: - return rffi.cast(lltype.Signed, obj) - -CPU = getcpuclass() - -class MockSTMRootMap(object): - is_shadow_stack = True - is_stm = True - def __init__(self): - TP = rffi.CArray(lltype.Signed) - self.stack = lltype.malloc(TP, 10, flavor='raw') - self.stack_addr = lltype.malloc(TP, 1, - flavor='raw') - self.stack_addr[0] = rffi.cast(lltype.Signed, self.stack) - def register_asm_addr(self, start, mark): - pass - def get_root_stack_top_addr(self): - return rffi.cast(lltype.Signed, self.stack_addr) - -class FakeSTMBarrier(BarrierDescr): - def __init__(self, gc_ll_descr, stmcat, func): - BarrierDescr.__init__(self, gc_ll_descr) - self.stmcat = stmcat - self.returns_modified_object = True - self.B_FUNCPTR_MOD = lltype.Ptr(lltype.FuncType( - [llmemory.Address], llmemory.Address)) - self.write_barrier_fn = llhelper(self.B_FUNCPTR_MOD, func) - def get_barrier_funcptr(self, returns_modified_object): - assert returns_modified_object - return self.write_barrier_fn - def get_barrier_fn(self, cpu, returns_modified_object): - assert returns_modified_object - return self.write_barrier_fn - -# ____________________________________________________________ - -def allocate_protected(TP, n=1, zero=True, tid=124): - obj = lltype.malloc(TP, n=n, zero=zero) - obj.h_tid = rffi.cast(lltype.Unsigned, - StmGC.GCFLAG_OLD|StmGC.GCFLAG_WRITE_BARRIER | tid) - obj.h_revision = rffi.cast(lltype.Signed, -sys.maxint) - return obj - -def allocate_prebuilt(TP, n=1, zero=True, tid=123): - obj = lltype.malloc(TP, n=n, zero=zero) - obj.h_tid = rffi.cast(lltype.Unsigned, StmGC.PREBUILT_FLAGS | tid) - obj.h_revision = rffi.cast(lltype.Signed, StmGC.PREBUILT_REVISION) - return obj - -def jitframe_allocate(frame_info): - frame = allocate_protected(JITFRAME, n=frame_info.jfi_frame_depth, - zero=True) - frame.jf_frame_info = frame_info - return frame - -JITFRAME = lltype.GcStruct( - 'JITFRAME', - ('h_tid', lltype.Unsigned), - ('h_revision', lltype.Signed), - ('h_original', lltype.Unsigned), - ('jf_frame_info', lltype.Ptr(jitframe.JITFRAMEINFO)), - ('jf_descr', llmemory.GCREF), - ('jf_force_descr', llmemory.GCREF), - ('jf_extra_stack_depth', lltype.Signed), - ('jf_guard_exc', llmemory.GCREF), - ('jf_gcmap', lltype.Ptr(jitframe.GCMAP)), - ('jf_gc_trace_state', lltype.Signed), - ('jf_frame', lltype.Array(lltype.Signed)), - adtmeths = { - 'allocate': jitframe_allocate, - }, -) - -JITFRAMEPTR = lltype.Ptr(JITFRAME) - -class FakeGCHeaderBuilder: - size_gc_header = WORD - -class fakellop: - PRIV_REV = 66 - def __init__(self): - self.TP = rffi.CArray(lltype.Signed) - self.privrevp = lltype.malloc(self.TP, n=1, flavor='raw', - track_allocation=False, zero=True) - self.privrevp[0] = fakellop.PRIV_REV - - entries = (StmGC.FX_MASK + 1) / WORD - self.read_cache = lltype.malloc(self.TP, n=entries, flavor='raw', - track_allocation=False, zero=True) - self.read_cache_adr = lltype.malloc(self.TP, 1, flavor='raw', - track_allocation=False) - self.read_cache_adr[0] = rffi.cast(lltype.Signed, self.read_cache) - - def set_cache_item(self, obj, value): - obj_int = rffi.cast(lltype.Signed, obj) - idx = (obj_int & StmGC.FX_MASK) / WORD - self.read_cache[idx] = rffi.cast(lltype.Signed, value) - - def stm_get_adr_of_private_rev_num(self, _): - return self.privrevp - - def stm_get_adr_of_read_barrier_cache(self, _): - return self.read_cache_adr - -class GCDescrStm(GCDescrShadowstackDirect): - def __init__(self): - GCDescrShadowstackDirect.__init__(self) - self.gcrootmap = MockSTMRootMap() - self.gcheaderbuilder = FakeGCHeaderBuilder() - self.write_barrier_descr = None - self.llop1 = None - self.rb_called_on = [] - self.wb_called_on = [] - self.ptr_eq_called_on = [] - self.stm = True - - def read_barrier(obj): - self.rb_called_on.append(obj) - return obj - def write_barrier(obj): - self.wb_called_on.append(obj) - return obj - - self.A2Rdescr = FakeSTMBarrier(self, 'A2R', read_barrier) - self.A2Idescr = FakeSTMBarrier(self, 'A2I', read_barrier) - self.Q2Rdescr = FakeSTMBarrier(self, 'Q2R', read_barrier) - self.A2Wdescr = FakeSTMBarrier(self, 'A2W', write_barrier) - self.A2Vdescr = FakeSTMBarrier(self, 'A2V', write_barrier) - self.V2Wdescr = FakeSTMBarrier(self, 'V2W', write_barrier) - - self.do_write_barrier = None - self.get_nursery_top_addr = None - self.get_nursery_free_addr = None - - def malloc_str(length): - assert False - self.generate_function('malloc_str', malloc_str, - [lltype.Signed]) - def malloc_unicode(length): - assert False - self.generate_function('malloc_unicode', malloc_unicode, - [lltype.Signed]) - def inevitable(): - pass - self.generate_function('stm_try_inevitable', - inevitable, [], - RESULT=lltype.Void) - def ptr_eq(x, y): - print "=== ptr_eq", hex(cast_to_int(x)), hex(cast_to_int(y)) - self.ptr_eq_called_on.append((cast_to_int(x), cast_to_int(y))) - return x == y - self.generate_function('stm_ptr_eq', ptr_eq, [llmemory.GCREF] * 2, - RESULT=lltype.Bool) - - def stm_allocate_nonmovable_int_adr(obj): - assert False # should not be reached - return rgc.cast_gcref_to_int(obj) - self.generate_function('stm_allocate_nonmovable_int_adr', - stm_allocate_nonmovable_int_adr, - [llmemory.GCREF], - RESULT=lltype.Signed) - - def malloc_big_fixedsize(size, tid): - print "malloc:", size, tid - if size > sys.maxint / 2: - # for testing exception - raise LLException(0, 0) - - entries = size + StmGC.GCHDRSIZE - TP = rffi.CArray(lltype.Char) - obj = lltype.malloc(TP, n=entries, flavor='raw', - track_allocation=False, zero=True) - objptr = rffi.cast(StmGC.GCHDRP, obj) - objptr.h_tid = rffi.cast(lltype.Unsigned, - StmGC.GCFLAG_OLD - | StmGC.GCFLAG_WRITE_BARRIER | tid) - objptr.h_revision = rffi.cast(lltype.Signed, -sys.maxint) - print "return:", obj, objptr - return rffi.cast(llmemory.GCREF, objptr) - self.generate_function('malloc_big_fixedsize', malloc_big_fixedsize, - [lltype.Signed] * 2) - - - def malloc_jitframe(self, frame_info): - """ Allocate a new frame, overwritten by tests - """ - frame = JITFRAME.allocate(frame_info) - self.frames.append(frame) - return frame - - def getframedescrs(self, cpu): - descrs = JitFrameDescrs() - descrs.arraydescr = cpu.arraydescrof(JITFRAME) - for name in ['jf_descr', 'jf_guard_exc', 'jf_force_descr', - 'jf_frame_info', 'jf_gcmap', 'jf_extra_stack_depth']: - setattr(descrs, name, cpu.fielddescrof(JITFRAME, name)) - descrs.jfi_frame_depth = cpu.fielddescrof(jitframe.JITFRAMEINFO, - 'jfi_frame_depth') - descrs.jfi_frame_size = cpu.fielddescrof(jitframe.JITFRAMEINFO, - 'jfi_frame_size') - return descrs - - def get_malloc_slowpath_addr(self): - return None - - def clear_lists(self): - self.rb_called_on[:] = [] - self.wb_called_on[:] = [] - self.ptr_eq_called_on[:] = [] - - -class TestGcStm(BaseTestRegalloc): - - def setup_method(self, meth): - cpu = CPU(None, None) - cpu.gc_ll_descr = GCDescrStm() - - def latest_descr(self, deadframe): - deadframe = lltype.cast_opaque_ptr(JITFRAMEPTR, deadframe) - descr = deadframe.jf_descr - res = history.AbstractDescr.show(self, descr) - assert isinstance(res, history.AbstractFailDescr) - return res - import types - cpu.get_latest_descr = types.MethodType(latest_descr, cpu, - cpu.__class__) - - - self.a2wd = cpu.gc_ll_descr.A2Wdescr - self.a2vd = cpu.gc_ll_descr.A2Vdescr - self.v2wd = cpu.gc_ll_descr.V2Wdescr - self.a2rd = cpu.gc_ll_descr.A2Rdescr - self.a2id = cpu.gc_ll_descr.A2Idescr - self.q2rd = cpu.gc_ll_descr.Q2Rdescr - - TP = rffi.CArray(lltype.Signed) - self.priv_rev_num = lltype.malloc(TP, 1, flavor='raw') - self.clear_read_cache() - - cpu.assembler._get_stm_private_rev_num_addr = self.get_priv_rev_num - cpu.assembler._get_stm_read_barrier_cache_addr = self.get_read_cache - - S = lltype.GcForwardReference() - S.become(lltype.GcStruct( - 'S', ('h_tid', lltype.Unsigned), - ('h_revision', lltype.Signed), - ('h_original', lltype.Unsigned))) - cpu.gc_ll_descr.fielddescr_tid = None # not needed - # = cpu.fielddescrof(S, 'h_tid') - self.S = S - self.cpu = cpu - - def teardown_method(self, meth): - rffi.aroundstate._cleanup_() - - def assert_in(self, called_on, args): - for i, ref in enumerate(args): - assert rffi.cast_ptr_to_adr(ref) in called_on - - def assert_not_in(self, called_on, args): - for ref in args: - assert rffi.cast_ptr_to_adr(ref) not in called_on - - def get_priv_rev_num(self): - return rffi.cast(lltype.Signed, self.priv_rev_num) - - def get_read_cache(self): - return rffi.cast(lltype.Signed, self.read_cache_adr) - - def clear_read_cache(self): - TP = rffi.CArray(lltype.Signed) - entries = (StmGC.FX_MASK + 1) / WORD - self.read_cache = lltype.malloc(TP, n=entries, flavor='raw', - track_allocation=False, zero=True) - self.read_cache_adr = lltype.malloc(TP, 1, flavor='raw', - track_allocation=False) - self.read_cache_adr[0] = rffi.cast(lltype.Signed, self.read_cache) - - def set_cache_item(self, obj): - obj_int = rffi.cast(lltype.Signed, obj) - idx = (obj_int & StmGC.FX_MASK) / WORD - self.read_cache[idx] = obj_int - - def allocate_prebuilt_s(self, tid=66): - s = lltype.malloc(self.S, zero=True) - s.h_tid = rffi.cast(lltype.Unsigned, StmGC.PREBUILT_FLAGS | tid) - s.h_revision = rffi.cast(lltype.Signed, StmGC.PREBUILT_REVISION) - return s - - - - def test_gc_read_barrier_fastpath(self): - from rpython.jit.backend.llsupport.gc import STMReadBarrierDescr - descr = STMReadBarrierDescr(self.cpu.gc_ll_descr, 'A2R') - - called = [] - def read(obj): - called.append(obj) - return obj - - functype = lltype.Ptr(lltype.FuncType( - [llmemory.Address], llmemory.Address)) - funcptr = llhelper(functype, read) - descr.b_failing_case_ptr = funcptr - descr.llop1 = fakellop() - - # -------- TEST -------- - for rev in [fakellop.PRIV_REV+4, fakellop.PRIV_REV]: - called[:] = [] - - s = self.allocate_prebuilt_s() - sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s) - s.h_revision = rev - - descr._do_barrier(sgcref, - returns_modified_object=True) - - # check if rev-fastpath worked - if rev == fakellop.PRIV_REV: - # fastpath - self.assert_not_in(called, [sgcref]) - else: - self.assert_in(called, [sgcref]) - - # now check if sgcref in readcache: - called[:] = [] - descr.llop1.set_cache_item(sgcref, sgcref) - descr._do_barrier(sgcref, - returns_modified_object=True) - self.assert_not_in(called, [sgcref]) - descr.llop1.set_cache_item(sgcref, 0) - - - def test_gc_repeat_read_barrier_fastpath(self): - from rpython.jit.backend.llsupport.gc import STMReadBarrierDescr - descr = STMReadBarrierDescr(self.cpu.gc_ll_descr, 'Q2R') - - called = [] - def read(obj): - called.append(obj) - return obj - - functype = lltype.Ptr(lltype.FuncType( - [llmemory.Address], llmemory.Address)) - funcptr = llhelper(functype, read) - descr.b_failing_case_ptr = funcptr - descr.llop1 = fakellop() - - # -------- TEST -------- - for flags in [StmGC.GCFLAG_PUBLIC_TO_PRIVATE|StmGC.GCFLAG_MOVED, 0]: - called[:] = [] - - s = self.allocate_prebuilt_s() - sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s) - s.h_tid |= flags - - descr._do_barrier(sgcref, - returns_modified_object=True) - - # check if rev-fastpath worked - if not flags: - # fastpath - self.assert_not_in(called, [sgcref]) - else: - self.assert_in(called, [sgcref]) - - def test_gc_immutable_read_barrier_fastpath(self): - from rpython.jit.backend.llsupport.gc import STMReadBarrierDescr - descr = STMReadBarrierDescr(self.cpu.gc_ll_descr, 'A2I') - - called = [] - def read(obj): - called.append(obj) - return obj - - functype = lltype.Ptr(lltype.FuncType( - [llmemory.Address], llmemory.Address)) - funcptr = llhelper(functype, read) - descr.b_failing_case_ptr = funcptr - descr.llop1 = fakellop() - - # -------- TEST -------- - for flags in [StmGC.GCFLAG_STUB, 0]: - called[:] = [] - - s = self.allocate_prebuilt_s() - sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s) - s.h_tid |= flags - - descr._do_barrier(sgcref, returns_modified_object=True) - - # check if rev-fastpath worked - if not flags: - # fastpath - self.assert_not_in(called, [sgcref]) - else: - self.assert_in(called, [sgcref]) - - - - def test_gc_write_barrier_fastpath(self): - from rpython.jit.backend.llsupport.gc import STMWriteBarrierDescr - descr = STMWriteBarrierDescr(self.cpu.gc_ll_descr, 'A2W') - - called = [] - def write(obj): - called.append(obj) - return obj - - functype = lltype.Ptr(lltype.FuncType( - [llmemory.Address], llmemory.Address)) - funcptr = llhelper(functype, write) - descr.b_failing_case_ptr = funcptr - descr.llop1 = fakellop() - - # -------- TEST -------- - for rev in [fakellop.PRIV_REV+4, fakellop.PRIV_REV]: - called[:] = [] - - s = self.allocate_prebuilt_s() - sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s) - s.h_revision = rev - - descr._do_barrier(sgcref, - returns_modified_object=True) - - # check if fastpath worked - if rev == fakellop.PRIV_REV: - # fastpath - self.assert_not_in(called, [sgcref]) - else: - self.assert_in(called, [sgcref]) - - # now set WRITE_BARRIER -> always call slowpath - called[:] = [] - s.h_tid |= StmGC.GCFLAG_WRITE_BARRIER - descr._do_barrier(sgcref, - returns_modified_object=True) - self.assert_in(called, [sgcref]) - - def test_gc_repeat_write_barrier_fastpath(self): - from rpython.jit.backend.llsupport.gc import STMWriteBarrierDescr - descr = STMWriteBarrierDescr(self.cpu.gc_ll_descr, 'V2W') - - called = [] - def write(obj): - called.append(obj) - return obj - - functype = lltype.Ptr(lltype.FuncType( - [llmemory.Address], llmemory.Address)) - funcptr = llhelper(functype, write) - descr.b_failing_case_ptr = funcptr - descr.llop1 = fakellop() - - # -------- TEST -------- - s = self.allocate_prebuilt_s() - sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s) - - descr._do_barrier(sgcref, - returns_modified_object=True) - - # fastpath (WRITE_BARRIER not set) - self.assert_not_in(called, [sgcref]) - - # now set WRITE_BARRIER -> always call slowpath - s.h_tid |= StmGC.GCFLAG_WRITE_BARRIER - descr._do_barrier(sgcref, - returns_modified_object=True) - self.assert_in(called, [sgcref]) - - def test_gc_noptr_write_barrier_fastpath(self): - from rpython.jit.backend.llsupport.gc import STMWriteBarrierDescr - descr = STMWriteBarrierDescr(self.cpu.gc_ll_descr, 'A2V') - - called = [] - def write(obj): - called.append(obj) - return obj - - functype = lltype.Ptr(lltype.FuncType( - [llmemory.Address], llmemory.Address)) - funcptr = llhelper(functype, write) - descr.b_failing_case_ptr = funcptr - descr.llop1 = fakellop() - - # -------- TEST -------- - for rev in [fakellop.PRIV_REV+4, fakellop.PRIV_REV]: - called[:] = [] - - s = self.allocate_prebuilt_s() - sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s) - s.h_revision = rev - - descr._do_barrier(sgcref, returns_modified_object=True) - - # check if fastpath worked - if rev == fakellop.PRIV_REV: - # fastpath - self.assert_not_in(called, [sgcref]) - else: - self.assert_in(called, [sgcref]) - - # now set WRITE_BARRIER -> no effect - called[:] = [] - s.h_tid |= StmGC.GCFLAG_WRITE_BARRIER - descr._do_barrier(sgcref, returns_modified_object=True) - if rev == fakellop.PRIV_REV: - # fastpath - self.assert_not_in(called, [sgcref]) - else: - self.assert_in(called, [sgcref]) - - - - def test_read_barrier_fastpath(self): - cpu = self.cpu - cpu.gc_ll_descr.init_nursery(100) - cpu.setup_once() - PRIV_REV = rffi.cast(lltype.Signed, StmGC.PREBUILT_REVISION) - self.priv_rev_num[0] = PRIV_REV - called_on = cpu.gc_ll_descr.rb_called_on - for rev in [PRIV_REV+4, PRIV_REV]: - cpu.gc_ll_descr.clear_lists() - self.clear_read_cache() - - s = self.allocate_prebuilt_s() - sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s) - s.h_revision = rev - - p0 = BoxPtr() - operations = [ - ResOperation(rop.COND_CALL_STM_B, [p0], None, - descr=self.a2rd), - ResOperation(rop.FINISH, [p0], None, - descr=BasicFinalDescr(0)), - ] - inputargs = [p0] - looptoken = JitCellToken() - cpu.compile_loop(None, inputargs, operations, looptoken) - self.cpu.execute_token(looptoken, sgcref) - - # check if rev-fastpath worked - if rev == PRIV_REV: - # fastpath - self.assert_not_in(called_on, [sgcref]) - else: - self.assert_in(called_on, [sgcref]) - - # now add it to the read-cache and check - # that it will never call the read_barrier - cpu.gc_ll_descr.clear_lists() - self.set_cache_item(sgcref) - - self.cpu.execute_token(looptoken, sgcref) - # not called: - assert not called_on - - def test_repeat_read_barrier_fastpath(self): - cpu = self.cpu - cpu.gc_ll_descr.init_nursery(100) - cpu.setup_once() - - called_on = cpu.gc_ll_descr.rb_called_on - for flags in [StmGC.GCFLAG_PUBLIC_TO_PRIVATE|StmGC.GCFLAG_MOVED, 0]: - cpu.gc_ll_descr.clear_lists() - self.clear_read_cache() - - s = self.allocate_prebuilt_s() - sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s) - s.h_tid |= flags - - p0 = BoxPtr() - operations = [ - ResOperation(rop.COND_CALL_STM_B, [p0], None, - descr=self.q2rd), - ResOperation(rop.FINISH, [p0], None, - descr=BasicFinalDescr(0)), - ] - inputargs = [p0] - looptoken = JitCellToken() - cpu.compile_loop(None, inputargs, operations, looptoken) - self.cpu.execute_token(looptoken, sgcref) - - # check if rev-fastpath worked - if not flags: - # fastpath - self.assert_not_in(called_on, [sgcref]) - else: - self.assert_in(called_on, [sgcref]) - - def test_immutable_read_barrier_fastpath(self): - cpu = self.cpu - cpu.gc_ll_descr.init_nursery(100) - cpu.setup_once() - - called_on = cpu.gc_ll_descr.rb_called_on - for flags in [StmGC.GCFLAG_STUB, 0]: - cpu.gc_ll_descr.clear_lists() - self.clear_read_cache() - - s = self.allocate_prebuilt_s() - sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s) - s.h_tid |= flags - - p0 = BoxPtr() - operations = [ - ResOperation(rop.COND_CALL_STM_B, [p0], None, - descr=self.a2id), - ResOperation(rop.FINISH, [p0], None, - descr=BasicFinalDescr(0)), - ] - inputargs = [p0] - looptoken = JitCellToken() - cpu.compile_loop(None, inputargs, operations, looptoken) - self.cpu.execute_token(looptoken, sgcref) - - # check if rev-fastpath worked - if not flags: - # fastpath - self.assert_not_in(called_on, [sgcref]) - else: - self.assert_in(called_on, [sgcref]) - - - - def test_write_barrier_fastpath(self): - cpu = self.cpu - cpu.gc_ll_descr.init_nursery(100) - cpu.setup_once() - PRIV_REV = rffi.cast(lltype.Signed, StmGC.PREBUILT_REVISION) - self.priv_rev_num[0] = PRIV_REV - called_on = cpu.gc_ll_descr.wb_called_on - - for rev in [PRIV_REV+4, PRIV_REV]: - cpu.gc_ll_descr.clear_lists() - - s = self.allocate_prebuilt_s() - sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s) - s.h_revision = rev - - p0 = BoxPtr() - operations = [ - ResOperation(rop.COND_CALL_STM_B, [p0], None, - descr=self.a2wd), - ResOperation(rop.FINISH, [p0], None, - descr=BasicFinalDescr(0)), - ] - inputargs = [p0] - looptoken = JitCellToken() - cpu.compile_loop(None, inputargs, operations, looptoken) - self.cpu.execute_token(looptoken, sgcref) - - # check if rev-fastpath worked - if rev == PRIV_REV: - # fastpath and WRITE_BARRIER not set - self.assert_not_in(called_on, [sgcref]) - else: - self.assert_in(called_on, [sgcref]) - - # now set WRITE_BARRIER -> always call slowpath - cpu.gc_ll_descr.clear_lists() - s.h_tid |= StmGC.GCFLAG_WRITE_BARRIER - self.cpu.execute_token(looptoken, sgcref) - self.assert_in(called_on, [sgcref]) - - def test_repeat_write_barrier_fastpath(self): - cpu = self.cpu - cpu.gc_ll_descr.init_nursery(100) - cpu.setup_once() - - called_on = cpu.gc_ll_descr.wb_called_on - cpu.gc_ll_descr.clear_lists() - - s = self.allocate_prebuilt_s() - sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s) - - p0 = BoxPtr() - operations = [ - ResOperation(rop.COND_CALL_STM_B, [p0], None, - descr=self.v2wd), - ResOperation(rop.FINISH, [p0], None, - descr=BasicFinalDescr(0)), - ] - - inputargs = [p0] - looptoken = JitCellToken() - cpu.compile_loop(None, inputargs, operations, looptoken) - self.cpu.execute_token(looptoken, sgcref) - - # fastpath and WRITE_BARRIER not set - self.assert_not_in(called_on, [sgcref]) - - # now set WRITE_BARRIER -> always call slowpath - cpu.gc_ll_descr.clear_lists() - s.h_tid |= StmGC.GCFLAG_WRITE_BARRIER - self.cpu.execute_token(looptoken, sgcref) - self.assert_in(called_on, [sgcref]) - - def test_noptr_write_barrier_fastpath(self): - cpu = self.cpu - cpu.gc_ll_descr.init_nursery(100) - cpu.setup_once() - PRIV_REV = rffi.cast(lltype.Signed, StmGC.PREBUILT_REVISION) - self.priv_rev_num[0] = PRIV_REV - called_on = cpu.gc_ll_descr.wb_called_on - - for rev in [PRIV_REV+4, PRIV_REV]: - cpu.gc_ll_descr.clear_lists() - - s = self.allocate_prebuilt_s() - sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s) - s.h_revision = rev - - p0 = BoxPtr() - operations = [ - ResOperation(rop.COND_CALL_STM_B, [p0], None, - descr=self.a2vd), - ResOperation(rop.FINISH, [p0], None, - descr=BasicFinalDescr(0)), - ] - inputargs = [p0] - looptoken = JitCellToken() - cpu.compile_loop(None, inputargs, operations, looptoken) - self.cpu.execute_token(looptoken, sgcref) - - # check if rev-fastpath worked - if rev == PRIV_REV: - # fastpath and WRITE_BARRIER not set - self.assert_not_in(called_on, [sgcref]) - else: - self.assert_in(called_on, [sgcref]) - - # now set WRITE_BARRIER -> no effect - cpu.gc_ll_descr.clear_lists() - s.h_tid |= StmGC.GCFLAG_WRITE_BARRIER - self.cpu.execute_token(looptoken, sgcref) - if rev == PRIV_REV: - # fastpath and WRITE_BARRIER not set - self.assert_not_in(called_on, [sgcref]) - else: - self.assert_in(called_on, [sgcref]) - - - def test_ptr_eq_fastpath(self): - cpu = self.cpu - cpu.gc_ll_descr.init_nursery(100) - cpu.setup_once() - called_on = cpu.gc_ll_descr.ptr_eq_called_on - - i0 = BoxInt() - i1 = BoxInt() - sa, sb = (rffi.cast(llmemory.GCREF, self.allocate_prebuilt_s()), - rffi.cast(llmemory.GCREF, self.allocate_prebuilt_s())) - ss = [sa, sa, sb, sb, - lltype.nullptr(llmemory.GCREF.TO), - lltype.nullptr(llmemory.GCREF.TO), - ] - for s1, s2 in itertools.combinations(ss, 2): - ps = [BoxPtr(), BoxPtr(), - ConstPtr(s1), - ConstPtr(s2)] - for p1, p2 in itertools.combinations(ps, 2): - for guard in [None, rop.GUARD_TRUE, rop.GUARD_FALSE, - rop.GUARD_VALUE]: - cpu.gc_ll_descr.clear_lists() - - # BUILD OPERATIONS: - i = i0 - guarddescr = BasicFailDescr() - finaldescr = BasicFinalDescr() - if guard == rop.GUARD_VALUE: - gop = ResOperation(rop.GUARD_VALUE, [p1, p2], None, - descr=guarddescr) - gop.setfailargs([]) - operations = [gop] - i = i1 - else: - operations = [ResOperation(rop.PTR_EQ, [p1, p2], i0)] - if guard is not None: - gop = ResOperation(guard, [i0], None, - descr=guarddescr) - gop.setfailargs([]) - operations.append(gop) - i = i1 - # finish must depend on result of ptr_eq if no guard - # is inbetween (otherwise ptr_eq gets deleted) - # if there is a guard, the result of ptr_eq must not - # be used after it again... -> i - operations.append( - ResOperation(rop.FINISH, [i], None, - descr=finaldescr) - ) - print operations - - - # COMPILE & EXECUTE LOOP: - inputargs = [p for p in (p1, p2) - if not isinstance(p, Const)] - looptoken = JitCellToken() - c_loop = cpu.compile_loop(None, inputargs + [i1], - operations, looptoken) - - args = [s for i, s in enumerate((s1, s2)) - if not isinstance((p1, p2)[i], Const)] + [7] - - deadframe = self.cpu.execute_token(looptoken, *args) - frame = rffi.cast(JITFRAMEPTR, deadframe) - frame_adr = rffi.cast(lltype.Signed, frame.jf_descr) - guard_failed = frame_adr != id(finaldescr) - - # CHECK: - a, b = cast_to_int(s1), cast_to_int(s2) - if isinstance(p1, Const): - a = cast_to_int(p1.value) - if isinstance(p2, Const): - b = cast_to_int(p2.value) - - # XXX: there is now no function being called in the - # slowpath, so we can't check if fast- vs. slowpath - # works :/ - - # if a == b or a == 0 or b == 0: - # assert (a, b) not in called_on - # assert (b, a) not in called_on - # else: - # assert ([(a, b)] == called_on - # or [(b, a)] == called_on) - - if guard is not None: - if a == b: - if guard in (rop.GUARD_TRUE, rop.GUARD_VALUE): - assert not guard_failed - else: - assert guard_failed - elif guard == rop.GUARD_FALSE: - assert not guard_failed - else: - assert guard_failed - - - - - def test_assembler_call(self): - cpu = self.cpu - cpu.gc_ll_descr.init_nursery(100) - cpu.setup_once() - - called = [] - def assembler_helper(deadframe, virtualizable): - frame = rffi.cast(JITFRAMEPTR, deadframe) - frame_adr = rffi.cast(lltype.Signed, frame.jf_descr) - called.append(frame_adr) - return 4 + 9 - - FUNCPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF, - llmemory.GCREF], - lltype.Signed)) - class FakeJitDriverSD: - index_of_virtualizable = -1 - _assembler_helper_ptr = llhelper(FUNCPTR, assembler_helper) - assembler_helper_adr = llmemory.cast_ptr_to_adr( - _assembler_helper_ptr) - - ops = ''' - [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9] - i10 = int_add(i0, i1) - i11 = int_add(i10, i2) - i12 = int_add(i11, i3) - i13 = int_add(i12, i4) - i14 = int_add(i13, i5) - i15 = int_add(i14, i6) - i16 = int_add(i15, i7) - i17 = int_add(i16, i8) - i18 = int_add(i17, i9) - finish(i18)''' - loop = parse(ops) - looptoken = JitCellToken() - looptoken.outermost_jitdriver_sd = FakeJitDriverSD() - finish_descr = loop.operations[-1].getdescr() - self.cpu.done_with_this_frame_descr_int = BasicFinalDescr() - self.cpu.compile_loop(None, loop.inputargs, loop.operations, looptoken) - ARGS = [lltype.Signed] * 10 - RES = lltype.Signed - FakeJitDriverSD.portal_calldescr = self.cpu.calldescrof( - lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES, - EffectInfo.MOST_GENERAL) - args = [i+1 for i in range(10)] - deadframe = self.cpu.execute_token(looptoken, *args) - - ops = ''' - [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9] - i10 = int_add(i0, 42) - i11 = call_assembler(i10, i1, i2, i3, i4, i5, i6, i7, i8, i9, descr=looptoken) - guard_not_forced()[] - finish(i11) - ''' - loop = parse(ops, namespace=locals()) - othertoken = JitCellToken() - self.cpu.compile_loop(None, loop.inputargs, loop.operations, - othertoken) - args = [i+1 for i in range(10)] - deadframe = self.cpu.execute_token(othertoken, *args) - assert called == [id(finish_descr)] - del called[:] - - # compile a replacement - ops = ''' - [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9] - i10 = int_sub(i0, i1) - i11 = int_sub(i10, i2) - i12 = int_sub(i11, i3) - i13 = int_sub(i12, i4) - i14 = int_sub(i13, i5) - i15 = int_sub(i14, i6) - i16 = int_sub(i15, i7) - i17 = int_sub(i16, i8) - i18 = int_sub(i17, i9) - finish(i18)''' - loop2 = parse(ops) - looptoken2 = JitCellToken() - looptoken2.outermost_jitdriver_sd = FakeJitDriverSD() - self.cpu.compile_loop(None, loop2.inputargs, loop2.operations, - looptoken2) - finish_descr2 = loop2.operations[-1].getdescr() - - # install it - self.cpu.redirect_call_assembler(looptoken, looptoken2) - - # now call_assembler should go to looptoken2 - args = [i+1 for i in range(10)] - deadframe = self.cpu.execute_token(othertoken, *args) - assert called == [id(finish_descr2)] - - - def test_call_malloc_gc(self): - cpu = self.cpu - cpu.gc_ll_descr.init_nursery(100) - cpu.setup_once() - - size = WORD*3 - addr = cpu.gc_ll_descr.get_malloc_fn_addr('malloc_big_fixedsize') - typeid = 11 - descr = cpu.gc_ll_descr.malloc_big_fixedsize_descr - - p0 = BoxPtr() - ops1 = [ResOperation(rop.CALL_MALLOC_GC, - [ConstInt(addr), ConstInt(size), ConstInt(typeid)], - p0, descr), - ResOperation(rop.FINISH, [p0], None, - descr=BasicFinalDescr(0)), - ] - - inputargs = [] - looptoken = JitCellToken() - c_loop = cpu.compile_loop(None, inputargs, ops1, - looptoken) - - args = [] - - frame = self.cpu.execute_token(looptoken, *args) - - - def test_assembler_call_propagate_exc(self): - cpu = self.cpu - cpu._setup_descrs() - cpu.gc_ll_descr.init_nursery(100) - - excdescr = BasicFailDescr(666) - cpu.propagate_exception_descr = excdescr - cpu.setup_once() # xxx redo it, because we added - # propagate_exception - - def assembler_helper(deadframe, virtualizable): - #assert cpu.get_latest_descr(deadframe) is excdescr - # let's assume we handled that - return 3 - - FUNCPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF, - llmemory.GCREF], - lltype.Signed)) - class FakeJitDriverSD: - index_of_virtualizable = -1 - _assembler_helper_ptr = llhelper(FUNCPTR, assembler_helper) - assembler_helper_adr = llmemory.cast_ptr_to_adr( - _assembler_helper_ptr) - - - - addr = cpu.gc_ll_descr.get_malloc_fn_addr('malloc_big_fixedsize') - typeid = 11 - descr = cpu.gc_ll_descr.malloc_big_fixedsize_descr - - p0 = BoxPtr() - i0 = BoxInt() - ops = [ResOperation(rop.CALL_MALLOC_GC, - [ConstInt(addr), i0, ConstInt(typeid)], - p0, descr), - ResOperation(rop.FINISH, [p0], None, - descr=BasicFinalDescr(0)), - ] - - inputargs = [i0] - looptoken = JitCellToken() - looptoken.outermost_jitdriver_sd = FakeJitDriverSD() - c_loop = cpu.compile_loop(None, inputargs, ops, looptoken) - - ARGS = [lltype.Signed] * 10 - RES = lltype.Signed - FakeJitDriverSD.portal_calldescr = cpu.calldescrof( - lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES, - EffectInfo.MOST_GENERAL) - i1 = ConstInt(sys.maxint - 1) - i2 = BoxInt() - finaldescr = BasicFinalDescr(1) - not_forced = ResOperation(rop.GUARD_NOT_FORCED, [], None, - descr=BasicFailDescr(1)) - not_forced.setfailargs([]) - no_exception = ResOperation(rop.GUARD_NO_EXCEPTION, [], None, - descr=BasicFailDescr(2)) - no_exception.setfailargs([]) - ops = [ResOperation(rop.CALL_ASSEMBLER, [i1], i2, descr=looptoken), - not_forced, - no_exception, - ResOperation(rop.FINISH, [i1], None, descr=finaldescr), - ] - othertoken = JitCellToken() - cpu.done_with_this_frame_descr_int = BasicFinalDescr() - c_loop = cpu.compile_loop(None, [], ops, othertoken) - - deadframe = cpu.execute_token(othertoken) - frame = rffi.cast(JITFRAMEPTR, deadframe) - descr = rffi.cast(lltype.Signed, frame.jf_descr) - assert descr != id(finaldescr) - - - def test_write_barrier_on_spilled(self): - cpu = self.cpu - - PRIV_REV = rffi.cast(lltype.Signed, StmGC.PREBUILT_REVISION) - self.priv_rev_num[0] = PRIV_REV - - s = self.allocate_prebuilt_s() - other_s = self.allocate_prebuilt_s() - sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s) - other_sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, other_s) - s.h_revision = PRIV_REV+4 - other_s.h_revision = PRIV_REV+4 - - called_on = [] - def write_barrier(obj): - called_on.append(obj) - if llmemory.cast_ptr_to_adr(sgcref) == obj: - return rffi.cast(llmemory.Address, other_sgcref) - return obj - A2W = FakeSTMBarrier(cpu.gc_ll_descr, 'A2W', write_barrier) - old_a2w = cpu.gc_ll_descr.A2Wdescr - cpu.gc_ll_descr.A2Wdescr = A2W - - cpu.gc_ll_descr.init_nursery(100) - cpu.setup_once() - - - from rpython.jit.tool.oparser import FORCE_SPILL - p0 = BoxPtr() - spill = FORCE_SPILL(None) - spill.initarglist([p0]) - operations = [ - ResOperation(rop.COND_CALL_STM_B, [p0], None, - descr=A2W), - spill, - ResOperation(rop.COND_CALL_STM_B, [p0], None, - descr=A2W), - ResOperation(rop.FINISH, [p0], None, - descr=BasicFinalDescr(0)), - ] - inputargs = [p0] - looptoken = JitCellToken() - print cpu.compile_loop(None, inputargs, operations, looptoken) - cpu.execute_token(looptoken, sgcref) - - # the second write-barrier must see the result of the - # first one - self.assert_in(called_on, [sgcref, other_sgcref]) - - # for other tests: - cpu.gc_ll_descr.A2Wdescr = old_a2w - - - - - - - - - From noreply at buildbot.pypy.org Mon Mar 23 15:40:21 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 23 Mar 2015 15:40:21 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Fix tests Message-ID: <20150323144021.6142E1C02BB@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76531:7d83bad9774a Date: 2015-03-23 15:37 +0100 http://bitbucket.org/pypy/pypy/changeset/7d83bad9774a/ Log: Fix tests diff --git a/rpython/jit/backend/llsupport/test/test_gc.py b/rpython/jit/backend/llsupport/test/test_gc.py --- a/rpython/jit/backend/llsupport/test/test_gc.py +++ b/rpython/jit/backend/llsupport/test/test_gc.py @@ -2,7 +2,7 @@ from rpython.rtyper.lltypesystem import lltype, llmemory, rstr from rpython.rtyper.lltypesystem.lloperation import llop from rpython.rtyper.annlowlevel import llhelper -from rpython.jit.backend.llsupport import jitframe, gc, descr, gcmap +from rpython.jit.backend.llsupport import jitframe, gc, descr from rpython.jit.backend.llsupport import symbolic from rpython.jit.metainterp.gc import get_description from rpython.jit.metainterp.history import BoxPtr, BoxInt, ConstPtr @@ -188,7 +188,7 @@ rewriter = GcRewriterAssembler(gc_ll_descr, None) newops = rewriter.newops v_base = BoxPtr() - rewriter.gen_write_barrier(v_base, stm_location=None) + rewriter.gen_write_barrier(v_base) assert llop1.record == [] assert len(newops) == 1 assert newops[0].getopnum() == rop.COND_CALL_GC_WB @@ -246,8 +246,7 @@ frame_info = lltype.malloc(jitframe.JITFRAMEINFO, zero=True, flavor='raw') frame = lltype.malloc(jitframe.JITFRAME, 200, zero=True) frame.jf_frame_info = frame_info - frame.jf_gcmap = lltype.malloc(jitframe.GCMAP, 4 + gcmap.GCMAP_STM_LOCATION, - flavor='raw') + frame.jf_gcmap = lltype.malloc(jitframe.GCMAP, 4, flavor='raw') if sys.maxint == 2**31 - 1: max = r_uint(2 ** 31) else: diff --git a/rpython/jit/backend/llsupport/test/test_gc_integration.py b/rpython/jit/backend/llsupport/test/test_gc_integration.py --- a/rpython/jit/backend/llsupport/test/test_gc_integration.py +++ b/rpython/jit/backend/llsupport/test/test_gc_integration.py @@ -10,7 +10,7 @@ GcLLDescr_framework, GcCache, JitFrameDescrs from rpython.jit.backend.detect_cpu import getcpuclass from rpython.jit.backend.llsupport.symbolic import WORD -from rpython.jit.backend.llsupport import jitframe, gcmap +from rpython.jit.backend.llsupport import jitframe from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.rtyper.annlowlevel import llhelper, llhelper_args @@ -315,13 +315,11 @@ def test_malloc_slowpath(self): def check(frame): - # xxx for now we always have GCMAP_STM_LOCATION, but it should - # be added only if we really have stm in the first place - expected_size = 1 + gcmap.GCMAP_STM_LOCATION + expected_size = 1 idx = 0 if self.cpu.backend_name.startswith('arm'): # jitframe fixed part is larger here - expected_size = 2 + gcmap.GCMAP_STM_LOCATION + expected_size = 2 idx = 1 assert len(frame.jf_gcmap) == expected_size if self.cpu.IS_64_BIT: @@ -357,11 +355,11 @@ def check(frame): x = frame.jf_gcmap if self.cpu.IS_64_BIT: - assert len(x) == 1 + gcmap.GCMAP_STM_LOCATION + assert len(x) == 1 assert (bin(x[0]).count('1') == '0b1111100000000000000001111111011110'.count('1')) else: - assert len(x) == 2 + gcmap.GCMAP_STM_LOCATION + assert len(x) == 2 s = bin(x[0]).count('1') + bin(x[1]).count('1') assert s == 16 # all but two registers + some stuff on stack diff --git a/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py b/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py --- a/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py +++ b/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py @@ -109,6 +109,7 @@ gcrootfinder = 'asmgcc' gctransformer = 'framework' gcremovetypeptr = False + stm = False gcdescr = get_description(config_) self.gc_ll_descr = GcLLDescr_framework(gcdescr, None, None, None, really_not_translated=True) From noreply at buildbot.pypy.org Mon Mar 23 16:27:28 2015 From: noreply at buildbot.pypy.org (groggi) Date: Mon, 23 Mar 2015 16:27:28 +0100 (CET) Subject: [pypy-commit] pypy gc-incminimark-pinning-improve: make variable names a bit more distinct Message-ID: <20150323152728.CE5341C0F0E@cobra.cs.uni-duesseldorf.de> Author: Gregor Wegberg Branch: gc-incminimark-pinning-improve Changeset: r76532:85a6845a0bde Date: 2015-03-23 16:24 +0100 http://bitbucket.org/pypy/pypy/changeset/85a6845a0bde/ Log: make variable names a bit more distinct diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -464,15 +464,15 @@ self.nursery_size = newsize self.allocate_nursery() # - max_number_of_pinned_objects = os.environ.get('PYPY_GC_MAX_PINNED') - if max_number_of_pinned_objects: + env_max_number_of_pinned_objects = os.environ.get('PYPY_GC_MAX_PINNED') + if env_max_number_of_pinned_objects: try: - max_number_of_pinned_objects = int(max_number_of_pinned_objects) + env_max_number_of_pinned_objects = int(env_max_number_of_pinned_objects) except ValueError: - max_number_of_pinned_objects = 0 + env_max_number_of_pinned_objects = 0 # - if max_number_of_pinned_objects >= 0: # 0 allows to disable pinning completely - self.max_number_of_pinned_objects = max_number_of_pinned_objects + if env_max_number_of_pinned_objects >= 0: # 0 allows to disable pinning completely + self.max_number_of_pinned_objects = env_max_number_of_pinned_objects else: # Estimate this number conservatively bigobj = self.nonlarge_max + 1 From noreply at buildbot.pypy.org Mon Mar 23 16:27:30 2015 From: noreply at buildbot.pypy.org (groggi) Date: Mon, 23 Mar 2015 16:27:30 +0100 (CET) Subject: [pypy-commit] pypy gc-incminimark-pinning-improve: translation fix, should be double checked (all tests pass) Message-ID: <20150323152730.0C1DF1C0F0E@cobra.cs.uni-duesseldorf.de> Author: Gregor Wegberg Branch: gc-incminimark-pinning-improve Changeset: r76533:58d1159cd44c Date: 2015-03-23 16:24 +0100 http://bitbucket.org/pypy/pypy/changeset/58d1159cd44c/ Log: translation fix, should be double checked (all tests pass) diff --git a/rpython/rlib/rzlib.py b/rpython/rlib/rzlib.py --- a/rpython/rlib/rzlib.py +++ b/rpython/rlib/rzlib.py @@ -365,6 +365,7 @@ """Common code for compress() and decompress(). """ # Prepare the input buffer for the stream + assert data is not None # XXX seems to be sane assumption, however not for sure with rffi.scoped_nonmovingbuffer(data) as inbuf: stream.c_next_in = rffi.cast(Bytefp, inbuf) rffi.setintfield(stream, 'c_avail_in', len(data)) From noreply at buildbot.pypy.org Mon Mar 23 16:27:31 2015 From: noreply at buildbot.pypy.org (groggi) Date: Mon, 23 Mar 2015 16:27:31 +0100 (CET) Subject: [pypy-commit] pypy gc-incminimark-pinning-improve: use pinning for input buffers in bz2 Message-ID: <20150323152731.288501C0F0E@cobra.cs.uni-duesseldorf.de> Author: Gregor Wegberg Branch: gc-incminimark-pinning-improve Changeset: r76534:a95dcace656d Date: 2015-03-23 16:25 +0100 http://bitbucket.org/pypy/pypy/changeset/a95dcace656d/ Log: use pinning for input buffers in bz2 diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py --- a/pypy/module/bz2/interp_bz2.py +++ b/pypy/module/bz2/interp_bz2.py @@ -562,10 +562,7 @@ in_bufsize = datasize with OutBuffer(self.bzs) as out: - with lltype.scoped_alloc(rffi.CCHARP.TO, in_bufsize) as in_buf: - - for i in range(datasize): - in_buf[i] = data[i] + with rffi.scoped_nonmovingbuffer(data) as in_buf: self.bzs.c_next_in = in_buf rffi.setintfield(self.bzs, 'c_avail_in', in_bufsize) @@ -663,9 +660,7 @@ in_bufsize = len(data) - with lltype.scoped_alloc(rffi.CCHARP.TO, in_bufsize) as in_buf: - for i in range(in_bufsize): - in_buf[i] = data[i] + with rffi.scoped_nonmovingbuffer(data) as in_buf: self.bzs.c_next_in = in_buf rffi.setintfield(self.bzs, 'c_avail_in', in_bufsize) @@ -716,9 +711,7 @@ with lltype.scoped_alloc(bz_stream.TO, zero=True) as bzs: in_bufsize = len(data) - with lltype.scoped_alloc(rffi.CCHARP.TO, in_bufsize) as in_buf: - for i in range(in_bufsize): - in_buf[i] = data[i] + with rffi.scoped_nonmovingbuffer(data) as in_buf: bzs.c_next_in = in_buf rffi.setintfield(bzs, 'c_avail_in', in_bufsize) @@ -758,9 +751,7 @@ return space.wrap("") with lltype.scoped_alloc(bz_stream.TO, zero=True) as bzs: - with lltype.scoped_alloc(rffi.CCHARP.TO, in_bufsize) as in_buf: - for i in range(in_bufsize): - in_buf[i] = data[i] + with rffi.scoped_nonmovingbuffer(data) as in_buf: bzs.c_next_in = in_buf rffi.setintfield(bzs, 'c_avail_in', in_bufsize) From noreply at buildbot.pypy.org Mon Mar 23 16:35:45 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 23 Mar 2015 16:35:45 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: fix Message-ID: <20150323153545.C1B941C0994@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76535:069f9100d94e Date: 2015-03-23 16:02 +0100 http://bitbucket.org/pypy/pypy/changeset/069f9100d94e/ Log: fix diff --git a/rpython/rtyper/rtyper.py b/rpython/rtyper/rtyper.py --- a/rpython/rtyper/rtyper.py +++ b/rpython/rtyper/rtyper.py @@ -371,6 +371,10 @@ # make sure that the return variables of all graphs # are concretetype'd self.setconcretetype(graph.getreturnvar()) + # + v1, v2 = graph.exceptblock.inputargs + v1.concretetype = self.exceptiondata.lltype_of_exception_type + v2.concretetype = self.exceptiondata.lltype_of_exception_value def _specialize_block(self, block): # give the best possible types to the input args From noreply at buildbot.pypy.org Mon Mar 23 16:35:46 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 23 Mar 2015 16:35:46 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Fix test Message-ID: <20150323153546.DBFF71C0994@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76536:87f3fee1cc6b Date: 2015-03-23 16:23 +0100 http://bitbucket.org/pypy/pypy/changeset/87f3fee1cc6b/ Log: Fix test diff --git a/rpython/jit/backend/llsupport/test/zrpy_gc_test.py b/rpython/jit/backend/llsupport/test/zrpy_gc_test.py --- a/rpython/jit/backend/llsupport/test/zrpy_gc_test.py +++ b/rpython/jit/backend/llsupport/test/zrpy_gc_test.py @@ -223,6 +223,7 @@ class CompileFrameworkTests(BaseFrameworkTests): # Test suite using (so far) the minimark GC. + can_pin = True ## def define_libffi_workaround(cls): ## # XXX: this is a workaround for a bug in database.py. It seems that @@ -921,6 +922,8 @@ return None, fn, None def test_pinned_simple(self): + if not self.can_pin: + py.test.skip("not in this configuration") self.run('pinned_simple') def define_pinned_unpin(cls): @@ -965,6 +968,8 @@ return None, fn, after def test_pinned_unpin(self): + if not self.can_pin: + py.test.skip("not in this configuration") self.run('pinned_unpin') def define_multiple_pinned(cls): @@ -1012,4 +1017,6 @@ return None, fn, None def test_multiple_pinned(self): + if not self.can_pin: + py.test.skip("not in this configuration") self.run('multiple_pinned') diff --git a/rpython/jit/backend/x86/test/test_zrpy_gc.py b/rpython/jit/backend/x86/test/test_zrpy_gc.py --- a/rpython/jit/backend/x86/test/test_zrpy_gc.py +++ b/rpython/jit/backend/x86/test/test_zrpy_gc.py @@ -3,6 +3,7 @@ class TestSTMShadowStack(CompileFrameworkTests): gcrootfinder = "stm" + can_pin = False class TestShadowStack(CompileFrameworkTests): From noreply at buildbot.pypy.org Mon Mar 23 17:16:24 2015 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 23 Mar 2015 17:16:24 +0100 (CET) Subject: [pypy-commit] pypy release-2.5.x: merge default into branch Message-ID: <20150323161624.371CF1C0835@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: release-2.5.x Changeset: r76537:9c4588d731b7 Date: 2015-03-23 18:17 +0200 http://bitbucket.org/pypy/pypy/changeset/9c4588d731b7/ Log: merge default into branch diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -12,3 +12,7 @@ 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1 10f1b29a2bd21f837090286174a9ca030b8680b2 release-2.5.0 8e24dac0b8e2db30d46d59f2c4daa3d4aaab7861 release-2.5.1 +8e24dac0b8e2db30d46d59f2c4daa3d4aaab7861 release-2.5.1 +0000000000000000000000000000000000000000 release-2.5.1 +0000000000000000000000000000000000000000 release-2.5.1 +e3d046c43451403f5969580fc1c41d5df6c4082a release-2.5.1 diff --git a/pypy/doc/stm.rst b/pypy/doc/stm.rst --- a/pypy/doc/stm.rst +++ b/pypy/doc/stm.rst @@ -174,6 +174,38 @@ User Guide ========== +How to write multithreaded programs: the 10'000-feet view +--------------------------------------------------------- + +PyPy-STM offers two ways to write multithreaded programs: + +* the traditional way, using the ``thread`` or ``threading`` modules, + described first__. + +* using ``TransactionQueue``, described next__, as a way to hide the + low-level notion of threads. + +.. __: `Drop-in replacement`_ +.. __: `transaction.TransactionQueue`_ + +The issue with low-level threads are well known (particularly in other +languages that don't have GIL-based interpreters): memory corruption, +deadlocks, livelocks, and so on. There are alternative approaches to +dealing directly with threads, like OpenMP_. These approaches +typically enforce some structure on your code. ``TransactionQueue`` +is in part similar: your program needs to have "some chances" of +parallelization before you can apply it. But I believe that the scope +of applicability is much larger with ``TransactionQueue`` than with +other approaches. It usually works without forcing a complete +reorganization of your existing code, and it works on any Python +program which has got *latent* and *imperfect* parallelism. Ideally, +it only requires that the end programmer identifies where this +parallelism is likely to be found, and communicates it to the system +using a simple API. + +.. _OpenMP: http://en.wikipedia.org/wiki/OpenMP + + Drop-in replacement ------------------- @@ -196,31 +228,6 @@ order. -How to write multithreaded programs: the 10'000-feet view ---------------------------------------------------------- - -PyPy-STM offers two ways to write multithreaded programs: - -* the traditional way, using the ``thread`` or ``threading`` modules. - -* using ``TransactionQueue``, described next__, as a way to hide the - low-level notion of threads. - -.. __: `transaction.TransactionQueue`_ - -``TransactionQueue`` hides the hard multithreading-related issues that -we typically encounter when using low-level threads. This is not the -first alternative approach to avoid dealing with low-level threads; -for example, OpenMP_ is one. However, it is one of the first ones -which does not require the code to be organized in a particular -fashion. Instead, it works on any Python program which has got -*latent* and *imperfect* parallelism. Ideally, it only requires that -the end programmer identifies where this parallelism is likely to be -found, and communicates it to the system using a simple API. - -.. _OpenMP: http://en.wikipedia.org/wiki/OpenMP - - transaction.TransactionQueue ---------------------------- @@ -256,8 +263,9 @@ behavior did not change because we are using ``TransactionQueue``. All the calls still *appear* to execute in some serial order. -However, the performance typically does not increase out of the box. -In fact, it is likely to be worse at first. Typically, this is +A typical usage of ``TransactionQueue`` goes like that: at first, +the performance does not increase. +In fact, it is likely to be worse. Typically, this is indicated by the total CPU usage, which remains low (closer to 1 than N cores). First note that it is expected that the CPU usage should not go much higher than 1 in the JIT warm-up phase: you must run a @@ -282,9 +290,9 @@ because of the reason shown in the two independent single-entry tracebacks: one thread ran the line ``someobj.stuff = 5``, whereas another thread concurrently ran the line ``someobj.other = 10`` on the -same object. Two writes to the same object cause a conflict, which -aborts one of the two transactions. In the example above this -occurred 12412 times. +same object. These two writes are done to the same object. This +causes a conflict, which aborts one of the two transactions. In the +example above this occurred 12412 times. The two other conflict sources are ``STM_CONTENTION_INEVITABLE``, which means that two transactions both tried to do an external @@ -303,7 +311,7 @@ each transaction starts with sending data to a log file. You should refactor this case so that it occurs either near the end of the transaction (which can then mostly run in non-inevitable mode), or - even delegate it to a separate thread. + delegate it to a separate transaction or even a separate thread. * Writing to a list or a dictionary conflicts with any read from the same list or dictionary, even one done with a different key. For @@ -322,7 +330,7 @@ results is fine, use ``transaction.time()`` or ``transaction.clock()``. -* ``transaction.threadlocalproperty`` can be used as class-level:: +* ``transaction.threadlocalproperty`` can be used at class-level:: class Foo(object): # must be a new-style class! x = transaction.threadlocalproperty() @@ -342,11 +350,11 @@ threads, each running the transactions one after the other; such thread-local properties will have the value last stored in them in the same thread,, which may come from a random previous transaction. - ``threadlocalproperty`` is still useful to avoid conflicts from - cache-like data structures. + This means that ``threadlocalproperty`` is useful mainly to avoid + conflicts from cache-like data structures. Note that Python is a complicated language; there are a number of less -common cases that may cause conflict (of any type) where we might not +common cases that may cause conflict (of any kind) where we might not expect it at priori. In many of these cases it could be fixed; please report any case that you don't understand. (For example, so far, creating a weakref to an object requires attaching an auxiliary @@ -395,8 +403,8 @@ it likely that such a piece of code will eventually block all other threads anyway. -Note that if you want to experiment with ``atomic``, you may have to add -manually a transaction break just before the atomic block. This is +Note that if you want to experiment with ``atomic``, you may have to +manually add a transaction break just before the atomic block. This is because the boundaries of the block are not guaranteed to be the boundaries of the transaction: the latter is at least as big as the block, but may be bigger. Therefore, if you run a big atomic block, it @@ -522,7 +530,7 @@ where, say, all threads try to increase the same global counter and do nothing else). -However, using if the program requires longer transactions, it comes +However, if the program requires longer transactions, it comes with less obvious rules. The exact details may vary from version to version, too, until they are a bit more stabilized. Here is an overview. diff --git a/pypy/module/_csv/test/__init__.py b/pypy/module/_csv/test/__init__.py new file mode 100644 diff --git a/pypy/module/_io/test/__init__.py b/pypy/module/_io/test/__init__.py new file mode 100644 diff --git a/pypy/module/_multiprocessing/test/__init__.py b/pypy/module/_multiprocessing/test/__init__.py new file mode 100644 diff --git a/pypy/module/_ssl/test/__init__.py b/pypy/module/_ssl/test/__init__.py new file mode 100644 diff --git a/pypy/module/itertools/test/__init__.py b/pypy/module/itertools/test/__init__.py new file mode 100644 diff --git a/pypy/module/pwd/test/__init__.py b/pypy/module/pwd/test/__init__.py new file mode 100644 diff --git a/pypy/module/pyexpat/__init__.py b/pypy/module/pyexpat/__init__.py --- a/pypy/module/pyexpat/__init__.py +++ b/pypy/module/pyexpat/__init__.py @@ -39,8 +39,6 @@ 'error': 'space.fromcache(interp_pyexpat.Cache).w_error', '__version__': 'space.wrap("85819")', - 'EXPAT_VERSION': 'interp_pyexpat.get_expat_version(space)', - 'version_info': 'interp_pyexpat.get_expat_version_info(space)', } submodules = { @@ -53,3 +51,9 @@ 'XML_PARAM_ENTITY_PARSING_ALWAYS']: interpleveldefs[name] = 'space.wrap(interp_pyexpat.%s)' % (name,) + def startup(self, space): + from pypy.module.pyexpat import interp_pyexpat + w_ver = interp_pyexpat.get_expat_version(space) + space.setattr(self, space.wrap("EXPAT_VERSION"), w_ver) + w_ver = interp_pyexpat.get_expat_version_info(space) + space.setattr(self, space.wrap("version_info"), w_ver) diff --git a/pypy/module/select/test/__init__.py b/pypy/module/select/test/__init__.py new file mode 100644 diff --git a/pypy/module/struct/test/__init__.py b/pypy/module/struct/test/__init__.py new file mode 100644 diff --git a/rpython/doc/rtyper.rst b/rpython/doc/rtyper.rst --- a/rpython/doc/rtyper.rst +++ b/rpython/doc/rtyper.rst @@ -118,8 +118,7 @@ given this representation. The RTyper also computes a ``concretetype`` for Constants, to match the way they are used in the low-level operations (for example, ``int_add(x, 1)`` requires a ``Constant(1)`` with -``concretetype=Signed``, but an untyped ``add(x, 1)`` works with a -``Constant(1)`` that must actually be a PyObject at run-time). +``concretetype=Signed``). In addition to ``lowleveltype``, each Repr subclass provides a set of methods called ``rtype_op_xxx()`` which define how each high-level operation ``op_xxx`` @@ -306,14 +305,14 @@ ~~~~~~~~~~~~~ As in C, pointers provide the indirection needed to make a reference modifiable -or sharable. Pointers can only point to a structure, an array, a function -(see below) or a PyObject (see below). Pointers to primitive types, if needed, -must be done by pointing to a structure with a single field of the required -type. Pointer types are declared by:: +or sharable. Pointers can only point to a structure, an array or a function +(see below). Pointers to primitive types, if needed, must be done by pointing +to a structure with a single field of the required type. Pointer types are +declared by:: Ptr(TYPE) -At run-time, pointers to GC structures (GcStruct, GcArray and PyObject) hold a +At run-time, pointers to GC structures (GcStruct, GcArray) hold a reference to what they are pointing to. Pointers to non-GC structures that can go away when their container is deallocated (Struct, Array) must be handled with care: the bigger structure of which they are part of could be freed while @@ -356,22 +355,6 @@ :graph: the flow graph of the function. -The PyObject Type -~~~~~~~~~~~~~~~~~ - -This is a special type, for compatibility with CPython: it stands for a -structure compatible with PyObject. This is also a "container" type (thinking -about C, this is ``PyObject``, not ``PyObject*``), so it is usually manipulated -via a Ptr. A typed graph can still contain generic space operations (add, -getitem, etc.) provided they are applied on objects whose low-level type is -``Ptr(PyObject)``. In fact, code generators that support this should consider -that the default type of a variable, if none is specified, is ``Ptr(PyObject)``. -In this way, they can generate the correct code for fully-untyped flow graphs. - -The testing implementation allows you to "create" PyObjects by calling -``pyobjectptr(obj)``. - - Opaque Types ~~~~~~~~~~~~ diff --git a/rpython/jit/metainterp/jitprof.py b/rpython/jit/metainterp/jitprof.py --- a/rpython/jit/metainterp/jitprof.py +++ b/rpython/jit/metainterp/jitprof.py @@ -44,7 +44,10 @@ pass def get_counter(self, num): - return -1.0 + return 0 + + def get_times(self, num): + return 0.0 class Profiler(BaseProfiler): initialized = False @@ -109,6 +112,9 @@ return self.cpu.tracker.total_freed_bridges return self.counters[num] + def get_times(self, num): + return self.times[num] + def count_ops(self, opnum, kind=Counters.OPS): from rpython.jit.metainterp.resoperation import rop self.counters[kind] += 1 diff --git a/rpython/jit/metainterp/test/test_jitiface.py b/rpython/jit/metainterp/test/test_jitiface.py --- a/rpython/jit/metainterp/test/test_jitiface.py +++ b/rpython/jit/metainterp/test/test_jitiface.py @@ -5,7 +5,7 @@ from rpython.jit.codewriter.policy import JitPolicy from rpython.jit.metainterp.resoperation import rop from rpython.rtyper.annlowlevel import hlstr -from rpython.jit.metainterp.jitprof import Profiler +from rpython.jit.metainterp.jitprof import Profiler, EmptyProfiler class JitHookInterfaceTests(object): @@ -178,6 +178,20 @@ self.meta_interp(main, [], ProfilerClass=Profiler) + def test_get_stats_empty(self): + driver = JitDriver(greens = [], reds = ['i']) + def loop(i): + while i > 0: + driver.jit_merge_point(i=i) + i -= 1 + def main(): + loop(30) + assert jit_hooks.stats_get_counter_value(None, + Counters.TOTAL_COMPILED_LOOPS) == 0 + assert jit_hooks.stats_get_times_value(None, Counters.TRACING) == 0 + self.meta_interp(main, [], ProfilerClass=EmptyProfiler) + + class LLJitHookInterfaceTests(JitHookInterfaceTests): # use this for any backend, instead of the super class diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -52,6 +52,9 @@ # XXX total addressable size. Maybe by keeping some minimarkpage arenas # XXX pre-reserved, enough for a few nursery collections? What about # XXX raw-malloced memory? + +# XXX try merging old_objects_pointing_to_pinned into +# XXX old_objects_pointing_to_young (IRC 2014-10-22, fijal and gregor_w) import sys from rpython.rtyper.lltypesystem import lltype, llmemory, llarena, llgroup from rpython.rtyper.lltypesystem.lloperation import llop @@ -63,6 +66,7 @@ from rpython.rlib.rarithmetic import LONG_BIT_SHIFT from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop from rpython.rlib.objectmodel import specialize +from rpython.memory.gc.minimarkpage import out_of_memory # # Handles the objects in 2 generations: @@ -471,10 +475,10 @@ # the start of the nursery: we actually allocate a bit more for # the nursery than really needed, to simplify pointer arithmetic # in malloc_fixedsize(). The few extra pages are never used - # anyway so it doesn't even counct. + # anyway so it doesn't even count. nursery = llarena.arena_malloc(self._nursery_memory_size(), 0) if not nursery: - raise MemoryError("cannot allocate nursery") + out_of_memory("cannot allocate nursery") return nursery def allocate_nursery(self): @@ -685,23 +689,48 @@ def collect_and_reserve(self, totalsize): """To call when nursery_free overflows nursery_top. - First check if the nursery_top is the real top, otherwise we - can just move the top of one cleanup and continue - - Do a minor collection, and possibly also a major collection, - and finally reserve 'totalsize' bytes at the start of the - now-empty nursery. + First check if pinned objects are in front of nursery_top. If so, + jump over the pinned object and try again to reserve totalsize. + Otherwise do a minor collection, and possibly a major collection, and + finally reserve totalsize bytes. """ minor_collection_count = 0 while True: self.nursery_free = llmemory.NULL # debug: don't use me + # note: no "raise MemoryError" between here and the next time + # we initialize nursery_free! if self.nursery_barriers.non_empty(): + # Pinned object in front of nursery_top. Try reserving totalsize + # by jumping into the next, yet unused, area inside the + # nursery. "Next area" in this case is the space between the + # pinned object in front of nusery_top and the pinned object + # after that. Graphically explained: + # + # |- allocating totalsize failed in this area + # | |- nursery_top + # | | |- pinned object in front of nursery_top, + # v v v jump over this + # +---------+--------+--------+--------+-----------+ } + # | used | pinned | empty | pinned | empty | }- nursery + # +---------+--------+--------+--------+-----------+ } + # ^- try reserving totalsize in here next + # + # All pinned objects are represented by entries in + # nursery_barriers (see minor_collection). The last entry is + # always the end of the nursery. Therefore if nursery_barriers + # contains only one element, we jump over a pinned object and + # the "next area" (the space where we will try to allocate + # totalsize) starts at the end of the pinned object and ends at + # nursery's end. + # + # find the size of the pinned object after nursery_top size_gc_header = self.gcheaderbuilder.size_gc_header pinned_obj_size = size_gc_header + self.get_size( self.nursery_top + size_gc_header) - + # + # update used nursery space to allocate objects self.nursery_free = self.nursery_top + pinned_obj_size self.nursery_top = self.nursery_barriers.popleft() else: @@ -729,6 +758,9 @@ "Seeing minor_collection() at least twice." "Too many pinned objects?") # + # Tried to do something about nursery_free overflowing + # nursery_top before this point. Try to reserve totalsize now. + # If this succeeds break out of loop. result = self.nursery_free if self.nursery_free + totalsize <= self.nursery_top: self.nursery_free = result + totalsize @@ -1491,7 +1523,7 @@ # being moved, not from being collected if it is not reachable anymore. self.surviving_pinned_objects = self.AddressStack() # The following counter keeps track of alive and pinned young objects - # inside the nursery. We reset it here and increace it in + # inside the nursery. We reset it here and increase it in # '_trace_drag_out()'. any_pinned_object_from_earlier = self.any_pinned_object_kept self.pinned_objects_in_nursery = 0 @@ -1625,7 +1657,9 @@ else: llarena.arena_reset(prev, self.nursery + self.nursery_size - prev, 0) # + # always add the end of the nursery to the list nursery_barriers.append(self.nursery + self.nursery_size) + # self.nursery_barriers = nursery_barriers self.surviving_pinned_objects.delete() # @@ -1950,7 +1984,7 @@ # arena = llarena.arena_malloc(raw_malloc_usage(totalsize), False) if not arena: - raise MemoryError("cannot allocate object") + out_of_memory("out of memory: couldn't allocate a few KB more") llarena.arena_reserve(arena, totalsize) # size_gc_header = self.gcheaderbuilder.size_gc_header @@ -2058,7 +2092,7 @@ # XXX A simplifying assumption that should be checked, # finalizers/weak references are rare and short which means that - # they do not need a seperate state and do not need to be + # they do not need a separate state and do not need to be # made incremental. if (not self.objects_to_trace.non_empty() and not self.more_objects_to_trace.non_empty()): @@ -2148,9 +2182,9 @@ # even higher memory consumption. To prevent it, if it's # the second time we are here, then abort the program. if self.max_heap_size_already_raised: - llop.debug_fatalerror(lltype.Void, - "Using too much memory, aborting") + out_of_memory("using too much memory, aborting") self.max_heap_size_already_raised = True + self.gc_state = STATE_SCANNING raise MemoryError self.gc_state = STATE_FINALIZING diff --git a/rpython/memory/gc/minimarkpage.py b/rpython/memory/gc/minimarkpage.py --- a/rpython/memory/gc/minimarkpage.py +++ b/rpython/memory/gc/minimarkpage.py @@ -2,7 +2,7 @@ from rpython.rtyper.lltypesystem import lltype, llmemory, llarena, rffi from rpython.rlib.rarithmetic import LONG_BIT, r_uint from rpython.rlib.objectmodel import we_are_translated -from rpython.rlib.debug import ll_assert +from rpython.rlib.debug import ll_assert, fatalerror WORD = LONG_BIT // 8 NULL = llmemory.NULL @@ -294,7 +294,7 @@ # be a page-aligned address arena_base = llarena.arena_malloc(self.arena_size, False) if not arena_base: - raise MemoryError("couldn't allocate the next arena") + out_of_memory("out of memory: couldn't allocate the next arena") arena_end = arena_base + self.arena_size # # 'firstpage' points to the first unused page @@ -593,3 +593,10 @@ if isinstance(size, int): size = llmemory.sizeof(lltype.Char) * size return size + +def out_of_memory(errmsg): + """Signal a fatal out-of-memory error and abort. For situations where + it is hard to write and test code that would handle a MemoryError + exception gracefully. + """ + fatalerror(errmsg) diff --git a/rpython/memory/gctransform/asmgcroot.py b/rpython/memory/gctransform/asmgcroot.py --- a/rpython/memory/gctransform/asmgcroot.py +++ b/rpython/memory/gctransform/asmgcroot.py @@ -368,6 +368,13 @@ if rpy_fastgil != 1: ll_assert(rpy_fastgil != 0, "walk_stack_from doesn't have the GIL") initialframedata = rffi.cast(llmemory.Address, rpy_fastgil) + # + # very rare issue: initialframedata.address[0] is uninitialized + # in this case, but "retaddr = callee.frame_address.address[0]" + # reads it. If it happens to be exactly a valid return address + # inside the C code, then bad things occur. + initialframedata.address[0] = llmemory.NULL + # self.walk_frames(curframe, otherframe, initialframedata) stackscount += 1 # @@ -519,17 +526,15 @@ from rpython.jit.backend.llsupport.jitframe import STACK_DEPTH_OFS tid = self.gc.get_possibly_forwarded_type_id(ebp_in_caller) - ll_assert(rffi.cast(lltype.Signed, tid) == - rffi.cast(lltype.Signed, self.frame_tid), - "found a stack frame that does not belong " - "anywhere I know, bug in asmgcc") - # fish the depth - extra_stack_depth = (ebp_in_caller + STACK_DEPTH_OFS).signed[0] - ll_assert((extra_stack_depth & (rffi.sizeof(lltype.Signed) - 1)) - == 0, "asmgcc: misaligned extra_stack_depth") - extra_stack_depth //= rffi.sizeof(lltype.Signed) - self._shape_decompressor.setjitframe(extra_stack_depth) - return + if (rffi.cast(lltype.Signed, tid) == + rffi.cast(lltype.Signed, self.frame_tid)): + # fish the depth + extra_stack_depth = (ebp_in_caller + STACK_DEPTH_OFS).signed[0] + ll_assert((extra_stack_depth & (rffi.sizeof(lltype.Signed) - 1)) + == 0, "asmgcc: misaligned extra_stack_depth") + extra_stack_depth //= rffi.sizeof(lltype.Signed) + self._shape_decompressor.setjitframe(extra_stack_depth) + return llop.debug_fatalerror(lltype.Void, "cannot find gc roots!") def getlocation(self, callee, ebp_in_caller, location): diff --git a/rpython/rlib/jit_hooks.py b/rpython/rlib/jit_hooks.py --- a/rpython/rlib/jit_hooks.py +++ b/rpython/rlib/jit_hooks.py @@ -130,7 +130,7 @@ @register_helper(annmodel.SomeFloat()) def stats_get_times_value(warmrunnerdesc, no): - return warmrunnerdesc.metainterp_sd.profiler.times[no] + return warmrunnerdesc.metainterp_sd.profiler.get_times(no) LOOP_RUN_CONTAINER = lltype.GcArray(lltype.Struct('elem', ('type', lltype.Char), diff --git a/rpython/rtyper/lltypesystem/rlist.py b/rpython/rtyper/lltypesystem/rlist.py --- a/rpython/rtyper/lltypesystem/rlist.py +++ b/rpython/rtyper/lltypesystem/rlist.py @@ -1,6 +1,7 @@ from rpython.rlib import rgc, jit, types from rpython.rlib.debug import ll_assert from rpython.rlib.signature import signature +from rpython.rtyper.error import TyperError from rpython.rtyper.lltypesystem import rstr from rpython.rtyper.lltypesystem.lltype import (GcForwardReference, Ptr, GcArray, GcStruct, Void, Signed, malloc, typeOf, nullptr, typeMethod) @@ -57,7 +58,7 @@ elif variant == ("reversed",): return ReversedListIteratorRepr(self) else: - raise NotImplementedError(variant) + raise TyperError("unsupported %r iterator over a list" % (variant,)) def get_itemarray_lowleveltype(self): ITEM = self.item_repr.lowleveltype diff --git a/rpython/rtyper/lltypesystem/rrange.py b/rpython/rtyper/lltypesystem/rrange.py --- a/rpython/rtyper/lltypesystem/rrange.py +++ b/rpython/rtyper/lltypesystem/rrange.py @@ -1,5 +1,6 @@ from rpython.rtyper.lltypesystem.lltype import Ptr, GcStruct, Signed, malloc, Void from rpython.rtyper.rrange import AbstractRangeRepr, AbstractRangeIteratorRepr +from rpython.rtyper.error import TyperError # ____________________________________________________________ # @@ -59,7 +60,10 @@ self.ll_newrange = ll_newrange self.ll_newrangest = ll_newrangest - def make_iterator_repr(self): + def make_iterator_repr(self, variant=None): + if variant is not None: + raise TyperError("unsupported %r iterator over a range list" % + (variant,)) return RangeIteratorRepr(self) diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py --- a/rpython/rtyper/lltypesystem/rstr.py +++ b/rpython/rtyper/lltypesystem/rstr.py @@ -156,7 +156,10 @@ self.CACHE[value] = p return p - def make_iterator_repr(self): + def make_iterator_repr(self, variant=None): + if variant is not None: + raise TyperError("unsupported %r iterator over a str/unicode" % + (variant,)) return self.repr.iterator_repr def can_ll_be_null(self, s_value): diff --git a/rpython/rtyper/rtuple.py b/rpython/rtyper/rtuple.py --- a/rpython/rtyper/rtuple.py +++ b/rpython/rtyper/rtuple.py @@ -210,7 +210,10 @@ ll_str = property(gen_str_function) - def make_iterator_repr(self): + def make_iterator_repr(self, variant=None): + if variant is not None: + raise TyperError("unsupported %r iterator over a tuple" % + (variant,)) if len(self.items_r) == 1: # subclasses are supposed to set the IteratorRepr attribute return self.IteratorRepr(self) diff --git a/rpython/rtyper/test/test_rstr.py b/rpython/rtyper/test/test_rstr.py --- a/rpython/rtyper/test/test_rstr.py +++ b/rpython/rtyper/test/test_rstr.py @@ -116,6 +116,16 @@ res = self.interpret(fn, [1]) assert res == 1 + ord('a') + 10000 + def test_str_iterator_reversed_unsupported(self): + const = self.const + def fn(): + total = 0 + t = const('foo') + for x in reversed(t): + total += ord(x) + return total + py.test.raises(TyperError, self.interpret, fn, []) + def test_char_constant(self): const = self.const def fn(s): diff --git a/rpython/rtyper/test/test_rtuple.py b/rpython/rtyper/test/test_rtuple.py --- a/rpython/rtyper/test/test_rtuple.py +++ b/rpython/rtyper/test/test_rtuple.py @@ -1,8 +1,10 @@ +import py from rpython.rtyper.rtuple import TUPLE_TYPE, TupleRepr from rpython.rtyper.lltypesystem.lltype import Signed, Bool from rpython.rtyper.rbool import bool_repr from rpython.rtyper.rint import signed_repr from rpython.rtyper.test.tool import BaseRtypingTest +from rpython.rtyper.error import TyperError from rpython.rlib.objectmodel import compute_hash from rpython.translator.translator import TranslationContext @@ -228,6 +230,15 @@ res = self.interpret(f, [93813]) assert res == 93813 + def test_tuple_iterator_reversed_unsupported(self): + def f(i): + total = 0 + t = (i,) + for x in reversed(t): + total += x + return total + py.test.raises(TyperError, self.interpret, f, [93813]) + def test_inst_tuple_iter(self): class A: pass diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py --- a/rpython/translator/c/test/test_newgc.py +++ b/rpython/translator/c/test/test_newgc.py @@ -2,6 +2,7 @@ import inspect import os import sys +import subprocess import py @@ -50,8 +51,8 @@ t.viewcg() exename = t.compile() - def run(s, i): - data = py.process.cmdexec("%s %s %d" % (exename, s, i)) + def run(s, i, runner=subprocess.check_output): + data = runner([str(exename), str(s), str(i)]) data = data.strip() if data == 'MEMORY-ERROR': raise MemoryError @@ -115,11 +116,11 @@ cls.c_allfuncs.close_isolate() cls.c_allfuncs = None - def run(self, name, *args): + def run(self, name, *args, **kwds): if not args: args = (-1, ) print 'Running %r)' % name - res = self.c_allfuncs(name, *args) + res = self.c_allfuncs(name, *args, **kwds) num = self.name_to_func[name] if self.funcsstr[num]: return res @@ -1524,6 +1525,38 @@ res = self.run("nongc_opaque_attached_to_gc") assert res == 0 + def define_limited_memory(self): + class A(object): + def __init__(self, next): + self.next = next + def g(i): + a = None + while i != 0: + a = A(a) + i -= 1 + return 0 + def f(i): + try: + return g(i) + except MemoryError: + g(20) # allocate some more stuff, but exit + return 42 + return f + + def test_limited_memory(self): + import random + # + def myrunner(args): + env = os.environ.copy() + env['PYPY_GC_MAX'] = '%dKB' % gcmax + return subprocess.check_output(args, env=env) + # + for i in range(10): + gcmax = random.randrange(50000, 100000) + print gcmax + res = self.run("limited_memory", -1, runner=myrunner) + assert res == 42 + class TestIncrementalMiniMarkGC(TestMiniMarkGC): gcpolicy = "incminimark" @@ -1577,6 +1610,29 @@ res = self.run("random_pin") assert res == 28495 + define_limited_memory_linux = TestMiniMarkGC.define_limited_memory.im_func + + def test_limited_memory_linux(self): + if not sys.platform.startswith('linux'): + py.test.skip("linux only") + # + import random + # + def myrunner(args): + args1 = ['/bin/bash', '-c', 'ulimit -v %d && %s' % + (ulimitv, ' '.join(args),)] + popen = subprocess.Popen(args1, stderr=subprocess.PIPE) + _, child_stderr = popen.communicate() + assert popen.wait() == 134 # aborted + assert 'out of memory:' in child_stderr + return '42' + # + for i in range(10): + ulimitv = random.randrange(50000, 100000) + print ulimitv + res = self.run("limited_memory_linux", -1, runner=myrunner) + assert res == 42 + # ____________________________________________________________________ diff --git a/rpython/translator/platform/darwin.py b/rpython/translator/platform/darwin.py --- a/rpython/translator/platform/darwin.py +++ b/rpython/translator/platform/darwin.py @@ -12,6 +12,12 @@ DEFAULT_CC = 'clang' rpath_flags = ['-Wl,-rpath', '-Wl, at executable_path/'] + def get_rpath_flags(self, rel_libdirs): + # needed for cross compiling on ARM, needs fixing if relevant for darwin + if len(rel_libdirs) > 0: + print 'in get_rpath_flags, rel_libdirs is not fixed up',rel_libdirs + return self.rpath_flags + def _args_for_shared(self, args): return (list(self.shared_only) + ['-dynamiclib', '-install_name', '@rpath/$(TARGET)', '-undefined', 'dynamic_lookup'] diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py --- a/rpython/translator/platform/posix.py +++ b/rpython/translator/platform/posix.py @@ -89,6 +89,11 @@ raise ValueError(msg) return result + def get_rpath_flags(self, rel_libdirs): + # needed for cross-compilation i.e. ARM + return self.rpath_flags + ['-Wl,-rpath-link=\'%s\'' % ldir + for ldir in rel_libdirs] + def get_shared_only_compile_flags(self): return tuple(self.shared_only) + ('-fvisibility=hidden',) @@ -167,8 +172,7 @@ ('CC', self.cc), ('CC_LINK', eci.use_cpp_linker and 'g++' or '$(CC)'), ('LINKFILES', eci.link_files), - ('RPATH_FLAGS', self.rpath_flags + ['-Wl,-rpath-link=\'%s\'' % ldir - for ldir in rel_libdirs]), + ('RPATH_FLAGS', self.get_rpath_flags(rel_libdirs)), ] for args in definitions: m.definition(*args) diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py --- a/rpython/translator/simplify.py +++ b/rpython/translator/simplify.py @@ -334,25 +334,25 @@ flowcontext.py). """ for block in list(graph.iterblocks()): - for i in range(len(block.exits)-1, -1, -1): - exit = block.exits[i] - if not (exit.target is graph.exceptblock and - exit.args[0] == Constant(AssertionError)): - continue - # can we remove this exit without breaking the graph? - if len(block.exits) < 2: + for i in range(len(block.exits)-1, -1, -1): + exit = block.exits[i] + if not (exit.target is graph.exceptblock and + exit.args[0] == Constant(AssertionError)): + continue + # can we remove this exit without breaking the graph? + if len(block.exits) < 2: + break + if block.canraise: + if exit.exitcase is None: break - if block.canraise: - if exit.exitcase is None: - break - if len(block.exits) == 2: - # removing the last non-exceptional exit - block.exitswitch = None - exit.exitcase = None - # remove this exit - lst = list(block.exits) - del lst[i] - block.recloseblock(*lst) + if len(block.exits) == 2: + # removing the last non-exceptional exit + block.exitswitch = None + exit.exitcase = None + # remove this exit + lst = list(block.exits) + del lst[i] + block.recloseblock(*lst) # _____________________________________________________________________ From noreply at buildbot.pypy.org Mon Mar 23 23:42:12 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 23 Mar 2015 23:42:12 +0100 (CET) Subject: [pypy-commit] pypy default: skip this test here, for now Message-ID: <20150323224212.A53081C0994@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76538:59755d05d223 Date: 2015-03-23 23:42 +0100 http://bitbucket.org/pypy/pypy/changeset/59755d05d223/ Log: skip this test here, for now diff --git a/rpython/translator/c/gcc/test/test_asmgcroot.py b/rpython/translator/c/gcc/test/test_asmgcroot.py --- a/rpython/translator/c/gcc/test/test_asmgcroot.py +++ b/rpython/translator/c/gcc/test/test_asmgcroot.py @@ -65,7 +65,9 @@ t.view() exe_name = cbuilder.compile() - def run(arg0, arg1): + def run(arg0, arg1, runner=None): + if runner is not None: + py.test.skip("unsupported test: runner=%r" % (runner,)) lines = [] print >> sys.stderr, 'RUN: starting', exe_name, arg0, arg1 if sys.platform == 'win32': From noreply at buildbot.pypy.org Mon Mar 23 23:51:46 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 23 Mar 2015 23:51:46 +0100 (CET) Subject: [pypy-commit] pypy default: use a more official interface Message-ID: <20150323225146.C4E871C0994@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76539:cd4b2cdca734 Date: 2015-03-23 23:51 +0100 http://bitbucket.org/pypy/pypy/changeset/cd4b2cdca734/ Log: use a more official interface diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py --- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py +++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py @@ -62,7 +62,7 @@ stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = pipe.communicate() - if getattr(pipe, 'returncode', 0) < 0: + if pipe.wait() < 0: raise IOError("subprocess was killed by signal %d" % ( pipe.returncode,)) if stderr.startswith('SKIP:'): From noreply at buildbot.pypy.org Tue Mar 24 19:01:50 2015 From: noreply at buildbot.pypy.org (antocuni) Date: Tue, 24 Mar 2015 19:01:50 +0100 (CET) Subject: [pypy-commit] pypy vmprof: disable vmprof around fork(), and re-enable it only in the parent process Message-ID: <20150324180150.C8F441C0F32@cobra.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: vmprof Changeset: r76540:78914547f680 Date: 2015-03-24 19:01 +0100 http://bitbucket.org/pypy/pypy/changeset/78914547f680/ Log: disable vmprof around fork(), and re-enable it only in the parent process diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c --- a/pypy/module/_vmprof/src/vmprof.c +++ b/pypy/module/_vmprof/src/vmprof.c @@ -26,6 +26,7 @@ #include #include #include +#include #define UNW_LOCAL_ONLY #include @@ -44,6 +45,8 @@ void* vmprof_mainloop_func; static ptrdiff_t mainloop_sp_offset; static vmprof_get_virtual_ip_t mainloop_get_virtual_ip; +static long last_period_usec = 0; +static int atfork_hook_installed = 0; /* ************************************************************* @@ -275,6 +278,7 @@ static int install_sigprof_timer(long period_usec) { static struct itimerval timer; + last_period_usec = period_usec; timer.it_interval.tv_sec = 0; timer.it_interval.tv_usec = period_usec; timer.it_value = timer.it_interval; @@ -295,6 +299,34 @@ return 0; } +static void atfork_disable_timer(void) { + remove_sigprof_timer(); +} + +static void atfork_enable_timer(void) { + install_sigprof_timer(last_period_usec); +} + +static int install_pthread_atfork_hooks(void) { + /* this is needed to prevent the problems described there: + - http://code.google.com/p/gperftools/issues/detail?id=278 + - http://lists.debian.org/debian-glibc/2010/03/msg00161.html + + TL;DR: if the RSS of the process is large enough, the clone() syscall + will be interrupted by the SIGPROF before it can complete, then + retried, interrupted again and so on, in an endless loop. The + solution is to disable the timer around the fork, and re-enable it + only inside the parent. + */ + if (atfork_hook_installed) + return 0; + int ret = pthread_atfork(atfork_disable_timer, atfork_enable_timer, NULL); + if (ret != 0) + return -1; + atfork_hook_installed = 1; + return 0; +} + /* ************************************************************* * public API * ************************************************************* @@ -321,6 +353,9 @@ if (install_sigprof_timer(period_usec) == -1) { return -1; } + if (install_pthread_atfork_hooks() == -1) { + return -1; + } return 0; } From noreply at buildbot.pypy.org Tue Mar 24 19:22:29 2015 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 24 Mar 2015 19:22:29 +0100 (CET) Subject: [pypy-commit] pypy default: Removed tag release-2.5.1 Message-ID: <20150324182229.AEF9C1C11DD@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r76541:a9a830775900 Date: 2015-03-24 20:23 +0200 http://bitbucket.org/pypy/pypy/changeset/a9a830775900/ Log: Removed tag release-2.5.1 diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -16,3 +16,5 @@ 0000000000000000000000000000000000000000 release-2.5.1 0000000000000000000000000000000000000000 release-2.5.1 e3d046c43451403f5969580fc1c41d5df6c4082a release-2.5.1 +e3d046c43451403f5969580fc1c41d5df6c4082a release-2.5.1 +0000000000000000000000000000000000000000 release-2.5.1 From noreply at buildbot.pypy.org Tue Mar 24 19:22:30 2015 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 24 Mar 2015 19:22:30 +0100 (CET) Subject: [pypy-commit] pypy default: Added tag release-2.5.1 for changeset 9c4588d731b7 Message-ID: <20150324182230.EAB951C11DD@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r76542:034d8cc89696 Date: 2015-03-24 20:23 +0200 http://bitbucket.org/pypy/pypy/changeset/034d8cc89696/ Log: Added tag release-2.5.1 for changeset 9c4588d731b7 diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -18,3 +18,5 @@ e3d046c43451403f5969580fc1c41d5df6c4082a release-2.5.1 e3d046c43451403f5969580fc1c41d5df6c4082a release-2.5.1 0000000000000000000000000000000000000000 release-2.5.1 +0000000000000000000000000000000000000000 release-2.5.1 +9c4588d731b7fe0b08669bd732c2b676cb0a8233 release-2.5.1 From noreply at buildbot.pypy.org Tue Mar 24 20:11:42 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Tue, 24 Mar 2015 20:11:42 +0100 (CET) Subject: [pypy-commit] pypy default: Add 2 tests for patterns found in PyPy code Message-ID: <20150324191142.B83901C043B@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r76543:84912eddc058 Date: 2015-03-24 19:12 +0000 http://bitbucket.org/pypy/pypy/changeset/84912eddc058/ Log: Add 2 tests for patterns found in PyPy code diff --git a/rpython/translator/c/test/test_exception.py b/rpython/translator/c/test/test_exception.py --- a/rpython/translator/c/test/test_exception.py +++ b/rpython/translator/c/test/test_exception.py @@ -3,6 +3,7 @@ from rpython.translator.c.test import test_typed from rpython.translator.c.test import test_backendoptimized from rpython.rtyper.lltypesystem import lltype +from rpython.rlib.rarithmetic import ovfcheck getcompiled = test_typed.TestTypedTestCase().getcompiled getcompiledopt = test_backendoptimized.TestTypedOptimizedTestCase().getcompiled @@ -28,9 +29,9 @@ b = raise_(i) + 12 c = raise_(i) + 13 return a+b+c - except TestException: + except TestException: return 7 - except MyException: + except MyException: return 123 except: return 22 @@ -173,3 +174,41 @@ f1 = getcompiledopt(fn, [int]) res = f1(100) assert res == 42 + +def test_getitem_custom_exception(): + class MyError(Exception): + pass + class BadContainer(object): + def __getitem__(self, n): + raise MyError + def f(): + d = BadContainer() + try: + return d[0] + except KeyError: + return 1 + def g(): + try: + return f() + except MyError: + return -1 + + assert g() == -1 + compiled = getcompiled(g, []) + assert compiled() == -1 + +def test_ovf_propagation(): + def div(a, b): + try: + return ovfcheck(a//b) + except ZeroDivisionError: + raise + def f(): + div(4, 2) + try: + return div(-sys.maxint-1, -1) + except OverflowError: + return 0 + assert f() == 0 + compiled = getcompiled(f, []) + assert compiled() == 0 From noreply at buildbot.pypy.org Tue Mar 24 20:53:25 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 24 Mar 2015 20:53:25 +0100 (CET) Subject: [pypy-commit] pypy default: Mention recursion detection Message-ID: <20150324195325.7A2611C0994@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76544:a450cfab98bf Date: 2015-03-24 20:52 +0100 http://bitbucket.org/pypy/pypy/changeset/a450cfab98bf/ Log: Mention recursion detection diff --git a/pypy/doc/stm.rst b/pypy/doc/stm.rst --- a/pypy/doc/stm.rst +++ b/pypy/doc/stm.rst @@ -147,6 +147,9 @@ crash on an assertion error because of a non-implemented overflow of an internal 28-bit counter. +* The recursion detection code was not reimplemented. Infinite + recursion just segfaults for now. + .. _`report any crash`: https://bitbucket.org/pypy/pypy/issues?status=new&status=open .. __: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/rpython/translator/stm/src_stm/stm/core.h From noreply at buildbot.pypy.org Tue Mar 24 20:53:26 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 24 Mar 2015 20:53:26 +0100 (CET) Subject: [pypy-commit] pypy default: Mention what pypy-stm is *not* good for Message-ID: <20150324195326.A59B51C0994@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76545:68f15684fb4d Date: 2015-03-24 20:53 +0100 http://bitbucket.org/pypy/pypy/changeset/68f15684fb4d/ Log: Mention what pypy-stm is *not* good for diff --git a/pypy/doc/stm.rst b/pypy/doc/stm.rst --- a/pypy/doc/stm.rst +++ b/pypy/doc/stm.rst @@ -25,8 +25,8 @@ .. _`2nd call for donation`: http://pypy.org/tmdonate2.html -Introduction -============ +What pypy-stm is for +==================== ``pypy-stm`` is a variant of the regular PyPy interpreter. (This version supports Python 2.7; see below for `Python 3`_.) With caveats_ @@ -55,6 +55,27 @@ and `transaction.TransactionQueue`_. +...and what pypy-stm is not for +------------------------------- + +``pypy-stm`` gives a Python without the GIL. This means that it is +useful in situations where the GIL is the problem in the first place. +(This includes cases where the program can easily be modified to run +in multiple threads; often, we don't consider doing that precisely +because of the GIL.) + +However, there are plenty of cases where the GIL is not the problem. +Do not hope ``pypy-stm`` to be helpful in these cases! This includes +all programs that use multiple threads but don't actually spend a lot +of time running Python code. For example, it may be spending all its +time waiting for I/O to occur, or performing some long computation on +a huge matrix. These are cases where the CPU is either idle, or in +some C/Fortran library anyway; in both cases, the interpreter (either +CPython or the regular PyPy) should release the GIL around the +external calls. The threads will thus not end up fighting for the +GIL. + + Getting Started =============== From noreply at buildbot.pypy.org Tue Mar 24 20:53:27 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 24 Mar 2015 20:53:27 +0100 (CET) Subject: [pypy-commit] pypy default: merge heads Message-ID: <20150324195327.C28AB1C0994@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76546:67167636348c Date: 2015-03-24 20:53 +0100 http://bitbucket.org/pypy/pypy/changeset/67167636348c/ Log: merge heads diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -16,3 +16,7 @@ 0000000000000000000000000000000000000000 release-2.5.1 0000000000000000000000000000000000000000 release-2.5.1 e3d046c43451403f5969580fc1c41d5df6c4082a release-2.5.1 +e3d046c43451403f5969580fc1c41d5df6c4082a release-2.5.1 +0000000000000000000000000000000000000000 release-2.5.1 +0000000000000000000000000000000000000000 release-2.5.1 +9c4588d731b7fe0b08669bd732c2b676cb0a8233 release-2.5.1 diff --git a/rpython/translator/c/test/test_exception.py b/rpython/translator/c/test/test_exception.py --- a/rpython/translator/c/test/test_exception.py +++ b/rpython/translator/c/test/test_exception.py @@ -3,6 +3,7 @@ from rpython.translator.c.test import test_typed from rpython.translator.c.test import test_backendoptimized from rpython.rtyper.lltypesystem import lltype +from rpython.rlib.rarithmetic import ovfcheck getcompiled = test_typed.TestTypedTestCase().getcompiled getcompiledopt = test_backendoptimized.TestTypedOptimizedTestCase().getcompiled @@ -28,9 +29,9 @@ b = raise_(i) + 12 c = raise_(i) + 13 return a+b+c - except TestException: + except TestException: return 7 - except MyException: + except MyException: return 123 except: return 22 @@ -173,3 +174,41 @@ f1 = getcompiledopt(fn, [int]) res = f1(100) assert res == 42 + +def test_getitem_custom_exception(): + class MyError(Exception): + pass + class BadContainer(object): + def __getitem__(self, n): + raise MyError + def f(): + d = BadContainer() + try: + return d[0] + except KeyError: + return 1 + def g(): + try: + return f() + except MyError: + return -1 + + assert g() == -1 + compiled = getcompiled(g, []) + assert compiled() == -1 + +def test_ovf_propagation(): + def div(a, b): + try: + return ovfcheck(a//b) + except ZeroDivisionError: + raise + def f(): + div(4, 2) + try: + return div(-sys.maxint-1, -1) + except OverflowError: + return 0 + assert f() == 0 + compiled = getcompiled(f, []) + assert compiled() == 0 From noreply at buildbot.pypy.org Tue Mar 24 20:55:52 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 24 Mar 2015 20:55:52 +0100 (CET) Subject: [pypy-commit] pypy default: plural :-) Message-ID: <20150324195552.DF18D1C0994@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76547:245b1e2daf67 Date: 2015-03-24 20:55 +0100 http://bitbucket.org/pypy/pypy/changeset/245b1e2daf67/ Log: plural :-) diff --git a/pypy/doc/stm.rst b/pypy/doc/stm.rst --- a/pypy/doc/stm.rst +++ b/pypy/doc/stm.rst @@ -212,7 +212,7 @@ .. __: `Drop-in replacement`_ .. __: `transaction.TransactionQueue`_ -The issue with low-level threads are well known (particularly in other +The issues with low-level threads are well known (particularly in other languages that don't have GIL-based interpreters): memory corruption, deadlocks, livelocks, and so on. There are alternative approaches to dealing directly with threads, like OpenMP_. These approaches From noreply at buildbot.pypy.org Tue Mar 24 21:05:21 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 24 Mar 2015 21:05:21 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: in-progress Message-ID: <20150324200521.582491C0994@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76548:367f2524c4f1 Date: 2015-03-24 18:48 +0100 http://bitbucket.org/pypy/pypy/changeset/367f2524c4f1/ Log: in-progress diff --git a/rpython/jit/backend/llsupport/test/zrpy_gc_test.py b/rpython/jit/backend/llsupport/test/zrpy_gc_test.py --- a/rpython/jit/backend/llsupport/test/zrpy_gc_test.py +++ b/rpython/jit/backend/llsupport/test/zrpy_gc_test.py @@ -97,7 +97,7 @@ from rpython.translator.c import genc # t = TranslationContext() - gcrootfinder = kwds['gcrootfinder'] + gcrootfinder = kwds.get('gcrootfinder') if gcrootfinder == 'stm': t.config.translation.stm = True t.config.translation.gc = 'stmgc' From noreply at buildbot.pypy.org Tue Mar 24 21:05:22 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 24 Mar 2015 21:05:22 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Turn inevitable when changing a thread-local value. This is needed because otherwise, the thread shutdown logic write NULL as the current ExecutionContext, but then if the transaction aborts, we're back in time with still NULL... Message-ID: <20150324200522.9460B1C0994@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76549:e466895ddd4c Date: 2015-03-24 18:49 +0100 http://bitbucket.org/pypy/pypy/changeset/e466895ddd4c/ Log: Turn inevitable when changing a thread-local value. This is needed because otherwise, the thread shutdown logic write NULL as the current ExecutionContext, but then if the transaction aborts, we're back in time with still NULL... diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py --- a/rpython/rlib/rthread.py +++ b/rpython/rlib/rthread.py @@ -332,6 +332,8 @@ _threadlocalref_seeme(self) addr = llop.threadlocalref_addr(llmemory.Address) if rgc.stm_is_enabled(): + from rpython.rlib import rstm + rstm.become_inevitable() p = llmemory.cast_adr_to_ptr(addr + offset, PSTRUCTTYPE) p.c_value = value else: From noreply at buildbot.pypy.org Tue Mar 24 21:05:23 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 24 Mar 2015 21:05:23 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: pom pom pom another try Message-ID: <20150324200523.BAAAD1C0994@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76550:93c0ee39bd3f Date: 2015-03-24 20:18 +0100 http://bitbucket.org/pypy/pypy/changeset/93c0ee39bd3f/ Log: pom pom pom another try diff --git a/pypy/module/pypystm/threadlocals.py b/pypy/module/pypystm/threadlocals.py --- a/pypy/module/pypystm/threadlocals.py +++ b/pypy/module/pypystm/threadlocals.py @@ -40,3 +40,7 @@ interval = space.actionflag.getcheckinterval() rstm.set_transaction_length(interval / 10000.0) + def leave_thread(self, space): + # must turn inevitable, for raw_thread_local.set(None) + rstm.become_inevitable() + OSThreadLocals.leave_thread(self, space) diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py --- a/rpython/rlib/rthread.py +++ b/rpython/rlib/rthread.py @@ -332,8 +332,6 @@ _threadlocalref_seeme(self) addr = llop.threadlocalref_addr(llmemory.Address) if rgc.stm_is_enabled(): - from rpython.rlib import rstm - rstm.become_inevitable() p = llmemory.cast_adr_to_ptr(addr + offset, PSTRUCTTYPE) p.c_value = value else: From noreply at buildbot.pypy.org Tue Mar 24 21:05:24 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 24 Mar 2015 21:05:24 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: make sure the PYPYLOG file is created Message-ID: <20150324200524.D821F1C0994@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76551:f8b5ab275a97 Date: 2015-03-24 20:36 +0100 http://bitbucket.org/pypy/pypy/changeset/f8b5ab275a97/ Log: make sure the PYPYLOG file is created diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py --- a/rpython/translator/c/genc.py +++ b/rpython/translator/c/genc.py @@ -816,6 +816,9 @@ print >> f, '\t/* XXX temporary workaround for late_initializations */' print >> f, '\tsyscall(SYS_arch_prctl, ARCH_SET_GS, (uint64_t)0);' + # make sure the PYPYLOG file is created + print >> f, '\tpypy_debug_ensure_opened();' + bk = database.translator.annotator.bookkeeper if bk.thread_local_fields: print >> f, '\tRPython_ThreadLocals_ProgramInit();' From noreply at buildbot.pypy.org Tue Mar 24 21:05:25 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 24 Mar 2015 21:05:25 +0100 (CET) Subject: [pypy-commit] pypy default: The weakref issue was resolved at the cost of weakref.getweakrefcount & co Message-ID: <20150324200525.F33E31C0994@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76552:058b23577be0 Date: 2015-03-24 21:05 +0100 http://bitbucket.org/pypy/pypy/changeset/058b23577be0/ Log: The weakref issue was resolved at the cost of weakref.getweakrefcount & co diff --git a/pypy/doc/stm.rst b/pypy/doc/stm.rst --- a/pypy/doc/stm.rst +++ b/pypy/doc/stm.rst @@ -155,7 +155,8 @@ show up occasionally elsewhere with accesses to some external resources, where the (apparent) serialized order doesn't match the underlying (multithreading) order. These are bugs (partially fixed - already in ``stmgc-c8``). + already in ``stmgc-c8``). Also, debugging helpers like + ``weakref.getweakrefcount()`` might give wrong answers. * The STM system is based on very efficient read/write barriers, which are mostly done (their placement could be improved a bit in @@ -380,10 +381,7 @@ Note that Python is a complicated language; there are a number of less common cases that may cause conflict (of any kind) where we might not expect it at priori. In many of these cases it could be fixed; please -report any case that you don't understand. (For example, so far, -creating a weakref to an object requires attaching an auxiliary -internal object to that object, and so it can cause write-write -conflicts.) +report any case that you don't understand. Atomic sections From noreply at buildbot.pypy.org Tue Mar 24 23:40:13 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Tue, 24 Mar 2015 23:40:13 +0100 (CET) Subject: [pypy-commit] pypy py3.3: "yield from": Correctly wrap/uwrap the returned value in the StopIteration exception. Message-ID: <20150324224013.13D991C024F@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3.3 Changeset: r76553:deae634f291c Date: 2015-03-24 15:14 +0100 http://bitbucket.org/pypy/pypy/changeset/deae634f291c/ Log: "yield from": Correctly wrap/uwrap the returned value in the StopIteration exception. It's important when the value is a tuple. diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py --- a/pypy/interpreter/generator.py +++ b/pypy/interpreter/generator.py @@ -99,7 +99,12 @@ # if the frame is now marked as finished, it was RETURNed from if frame.frame_finished_execution: self.frame = None - raise OperationError(space.w_StopIteration, w_result) + if space.is_none(w_result): + # Delay exception instantiation if we can + raise OperationError(space.w_StopIteration, space.w_None) + else: + raise OperationError(space.w_StopIteration, + space.newtuple([w_result])) else: return w_result # YIELDed finally: @@ -170,7 +175,9 @@ # Termination repetition of YIELD_FROM self.frame.last_instr += 1 if operr.match(space, space.w_StopIteration): - w_val = operr.get_w_value(space) + operr.normalize_exception(space) + w_val = space.getattr(operr.get_w_value(space), + space.wrap("value")) return self.send_ex(w_val) else: return self.send_ex(space.w_None, operr) diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py --- a/pypy/interpreter/test/test_generator.py +++ b/pypy/interpreter/test/test_generator.py @@ -348,6 +348,24 @@ assert False, 'Expected StopIteration' """ + def test_yield_from_return_tuple(self): + """ + def f1(): + result = yield from f2() + return result + def f2(): + yield 1 + return (1, 2) + g = f1() + assert next(g) == 1 + try: + next(g) + except StopIteration as e: + assert e.value == (1, 2) + else: + assert False, 'Expected StopIteration' + """ + def test_should_not_inline(space): from pypy.interpreter.generator import should_not_inline From noreply at buildbot.pypy.org Tue Mar 24 23:56:21 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Tue, 24 Mar 2015 23:56:21 +0100 (CET) Subject: [pypy-commit] pypy default: issue2007: Ensure that all codecs decode() functions can accept a bytearray. Message-ID: <20150324225621.D41541C0994@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: Changeset: r76554:3679f9727dae Date: 2015-03-24 23:52 +0100 http://bitbucket.org/pypy/pypy/changeset/3679f9727dae/ Log: issue2007: Ensure that all codecs decode() functions can accept a bytearray. diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -470,7 +470,7 @@ allow_surrogates=True) return space.newtuple([space.wrap(result), space.wrap(consumed)]) - at unwrap_spec(data=str, errors='str_or_None', byteorder=int, + at unwrap_spec(data='bufferstr', errors='str_or_None', byteorder=int, w_final=WrappedDefault(False)) def utf_16_ex_decode(space, data, errors='strict', byteorder=0, w_final=None): if errors is None: @@ -491,7 +491,7 @@ return space.newtuple([space.wrap(res), space.wrap(consumed), space.wrap(byteorder)]) - at unwrap_spec(data=str, errors='str_or_None', byteorder=int, + at unwrap_spec(data='bufferstr', errors='str_or_None', byteorder=int, w_final=WrappedDefault(False)) def utf_32_ex_decode(space, data, errors='strict', byteorder=0, w_final=None): final = space.is_true(w_final) @@ -589,7 +589,7 @@ "character mapping must return integer, None or str") - at unwrap_spec(string=str, errors='str_or_None') + at unwrap_spec(string='bufferstr', errors='str_or_None') def charmap_decode(space, string, errors="strict", w_mapping=None): if errors is None: errors = 'strict' diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -727,3 +727,23 @@ _codecs.register_error("test.test_codecs_not_a_string", f) raises(TypeError, u'\u1234'.encode, 'ascii', 'test.test_codecs_not_a_string') + + def test_decode_bytearray(self): + import _codecs + b = bytearray() + assert _codecs.ascii_decode(b) == (u'', 0) + assert _codecs.latin_1_decode(b) == (u'', 0) + assert _codecs.utf_7_decode(b) == (u'', 0) + assert _codecs.utf_8_decode(b) == (u'', 0) + assert _codecs.utf_16_be_decode(b) == (u'', 0) + assert _codecs.utf_16_decode(b) == (u'', 0) + assert _codecs.utf_16_le_decode(b) == (u'', 0) + assert _codecs.utf_16_ex_decode(b) == (u'', 0, 0) + assert _codecs.utf_32_decode(b) == (u'', 0) + assert _codecs.utf_32_be_decode(b) == (u'', 0) + assert _codecs.utf_32_le_decode(b) == (u'', 0) + assert _codecs.utf_32_ex_decode(b) == (u'', 0, 0) + assert _codecs.charmap_decode(b) == (u'', 0) + assert _codecs.unicode_escape_decode(b) == (u'', 0) + assert _codecs.raw_unicode_escape_decode(b) == (u'', 0) + assert _codecs.unicode_internal_decode(b) == (u'', 0) From noreply at buildbot.pypy.org Wed Mar 25 08:54:56 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 25 Mar 2015 08:54:56 +0100 (CET) Subject: [pypy-commit] buildbot default: Add a title to the longrepr pages Message-ID: <20150325075456.B35B11C02BB@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r942:79ce0aa6cfae Date: 2015-03-25 08:49 +0100 http://bitbucket.org/pypy/buildbot/changeset/79ce0aa6cfae/ Log: Add a title to the longrepr pages diff --git a/bot2/pypybuildbot/summary.py b/bot2/pypybuildbot/summary.py --- a/bot2/pypybuildbot/summary.py +++ b/bot2/pypybuildbot/summary.py @@ -597,7 +597,8 @@ longrepr = outcome_set.get_longrepr(namekey) - return html.div([html.pre(longrepr), + return html.div([html.h2(self.getTitle(request)), + html.pre(longrepr), py.xml.raw("" % outcome_set_cache.stats()) ]).unicode() From noreply at buildbot.pypy.org Wed Mar 25 08:54:57 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 25 Mar 2015 08:54:57 +0100 (CET) Subject: [pypy-commit] buildbot default: one more test passing, still a few failures Message-ID: <20150325075457.C60201C02BB@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r943:559639eb2e13 Date: 2015-03-25 08:55 +0100 http://bitbucket.org/pypy/buildbot/changeset/559639eb2e13/ Log: one more test passing, still a few failures diff --git a/bot2/pypybuildbot/test/test_builds.py b/bot2/pypybuildbot/test/test_builds.py --- a/bot2/pypybuildbot/test/test_builds.py +++ b/bot2/pypybuildbot/test/test_builds.py @@ -34,6 +34,7 @@ class FakeBuild(object): slaveEnvironment = None + master = None def __init__(self, properties=None): self.properties = FakeProperties(properties) From noreply at buildbot.pypy.org Wed Mar 25 08:54:58 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 25 Mar 2015 08:54:58 +0100 (CET) Subject: [pypy-commit] buildbot default: merge heads Message-ID: <20150325075458.E23321C02BB@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r944:fb73d347aa69 Date: 2015-03-25 08:55 +0100 http://bitbucket.org/pypy/buildbot/changeset/fb73d347aa69/ Log: merge heads diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -405,7 +405,7 @@ '/D', '-' + nDays, '/c', "cmd /c rmdir /q /s @path"] else: command = ['find', Interpolate(tmp_or_crazy + pytest), '-mtime', - '+' + nDays, '-exec', 'rm -r', '{}', ';'] + '+' + nDays, '-exec', 'rm', '-r', '{}', ';'] factory.addStep(SuccessAlways( description="cleanout old test files", command = command, @@ -504,7 +504,7 @@ '/D', '-' + nDays, '/c', "cmd /c rmdir /q /s @path"] else: command = ['find', Interpolate(tmp_or_crazy + pytest), '-mtime', - '+' + nDays, '-exec', 'rm -r', '{}', ';'] + '+' + nDays, '-exec', 'rm', '-r', '{}', ';'] self.addStep(SuccessAlways( description="cleanout old test files", command = command, From noreply at buildbot.pypy.org Wed Mar 25 09:01:09 2015 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 25 Mar 2015 09:01:09 +0100 (CET) Subject: [pypy-commit] pypy vmprof: hopefully improve the situation of write barriers Message-ID: <20150325080109.4FAC61C1C49@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76555:110dffa78b50 Date: 2015-03-25 10:01 +0200 http://bitbucket.org/pypy/pypy/changeset/110dffa78b50/ Log: hopefully improve the situation of write barriers diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c --- a/pypy/module/_vmprof/src/vmprof.c +++ b/pypy/module/_vmprof/src/vmprof.c @@ -37,10 +37,11 @@ #define MAX_FUNC_NAME 128 #define MAX_STACK_DEPTH 1024 +#define BUFFER_SIZE 8192 static int profile_file = 0; -static char profile_write_buffer[100000]; +static char profile_write_buffer[BUFFER_SIZE]; static int profile_buffer_position = 0; void* vmprof_mainloop_func; static ptrdiff_t mainloop_sp_offset; @@ -99,12 +100,14 @@ void* _unused3[sizeof(unw_cursor_t)/sizeof(void*) - 4]; } vmprof_hacked_unw_cursor_t; -static int vmprof_unw_step(unw_cursor_t *cp) { +static int vmprof_unw_step(unw_cursor_t *cp, int first_run) { void* ip; void* sp; ptrdiff_t sp_offset; unw_get_reg (cp, UNW_REG_IP, (unw_word_t*)&ip); unw_get_reg (cp, UNW_REG_SP, (unw_word_t*)&sp); + if (!first_run) + ip -= 1; sp_offset = vmprof_unw_get_custom_offset(ip, cp); if (sp_offset == -1) { @@ -158,6 +161,7 @@ int ret = unw_init_local(&cursor, &uc); assert(ret >= 0); _unused(ret); + int first_run = 1; while (n < max_depth) { if (unw_get_reg(&cursor, UNW_REG_IP, (unw_word_t *) &ip) < 0) { @@ -194,9 +198,10 @@ result[n++] = ip; n = vmprof_write_header_for_jit_addr(result, n, ip, max_depth); - if (vmprof_unw_step(&cursor) <= 0) { + if (vmprof_unw_step(&cursor, first_run) <= 0) { break; } + first_run = 0; } --recursive; return n; From noreply at buildbot.pypy.org Wed Mar 25 09:13:24 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 25 Mar 2015 09:13:24 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Fix some of these tests Message-ID: <20150325081324.F229D1C0F98@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76556:23ce9fa6042c Date: 2015-03-25 09:12 +0100 http://bitbucket.org/pypy/pypy/changeset/23ce9fa6042c/ Log: Fix some of these tests diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py --- a/pypy/module/pypyjit/test_pypy_c/model.py +++ b/pypy/module/pypyjit/test_pypy_c/model.py @@ -327,7 +327,8 @@ ticker0 = getfield_raw(#, descr=) ticker_cond0 = int_lt(ticker0, 0) guard_false(ticker_cond0, descr=...) - """ + guard_not_invalidated? + """ # guard_not_invalidated might be at the end, in pypy-stm src = src.replace('--TICK--', ticker_check) # # this is the ticker check generated if we have threads diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py --- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py +++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py @@ -446,14 +446,19 @@ assert loop.filename == self.filepath assert loop.code.co.co_name == 'f' # - ops = loop.allops() - assert log.opnames(ops) == [ - # this is the actual loop - 'int_lt', 'guard_true', 'int_add', - # this is the signal checking stuff - 'guard_not_invalidated', 'getfield_raw', 'int_lt', 'guard_false', - 'jump' - ] + ops = log.opnames(loop.allops()) + found = False + for SIGCHECK in ( + ['guard_not_invalidated', 'getfield_raw', 'int_lt', 'guard_false'], + ['getfield_raw', 'int_lt', 'guard_false', 'guard_not_invalidated'], + ): + found |= (ops == [ + # this is the actual loop + 'int_lt', 'guard_true', 'int_add', + # this is the signal checking stuff + ] + SIGCHECK + ['jump']) + # + assert found def test_ops_by_id(self): def f(): @@ -512,13 +517,18 @@ assert add_ops == ['int_add'] # ops = log.opnames(loop.allops()) - assert ops == [ - # this is the actual loop - 'int_lt', 'guard_true', 'force_token', 'int_add', - # this is the signal checking stuff - 'guard_not_invalidated', 'getfield_raw', 'int_lt', 'guard_false', - 'jump' - ] + found = False + for SIGCHECK in ( + ['guard_not_invalidated', 'getfield_raw', 'int_lt', 'guard_false'], + ['getfield_raw', 'int_lt', 'guard_false', 'guard_not_invalidated'], + ): + found |= (ops == [ + # this is the actual loop + 'int_lt', 'guard_true', 'force_token', 'int_add', + # this is the signal checking stuff + ] + SIGCHECK + ['jump']) + # + assert found def test_loop_match(self): def f(): @@ -534,10 +544,11 @@ guard_true(i6, descr=...) i8 = int_add(i4, 1) # signal checking stuff - guard_not_invalidated(descr=...) + guard_not_invalidated? i10 = getfield_raw(..., descr=<.* pypysig_long_struct.c_value .*>) i14 = int_lt(i10, 0) guard_false(i14, descr=...) + guard_not_invalidated? jump(..., descr=...) """) # diff --git a/pypy/module/pypyjit/test_pypy_c/test_thread.py b/pypy/module/pypyjit/test_pypy_c/test_thread.py --- a/pypy/module/pypyjit/test_pypy_c/test_thread.py +++ b/pypy/module/pypyjit/test_pypy_c/test_thread.py @@ -71,7 +71,6 @@ i59 = call(ConstClass(RPyThreadReleaseLock), i37, descr=) i60 = int_is_true(i59) guard_false(i60, descr=...) - guard_not_invalidated(descr=...) --TICK-- jump(..., descr=...) """) From noreply at buildbot.pypy.org Wed Mar 25 10:52:39 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 25 Mar 2015 10:52:39 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Explicitly introduce 'is_stm' and check different things Message-ID: <20150325095239.741C91C0334@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76557:e656b880e267 Date: 2015-03-25 10:52 +0100 http://bitbucket.org/pypy/pypy/changeset/e656b880e267/ Log: Explicitly introduce 'is_stm' and check different things diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py --- a/pypy/module/pypyjit/test_pypy_c/model.py +++ b/pypy/module/pypyjit/test_pypy_c/model.py @@ -10,6 +10,8 @@ TraceForOpcode) from rpython.tool.jitlogparser.storage import LoopStorage +is_stm = 'pypystm' in sys.builtin_module_names + def find_ids_range(code): """ @@ -322,24 +324,41 @@ # in jump_absolute() in pypyjit/interp.py. The string --TICK-- is # replaced with the corresponding operations, so that tests don't have # to repeat it every time - ticker_check = """ - guard_not_invalidated? - ticker0 = getfield_raw(#, descr=) - ticker_cond0 = int_lt(ticker0, 0) - guard_false(ticker_cond0, descr=...) - guard_not_invalidated? - """ # guard_not_invalidated might be at the end, in pypy-stm + if not is_stm: + ticker_check = """ + guard_not_invalidated? + ticker0 = getfield_raw(#, descr=) + ticker_cond0 = int_lt(ticker0, 0) + guard_false(ticker_cond0, descr=...) + """ + else: + ticker_check = """ + ticker0 = getfield_raw(#, descr=) + ticker_cond0 = int_lt(ticker0, 0) + guard_false(ticker_cond0, descr=...) + guard_not_invalidated? + """ src = src.replace('--TICK--', ticker_check) # # this is the ticker check generated if we have threads - thread_ticker_check = """ - guard_not_invalidated? - ticker0 = getfield_raw(#, descr=) - ticker1 = int_sub(ticker0, #) - setfield_raw(#, ticker1, descr=) - ticker_cond0 = int_lt(ticker1, 0) - guard_false(ticker_cond0, descr=...) - """ + if not is_stm: + thread_ticker_check = """ + guard_not_invalidated? + ticker0 = getfield_raw(#, descr=) + ticker1 = int_sub(ticker0, #) + setfield_raw(#, ticker1, descr=) + ticker_cond0 = int_lt(ticker1, 0) + guard_false(ticker_cond0, descr=...) + """ + else: + thread_ticker_check = """ + ticker0 = getfield_raw(#, descr=) + ticker_cond0 = int_lt(ticker0, 0) + guard_false(ticker_cond0, descr=...) + guard_not_invalidated? + i_sbt = stm_should_break_transaction() + guard_false(i_sbt, descr=...) + """ src = src.replace('--THREAD-TICK--', thread_ticker_check) # # this is the ticker check generated in PyFrame.handle_operation_error diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py --- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py +++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py @@ -8,7 +8,15 @@ from rpython.tool import logparser from rpython.jit.tool.jitoutput import parse_prof from pypy.module.pypyjit.test_pypy_c.model import \ - Log, find_ids_range, find_ids, OpMatcher, InvalidMatch + Log, find_ids_range, find_ids, OpMatcher, InvalidMatch, is_stm + + +if not is_stm: + SIGCHECK = ['guard_not_invalidated', + 'getfield_raw', 'int_lt', 'guard_false'] +else: + SIGCHECK = ['getfield_raw', 'int_lt', 'guard_false', + 'guard_not_invalidated'] class BaseTestPyPyC(object): @@ -446,19 +454,14 @@ assert loop.filename == self.filepath assert loop.code.co.co_name == 'f' # - ops = log.opnames(loop.allops()) - found = False - for SIGCHECK in ( - ['guard_not_invalidated', 'getfield_raw', 'int_lt', 'guard_false'], - ['getfield_raw', 'int_lt', 'guard_false', 'guard_not_invalidated'], - ): - found |= (ops == [ - # this is the actual loop - 'int_lt', 'guard_true', 'int_add', - # this is the signal checking stuff - ] + SIGCHECK + ['jump']) - # - assert found + ops = loop.allops() + assert log.opnames(ops) == [ + # this is the actual loop + 'int_lt', 'guard_true', 'int_add', + # this is the signal checking stuff + ] + SIGCHECK + [ + 'jump' + ] def test_ops_by_id(self): def f(): @@ -517,18 +520,13 @@ assert add_ops == ['int_add'] # ops = log.opnames(loop.allops()) - found = False - for SIGCHECK in ( - ['guard_not_invalidated', 'getfield_raw', 'int_lt', 'guard_false'], - ['getfield_raw', 'int_lt', 'guard_false', 'guard_not_invalidated'], - ): - found |= (ops == [ - # this is the actual loop - 'int_lt', 'guard_true', 'force_token', 'int_add', - # this is the signal checking stuff - ] + SIGCHECK + ['jump']) - # - assert found + assert ops == [ + # this is the actual loop + 'int_lt', 'guard_true', 'force_token', 'int_add', + # this is the signal checking stuff + ] + SIGCHECK + [ + 'jump' + ] def test_loop_match(self): def f(): From noreply at buildbot.pypy.org Wed Mar 25 11:04:53 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 25 Mar 2015 11:04:53 +0100 (CET) Subject: [pypy-commit] pypy default: getsignalname(6) -> SIGABRT, not SIGIOT (strange synonym) Message-ID: <20150325100453.632F61C0334@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76558:016d3b84b7b0 Date: 2015-03-25 11:04 +0100 http://bitbucket.org/pypy/pypy/changeset/016d3b84b7b0/ Log: getsignalname(6) -> SIGABRT, not SIGIOT (strange synonym) diff --git a/testrunner/runner.py b/testrunner/runner.py --- a/testrunner/runner.py +++ b/testrunner/runner.py @@ -101,7 +101,9 @@ return 0 def getsignalname(n): - for name, value in signal.__dict__.items(): + # "sorted()" to pick a deterministic answer in case of synonyms. + # Also, getting SIGABRT is more understandable than SIGIOT... + for name, value in sorted(signal.__dict__.items()): if value == n and name.startswith('SIG'): return name return 'signal %d' % (n,) From noreply at buildbot.pypy.org Wed Mar 25 15:08:49 2015 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 25 Mar 2015 15:08:49 +0100 (CET) Subject: [pypy-commit] pypy vmprof: add a comment why we need to do -1 Message-ID: <20150325140849.DC95B1C117D@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76559:8e77be4b6db3 Date: 2015-03-25 16:08 +0200 http://bitbucket.org/pypy/pypy/changeset/8e77be4b6db3/ Log: add a comment why we need to do -1 diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c --- a/pypy/module/_vmprof/src/vmprof.c +++ b/pypy/module/_vmprof/src/vmprof.c @@ -107,6 +107,9 @@ unw_get_reg (cp, UNW_REG_IP, (unw_word_t*)&ip); unw_get_reg (cp, UNW_REG_SP, (unw_word_t*)&sp); if (!first_run) + // make sure we're pointing to the CALL and not to the first + // instruction after. If the callee adjusts the stack for us + // it's not safe to be at the instruction after ip -= 1; sp_offset = vmprof_unw_get_custom_offset(ip, cp); From noreply at buildbot.pypy.org Wed Mar 25 15:47:12 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 25 Mar 2015 15:47:12 +0100 (CET) Subject: [pypy-commit] pypy default: Expand on transaction.time and .clock Message-ID: <20150325144712.C82FD1C00F8@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76560:623926e972d5 Date: 2015-03-25 15:47 +0100 http://bitbucket.org/pypy/pypy/changeset/623926e972d5/ Log: Expand on transaction.time and .clock diff --git a/pypy/doc/stm.rst b/pypy/doc/stm.rst --- a/pypy/doc/stm.rst +++ b/pypy/doc/stm.rst @@ -350,10 +350,17 @@ ``stmidset`` classes using the identity of the key. * ``time.time()`` and ``time.clock()`` turn the transaction inevitable - in order to guarantee that a call that appears to be later will - really return a higher number. If getting slightly unordered - results is fine, use ``transaction.time()`` or - ``transaction.clock()``. + in order to guarantee that a call that appears to be later will really + return a higher number. If getting slightly unordered results is + fine, use ``transaction.time()`` or ``transaction.clock()``. The + latter operations guarantee to return increasing results only if you + can "prove" that two calls occurred in a specific order (for example + because they are both called by the same thread). In cases where no + such proof is possible, you might get randomly interleaved values. + (If you have two independent transactions, they normally behave as if + one of them was fully executed before the other; but using + ``transaction.time()`` you might see the "hidden truth" that they are + actually interleaved.) * ``transaction.threadlocalproperty`` can be used at class-level:: From noreply at buildbot.pypy.org Wed Mar 25 17:33:50 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 25 Mar 2015 17:33:50 +0100 (CET) Subject: [pypy-commit] stmgc default: Update README Message-ID: <20150325163350.322E41C02BB@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1740:ded5aa39a871 Date: 2015-03-25 17:34 +0100 http://bitbucket.org/pypy/stmgc/changeset/ded5aa39a871/ Log: Update README diff --git a/c7/llvmfix/README.txt b/c7/llvmfix/README.txt --- a/c7/llvmfix/README.txt +++ b/c7/llvmfix/README.txt @@ -15,6 +15,7 @@ mkdir llvm-build cd llvm-build ../llvm/configure --enable-optimized # requires gcc >= 4.7! + --disable-assertions # optional make From noreply at buildbot.pypy.org Wed Mar 25 19:23:57 2015 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 25 Mar 2015 19:23:57 +0100 (CET) Subject: [pypy-commit] pypy release-2.5.x: extend the description of stdlib 2.7.9 by quoting cpython's release notice Message-ID: <20150325182357.78A201C02BB@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: release-2.5.x Changeset: r76561:5df58d7d9850 Date: 2015-03-25 20:25 +0200 http://bitbucket.org/pypy/pypy/changeset/5df58d7d9850/ Log: extend the description of stdlib 2.7.9 by quoting cpython's release notice diff --git a/pypy/doc/release-2.5.1.rst b/pypy/doc/release-2.5.1.rst --- a/pypy/doc/release-2.5.1.rst +++ b/pypy/doc/release-2.5.1.rst @@ -67,7 +67,19 @@ `PyPy documentation`_ and we now have seperate `RPython documentation`_. Tell us what still isn't clear, or even better help us improve the documentation. -* We merged version 2.7.9 of python's stdlib +* We merged version 2.7.9 of python's stdlib. From the python release notice: + + * The entirety of Python 3.4's `ssl module`_ has been backported. + See `PEP 466`_ for justification. + + * HTTPS certificate validation using the system's certificate store is now + enabled by default. See `PEP 476`_ for details. + + * SSLv3 has been disabled by default in httplib and its reverse dependencies + due to the `POODLE attack`_. + + * The `ensurepip module`_ has been backported, which provides the pip + package manager in every Python 2.7 installation. See `PEP 477`_. * The garbage collector now ignores parts of the stack which did not change since the last collection, another performance boost @@ -84,6 +96,12 @@ .. _`PyPy documentation`: http://doc.pypy.org .. _`RPython documentation`: http://rpython.readthedocs.org +.. _`ssl module`: https://docs.python.org/3/library/ssl.html +.. _`PEP 466`: https://www.python.org/dev/peps/pep-0466 +.. _`PEP 476`: https://www.python.org/dev/peps/pep-0476 +.. _`PEP 477`: https://www.python.org/dev/peps/pep-0477 +.. _`POODLE attack`: https://www.imperialviolet.org/2014/10/14/poodle.html +.. _`ensurepip module`: https://docs.python.org/2/library/ensurepip.html .. _resolved: http://doc.pypy.org/en/latest/whatsnew-2.5.1.html Please try it out and let us know what you think. We welcome From noreply at buildbot.pypy.org Wed Mar 25 20:06:08 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Wed, 25 Mar 2015 20:06:08 +0100 (CET) Subject: [pypy-commit] pypy default: Add test for issue #260 Message-ID: <20150325190608.52BD01C1FB3@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r76562:f3ff810e8559 Date: 2015-03-25 19:04 +0000 http://bitbucket.org/pypy/pypy/changeset/f3ff810e8559/ Log: Add test for issue #260 diff --git a/rpython/annotator/test/test_model.py b/rpython/annotator/test/test_model.py --- a/rpython/annotator/test/test_model.py +++ b/rpython/annotator/test/test_model.py @@ -37,6 +37,10 @@ (s4, s4), (s4, s6), (s6, s6)]) +def test_signedness(): + assert not SomeInteger(unsigned=True).contains(SomeInteger()) + assert SomeInteger(unsigned=True).contains(SomeInteger(nonneg=True)) + def test_commonbase_simple(): class A0: pass From noreply at buildbot.pypy.org Thu Mar 26 02:42:49 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Thu, 26 Mar 2015 02:42:49 +0100 (CET) Subject: [pypy-commit] pypy default: Simplify ListItem.merge() Message-ID: <20150326014249.D905A1C00F8@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r76563:152a2c95b462 Date: 2015-03-26 01:42 +0000 http://bitbucket.org/pypy/pypy/changeset/152a2c95b462/ Log: Simplify ListItem.merge() diff --git a/rpython/annotator/listdef.py b/rpython/annotator/listdef.py --- a/rpython/annotator/listdef.py +++ b/rpython/annotator/listdef.py @@ -83,9 +83,6 @@ self.setrangestep(self._step_map[type(self.range_step), type(other.range_step)]) self.itemof.update(other.itemof) - read_locations = self.read_locations.copy() - other_read_locations = other.read_locations.copy() - self.read_locations.update(other.read_locations) s_value = self.s_value s_other_value = other.s_value s_new_value = unionof(s_value, s_other_value) @@ -95,18 +92,20 @@ self.patch() # which should patch all refs to 'other' if s_new_value != s_value: self.s_value = s_new_value - # reflow from reading points - for position_key in read_locations: - self.bookkeeper.annotator.reflowfromposition(position_key) + self.notify_update() if s_new_value != s_other_value: - # reflow from reading points - for position_key in other_read_locations: - other.bookkeeper.annotator.reflowfromposition(position_key) + other.notify_update() + self.read_locations.update(other.read_locations) def patch(self): for listdef in self.itemof: listdef.listitem = self + def notify_update(self): + '''Reflow from all reading points''' + for position_key in self.read_locations: + self.bookkeeper.annotator.reflowfromposition(position_key) + def generalize(self, s_other_value): s_new_value = unionof(self.s_value, s_other_value) updated = s_new_value != self.s_value @@ -114,9 +113,7 @@ if self.dont_change_any_more: raise TooLateForChange self.s_value = s_new_value - # reflow from all reading points - for position_key in self.read_locations: - self.bookkeeper.annotator.reflowfromposition(position_key) + self.notify_update() return updated From noreply at buildbot.pypy.org Thu Mar 26 04:57:22 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 26 Mar 2015 04:57:22 +0100 (CET) Subject: [pypy-commit] pypy default: merge release 2.5.1 into default Message-ID: <20150326035722.8390A1C00F8@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r76564:e6df383db0e6 Date: 2015-03-26 05:56 +0200 http://bitbucket.org/pypy/pypy/changeset/e6df383db0e6/ Log: merge release 2.5.1 into default diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -67,7 +67,7 @@ # The short X.Y version. version = '2.5' # The full version, including alpha/beta/rc tags. -release = '2.5.0' +release = '2.5.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-2.5.1.rst release-2.5.0.rst release-2.4.0.rst release-2.3.1.rst diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -7,6 +7,7 @@ .. toctree:: whatsnew-head.rst + whatsnew-2.5.1.rst whatsnew-2.5.0.rst whatsnew-2.4.0.rst whatsnew-2.3.1.rst diff --git a/pypy/doc/release-2.5.1.rst b/pypy/doc/release-2.5.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-2.5.1.rst @@ -0,0 +1,115 @@ +================================ +PyPy 2.5.1 - Pineapple Bromeliad +================================ + +We're pleased to announce PyPy 2.5.1, Pineapple `Bromeliad`_ following on the heels of 2.5.0 + +You can download the PyPy 2.5.1 release here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project, and for those who donate to our three sub-projects, as well as our +volunteers and contributors. +We've shown quite a bit of progress, but we're slowly running out of funds. +Please consider donating more, or even better convince your employer to donate, +so we can finish those projects! The three sub-projects are: + +* `Py3k`_ (supporting Python 3.x): We have released a Python 3.2.5 compatible version + we call PyPy3 2.4.0, and are working toward a Python 3.3 compatible version + +* `STM`_ (software transactional memory): We have released a first working version, + and continue to try out new promising paths of achieving a fast multithreaded Python + +* `NumPy`_ which requires installation of our fork of upstream numpy, + available `on bitbucket`_ + +.. _`Bromeliad`: http://xkcd.com/1498 +.. _`Py3k`: http://pypy.org/py3donate.html +.. _`STM`: http://pypy.org/tmdonate2.html +.. _`NumPy`: http://pypy.org/numpydonate.html +.. _`on bitbucket`: https://www.bitbucket.org/pypy/numpy + +We would also like to encourage new people to join the project. PyPy has many +layers and we need help with all of them: `PyPy`_ and `Rpython`_ documentation +improvements, tweaking popular `modules`_ to run on pypy, or general `help`_ with making +Rpython's JIT even better. + +.. _`PyPy`: http://doc.pypy.org +.. _`Rpython`: http://rpython.readthedocs.org +.. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly +.. _`help`: http://doc.pypy.org/en/latest/project-ideas.html + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7. It's fast (`pypy and cpython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +This release supports **x86** machines on most common operating systems +(Linux 32/64, Mac OS X 64, Windows, and OpenBSD), +as well as newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux. + +While we support 32 bit python on Windows, work on the native Windows 64 +bit python is still stalling, we would welcome a volunteer +to `handle that`_. + +.. _`pypy and cpython 2.7.x`: http://speed.pypy.org +.. _`handle that`: http://doc.pypy.org/en/latest/windows.html#what-is-missing-for-a-full-64-bit-translation + +Highlights +========== + +* The past months have seen pypy mature and grow, as rpython becomes the goto + solution for writing fast dynamic language interpreters. Our separation of + Rpython and the python interpreter PyPy is now much clearer in the + `PyPy documentation`_ and we now have seperate `RPython documentation`_. + Tell us what still isn't clear, or even better help us improve the documentation. + +* We merged version 2.7.9 of python's stdlib. From the python release notice: + + * The entirety of Python 3.4's `ssl module`_ has been backported. + See `PEP 466`_ for justification. + + * HTTPS certificate validation using the system's certificate store is now + enabled by default. See `PEP 476`_ for details. + + * SSLv3 has been disabled by default in httplib and its reverse dependencies + due to the `POODLE attack`_. + + * The `ensurepip module`_ has been backported, which provides the pip + package manager in every Python 2.7 installation. See `PEP 477`_. + +* The garbage collector now ignores parts of the stack which did not change + since the last collection, another performance boost + +* errno and LastError are saved around cffi calls so things like pdb will not + overwrite it + +* We continue to asymptotically approach a score of 7 times faster than cpython + on our benchmark suite, we now rank 6.98 on latest runs + +* Issues reported with our previous release were resolved_ after reports from users on + our issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at + #pypy. + +.. _`PyPy documentation`: http://doc.pypy.org +.. _`RPython documentation`: http://rpython.readthedocs.org +.. _`ssl module`: https://docs.python.org/3/library/ssl.html +.. _`PEP 466`: https://www.python.org/dev/peps/pep-0466 +.. _`PEP 476`: https://www.python.org/dev/peps/pep-0476 +.. _`PEP 477`: https://www.python.org/dev/peps/pep-0477 +.. _`POODLE attack`: https://www.imperialviolet.org/2014/10/14/poodle.html +.. _`ensurepip module`: https://docs.python.org/2/library/ensurepip.html +.. _resolved: http://doc.pypy.org/en/latest/whatsnew-2.5.1.html + +Please try it out and let us know what you think. We welcome +success stories, `experiments`_, or `benchmarks`_, we know you are using PyPy, please tell us about it! + +Cheers + +The PyPy Team + +.. _`experiments`: http://morepypy.blogspot.com/2015/02/experiments-in-pyrlang-with-rpython.html +.. _`benchmarks`: https://mithrandi.net/blog/2015/03/axiom-benchmark-results-on-pypy-2-5-0 diff --git a/pypy/doc/whatsnew-2.5.0.rst b/pypy/doc/whatsnew-2.5.0.rst --- a/pypy/doc/whatsnew-2.5.0.rst +++ b/pypy/doc/whatsnew-2.5.0.rst @@ -1,6 +1,6 @@ -======================= -What's new in PyPy 2.5 -======================= +======================== +What's new in PyPy 2.5.0 +======================== .. this is a revision shortly after release-2.4.x .. startrev: 7026746cbb1b diff --git a/pypy/doc/whatsnew-2.5.1.rst b/pypy/doc/whatsnew-2.5.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-2.5.1.rst @@ -0,0 +1,40 @@ +======================== +What's new in PyPy 2.5.1 +======================== + +.. this is a revision shortly after release-2.5.0 +.. startrev: 397b96217b85 + + +Non-blocking file reads sometimes raised EAGAIN even though they +had buffered data waiting, fixed in b1c4fcb04a42 + +Fix a bug in cpyext in multithreded programs acquiring/releasing the GIL + +.. branch: vmprof + +.. branch: stackroot-speedup-2 +Avoid tracing all stack roots during repeated minor collections, +by ignoring the part of the stack that didn't change + +.. branch: stdlib-2.7.9 +Update stdlib to version 2.7.9 + +.. branch: fix-kqueue-error2 +Fix exception being raised by kqueue.control (CPython compatibility) + +.. branch: gitignore + +.. branch: framestate2 +Refactor rpython.flowspace.framestate.FrameState. + +.. branch: alt_errno +Add an alternative location to save LastError, errno around ctypes, +cffi external calls so things like pdb will not overwrite it + +.. branch: nonquadratic-heapcache +Speed up the warmup times of the JIT by removing a quadratic algorithm in the +heapcache. + +.. branch: online-transforms-2 +Simplify flow graphs on the fly during annotation phase. diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -2,38 +2,6 @@ What's new in PyPy 2.5+ ======================= -.. this is a revision shortly after release-2.5.x +.. this is a revision shortly after release-2.5.1 .. startrev: 397b96217b85 - -Non-blocking file reads sometimes raised EAGAIN even though they -had buffered data waiting, fixed in b1c4fcb04a42 - - -.. branch: vmprof - -.. branch: stackroot-speedup-2 -Avoid tracing all stack roots during repeated minor collections, -by ignoring the part of the stack that didn't change - -.. branch: stdlib-2.7.9 -Update stdlib to version 2.7.9 - -.. branch: fix-kqueue-error2 -Fix exception being raised by kqueue.control (CPython compatibility) - -.. branch: gitignore - -.. branch: framestate2 -Refactor rpython.flowspace.framestate.FrameState. - -.. branch: alt_errno -Add an alternative location to save LastError, errno around ctypes, -cffi external calls so things like pdb will not overwrite it - -.. branch: nonquadratic-heapcache -Speed up the warmup times of the JIT by removing a quadratic algorithm in the -heapcache. - -.. branch: online-transforms-2 -Simplify flow graphs on the fly during annotation phase. From noreply at buildbot.pypy.org Thu Mar 26 04:57:23 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 26 Mar 2015 04:57:23 +0100 (CET) Subject: [pypy-commit] pypy default: merge heads Message-ID: <20150326035723.B218F1C00F8@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r76565:fcd88928b3e9 Date: 2015-03-26 05:58 +0200 http://bitbucket.org/pypy/pypy/changeset/fcd88928b3e9/ Log: merge heads diff --git a/rpython/annotator/listdef.py b/rpython/annotator/listdef.py --- a/rpython/annotator/listdef.py +++ b/rpython/annotator/listdef.py @@ -83,9 +83,6 @@ self.setrangestep(self._step_map[type(self.range_step), type(other.range_step)]) self.itemof.update(other.itemof) - read_locations = self.read_locations.copy() - other_read_locations = other.read_locations.copy() - self.read_locations.update(other.read_locations) s_value = self.s_value s_other_value = other.s_value s_new_value = unionof(s_value, s_other_value) @@ -95,18 +92,20 @@ self.patch() # which should patch all refs to 'other' if s_new_value != s_value: self.s_value = s_new_value - # reflow from reading points - for position_key in read_locations: - self.bookkeeper.annotator.reflowfromposition(position_key) + self.notify_update() if s_new_value != s_other_value: - # reflow from reading points - for position_key in other_read_locations: - other.bookkeeper.annotator.reflowfromposition(position_key) + other.notify_update() + self.read_locations.update(other.read_locations) def patch(self): for listdef in self.itemof: listdef.listitem = self + def notify_update(self): + '''Reflow from all reading points''' + for position_key in self.read_locations: + self.bookkeeper.annotator.reflowfromposition(position_key) + def generalize(self, s_other_value): s_new_value = unionof(self.s_value, s_other_value) updated = s_new_value != self.s_value @@ -114,9 +113,7 @@ if self.dont_change_any_more: raise TooLateForChange self.s_value = s_new_value - # reflow from all reading points - for position_key in self.read_locations: - self.bookkeeper.annotator.reflowfromposition(position_key) + self.notify_update() return updated diff --git a/rpython/annotator/test/test_model.py b/rpython/annotator/test/test_model.py --- a/rpython/annotator/test/test_model.py +++ b/rpython/annotator/test/test_model.py @@ -37,6 +37,10 @@ (s4, s4), (s4, s6), (s6, s6)]) +def test_signedness(): + assert not SomeInteger(unsigned=True).contains(SomeInteger()) + assert SomeInteger(unsigned=True).contains(SomeInteger(nonneg=True)) + def test_commonbase_simple(): class A0: pass From noreply at buildbot.pypy.org Thu Mar 26 05:27:01 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 26 Mar 2015 05:27:01 +0100 (CET) Subject: [pypy-commit] pypy.org extradoc: update for 2.5.1 Message-ID: <20150326042701.77BC01C1CA9@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: extradoc Changeset: r584:ecf6d04897c4 Date: 2015-03-26 06:26 +0200 http://bitbucket.org/pypy/pypy.org/changeset/ecf6d04897c4/ Log: update for 2.5.1 diff --git a/download.html b/download.html --- a/download.html +++ b/download.html @@ -71,12 +71,10 @@

Download and install

There are nightly binary builds available. Those builds are not always as stable as the release, but they contain numerous bugfixes and -performance improvements. Note that the OS X nightly builds -(but not the release) are slightly -miscompiled due to buildslave being old. Contributions are welcomed.

+performance improvements.

We provide binaries for x86 and ARM Linux, Mac OS/X and Windows for:

    @@ -112,18 +110,18 @@ degrees of being up-to-date. You may have more luck trying out Squeaky's portable Linux binaries.

    -
    -

    Python2.7 compatible PyPy 2.5.0

    +
    +

    Python2.7 compatible PyPy 2.5.1

    @@ -159,7 +157,7 @@

    Other versions

    The other versions of PyPy are:

      -
    • The most up-to-date nightly build with a JIT, if the official +
    • The most up-to-date nightly binary builds with a JIT, if the official release is too old for what you want to do. There are versions for different libc on this site too.
    • Sandboxing: A special safe version. Read the docs about sandboxing. @@ -181,7 +179,7 @@ uncompressed, they run in-place. For now you can uncompress them either somewhere in your home directory or, say, in /opt, and if you want, put a symlink from somewhere like -/usr/local/bin/pypy to /path/to/pypy-2.5.0/bin/pypy. Do +/usr/local/bin/pypy to /path/to/pypy-2.5.1/bin/pypy. Do not move or copy the executable pypy outside the tree – put a symlink to it, otherwise it will not find its libraries.

    @@ -223,7 +221,7 @@

  • Get the source code. The following packages contain the source at the same revision as the above binaries:

    Or you can checkout the current trunk using Mercurial (the trunk usually works and is of course more up-to-date):

    @@ -264,7 +262,7 @@ then note that we are talking about CPython 2.5-2.7 here, not CPython 3.x.

  • If RAM usage is a problem (or if you are on Windows, because win32's limit -is 2 GB unless you hack a lot), then you can (for now) tweak some parameters +is 2 GB unless you have a 64 bit OS), then you can (for now) tweak some parameters via environment variables and command-line options. The following command takes a bit more time, but finishes with only using 3.0 GB of RAM (on Linux 64-bit; probably not much more than 1.6 GB on 32-bit). It should be @@ -303,16 +301,21 @@

    • Checksums

    -

    Here are the checksums for each of the downloads (md5 and sha1):

    +

    Here are the checksums for each of the downloads

    +

    pypy-2.5.1 md5:

    -0a109c96bbce20931c17dfded57a3d46  pypy-2.5.0-linux64.tar.bz2
-fc79e0ef640371fa7bdbb040c3211bcf  pypy-2.5.0-linux-armel.tar.bz2
-d8d0fe8a1503a7ed4ec5ff52d0630b88  pypy-2.5.0-linux-armhf-raring.tar.bz2
-9a4430abd36dd619c45705c3fe70f5ca  pypy-2.5.0-linux-armhf-raspbian.tar.bz2
-1c14e954fdd1d112540790ec5b0cea26  pypy-2.5.0-linux.tar.bz2
-9916a1485f32953a46afdd36f8f01f83  pypy-2.5.0-osx64.tar.bz2
-f4700c0af45e986178b36ce91a45136e  pypy-2.5.0-src.tar.bz2
-2461a4e4fe4ea64ce7fb1a07aa310e6a  pypy-2.5.0-win32.zip
+b3cc9f8a419f9f89c3fac34b39e92e0a  pypy-2.5.1-linux64.tar.bz2
+fe663500fb87d251ebf02917d25dca23  pypy-2.5.1-linux-armel.tar.bz2
+7a0f845baec7a6ccfb57a66f0e7980e9  pypy-2.5.1-linux-armhf-raring.tar.bz2
+27ad5e2ca3b0abd00be74ee707ef9e53  pypy-2.5.1-linux-armhf-raspbian.tar.bz2
+ca07245e27417034a786365947022eb5  pypy-2.5.1-linux.tar.bz2
+c26e06f3de54fdaaaf1830fb7ca99b70  pypy-2.5.1-osx64.tar.bz2
+de4da75efe3e3b1325861c8883504fdc  pypy-2.5.1-src.tar.bz2
+3b309573ea7ec0835dc922a5940a4bdc  pypy-2.5.1-src.zip
+99a77a5610f6a4941ea310d01933e71f  pypy-2.5.1-win32.zip
+
    +

    pypy3-2.4.0 md5:

    +
     eadbc9790823fc0ae40c943087cd7cb3  pypy3-2.4.0-linux64.tar.bz2
 7ab84727da2d5363866907f2f7921d86  pypy3-2.4.0-linux-armel.tar.bz2
 83158d3a55ca134b179ef01dc2bb6a30  pypy3-2.4.0-linux-armhf-raring.tar.bz2
@@ -322,16 +325,26 @@
 8514f16b1a6262828e824bd8b37607db  pypy3-2.4.0-win32.zip
 96ba72916114d16904e12562b5d84e51  pypy3-2.4.0-src.tar.bz2
 c58015d0d3e08a9f24b93b8edca26d4d  pypy3-2.4.0-src.zip
+
    +

    pypy-1.8 sandbox md5:

    +
     2c9f0054f3b93a6473f10be35277825a  pypy-1.8-sandbox-linux64.tar.bz2
 009c970b5fa75754ae4c32a5d108a8d4  pypy-1.8-sandbox-linux.tar.bz2
-303bdc29e80640e337f1d65c7787870738a3ebf5  pypy-2.5.0-linux64.tar.bz2
-1bc8de2db37233051d84c9c27e1e64608283b67c  pypy-2.5.0-linux-armel.tar.bz2
-01427d7dd52e2b679c38bfad414ebe676b019ea0  pypy-2.5.0-linux-armhf-raring.tar.bz2
-e3483d52182ab3ee02f665fda02f9fac4ab40ab1  pypy-2.5.0-linux-armhf-raspbian.tar.bz2
-7334013255670f4ae60b316a5b35499861e0714f  pypy-2.5.0-linux.tar.bz2
-ad47285526b1b3c14f4eecc874bb82a133a8e551  pypy-2.5.0-osx64.tar.bz2
-1d215a22ea16581de338700d556b21a8c02b4eff  pypy-2.5.0-src.tar.bz2
-3492aa6fe96987537b23d09d66bb0c52f1760652  pypy-2.5.0-win32.zip
+
    +

    pypy-2.5.1 sha1:

    +
    +e598559cdf819707d8f89b31605118385323d8e4  pypy-2.5.1-linux64.tar.bz2
+747d75b36960788692a64a652b6397b2ccdda227  pypy-2.5.1-linux-armel.tar.bz2
+f142697aadc5c7dbe9099331668b2f3f851f51a2  pypy-2.5.1-linux-armhf-raring.tar.bz2
+110bd34f0a648dc0b4e3bd80d72ce953276ea54f  pypy-2.5.1-linux-armhf-raspbian.tar.bz2
+97962ccaa3d7eecff95d71abea3514491563d59f  pypy-2.5.1-linux.tar.bz2
+1daf39a6fafa757c7a96189b21dac40071db2284  pypy-2.5.1-osx64.tar.bz2
+e642ad3968e40399cf1989e7b6c70860a5675a65  pypy-2.5.1-src.tar.bz2
+df0ef936ba0e689e3ed9729f1f0569b91d8e0088  pypy-2.5.1-src.zip
+4af985fad28e4eb7d7400c7475acee65ddf3ebcc  pypy-2.5.1-win32.zip
+
    +

    pypy3-2.4.0 sha1:

    +
     7d715742f6929351b310a2ca3b924cab35913089  pypy3-2.4.0-linux64.tar.bz2
 b33e817f3557f91c434032c9f74e5220fe70036c  pypy3-2.4.0-linux-armel.tar.bz2
 bb098b72ecc83a0e73c426f364bb6a0974fb9360  pypy3-2.4.0-linux-armhf-raring.tar.bz2
@@ -341,6 +354,9 @@
 2efca54daa4c5da23ef577d70006376e13cfb6b8  pypy3-2.4.0-win32.zip
 438572443ae6f54eb6122d807f104787c5247e01  pypy3-2.4.0-src.tar.bz2
 bd0a91cdda965f42c382bf00f4a2fb8677b929a6  pypy3-2.4.0-src.zip
+
    +

    pypy-1.8 sandbox sha1:

    +
     895aaf7bba5787dd30adda5cc0e0e7fc297c0ca7  pypy-1.8-sandbox-linux64.tar.bz2
 be94460bed8b2682880495435c309b6611ae2c31  pypy-1.8-sandbox-linux.tar.bz2
    diff --git a/source/download.txt b/source/download.txt --- a/source/download.txt +++ b/source/download.txt @@ -10,16 +10,14 @@ There are `nightly binary builds`_ available. Those builds are not always as stable as the release, but they contain numerous bugfixes and - performance improvements. **Note that the OS X nightly builds - (but not the release) are slightly - miscompiled due to buildslave being old. Contributions are welcomed**. + performance improvements. We provide binaries for x86 and ARM Linux, Mac OS/X and Windows for: -* the Python2.7 compatible release — **PyPy 2.5.0** — (`what's new in PyPy 2.5.0?`_ ) +* the Python2.7 compatible release — **PyPy 2.5.1** — (`what's new in PyPy 2.5.1?`_ ) * the Python3.2.5 compatible release — **PyPy3 2.4.0** — (`what's new in PyPy3 2.4.0?`_). -.. _what's new in PyPy 2.5.0?: http://doc.pypy.org/en/latest/release-2.5.0.html +.. _what's new in PyPy 2.5.1?: http://doc.pypy.org/en/latest/release-2.5.1.html .. _what's new in PyPy3 2.4.0?: http://doc.pypy.org/en/latest/release-pypy3-2.4.0.html @@ -71,7 +69,7 @@ .. _`portable Linux binaries`: https://github.com/squeaky-pl/portable-pypy -Python2.7 compatible PyPy 2.5.0 +Python2.7 compatible PyPy 2.5.1 ----------------------------------- * `Linux x86 binary (32bit, tar.bz2 built on Ubuntu 12.04 - 14.04)`__ (see ``[1]`` below) @@ -80,22 +78,22 @@ * `ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Ubuntu Raring)`__ (see ``[1]`` below) * `ARM Softfloat Linux binary (ARMEL/gnueabi, tar.bz2, Ubuntu Precise)`__ (see ``[1]`` below) * `Mac OS/X binary (64bit)`__ -* `Windows binary (32bit)`__ (you might need the `VS 2008 runtime library - installer vcredist_x86.exe`_.) +* `Windows binary (32bit)`__ (you might need the VS 2008 runtime library + installer `vcredist_x86.exe`_.) * `Source (tar.bz2)`__; `Source (zip)`__. See below for more about the sources. * `All our downloads,`__ including previous versions. We also have a mirror_, but please use only if you have troubles accessing the links above -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.0-linux.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.0-linux64.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.0-linux-armhf-raspbian.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.0-linux-armhf-raring.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.0-linux-armel.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.0-osx64.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.0-win32.zip -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.0-src.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.0-src.zip -.. _`VS 2008 runtime library installer vcredist_x86.exe`: http://www.microsoft.com/en-us/download/details.aspx?id=5582 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.1-linux.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.1-linux64.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.1-linux-armhf-raspbian.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.1-linux-armhf-raring.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.1-linux-armel.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.1-osx64.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.1-win32.zip +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.1-src.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.1-src.zip +.. _`vcredist_x86.exe`: http://www.microsoft.com/en-us/download/details.aspx?id=5582 .. __: https://bitbucket.org/pypy/pypy/downloads .. _mirror: http://cobra.cs.uni-duesseldorf.de/~buildmaster/mirror/ @@ -147,7 +145,7 @@ The other versions of PyPy are: -* The most up-to-date `nightly build`_ with a JIT, if the official +* The most up-to-date `nightly binary builds`_ with a JIT, if the official release is too old for what you want to do. There are versions for different libc on this site too. @@ -167,7 +165,7 @@ .. __: https://bitbucket.org/pypy/pypy/downloads/pypy-1.8-sandbox-linux.tar.bz2 .. _`sandbox docs`: http://doc.pypy.org/en/latest/sandbox.html -.. _`nightly build`: http://buildbot.pypy.org/nightly/trunk/ +.. _`nightly binary builds`: http://buildbot.pypy.org/nightly/trunk/ Installing ------------------------------- @@ -176,7 +174,7 @@ uncompressed, they run in-place. For now you can uncompress them either somewhere in your home directory or, say, in ``/opt``, and if you want, put a symlink from somewhere like -``/usr/local/bin/pypy`` to ``/path/to/pypy-2.5.0/bin/pypy``. Do +``/usr/local/bin/pypy`` to ``/path/to/pypy-2.5.1/bin/pypy``. Do not move or copy the executable ``pypy`` outside the tree --- put a symlink to it, otherwise it will not find its libraries. @@ -232,9 +230,9 @@ 1. Get the source code. The following packages contain the source at the same revision as the above binaries: - * `pypy-2.5.0-src.tar.bz2`__ (sources) + * `pypy-2.5.1-src.tar.bz2`__ (sources) - .. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.0-src.tar.bz2 + .. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.1-src.tar.bz2 Or you can checkout the current trunk using Mercurial_ (the trunk usually works and is of course more up-to-date):: @@ -273,7 +271,7 @@ then note that we are talking about CPython 2.5-2.7 here, not CPython 3.x. * If RAM usage is a problem (or if you are on Windows, because win32's limit - is 2 GB unless you hack a lot), then you can (for now) tweak some parameters + is 2 GB unless you `have a 64 bit OS`_), then you can (for now) tweak some parameters via environment variables and command-line options. The following command takes a bit more time, but finishes with only using 3.0 GB of RAM (on Linux 64-bit; probably not much more than 1.6 GB on 32-bit). It should be @@ -296,10 +294,9 @@ .. _`sandboxing`: features.html#sandboxing .. _`stackless`: http://www.stackless.com/ .. _`greenlets`: http://pypy.readthedocs.org/en/latest/stackless.html#greenlets +.. _`have a 64 bit OS`: http://doc.pypy.org/en/latest/windows.html#preparing-windows-for-the-large-build +.. _`shadow stack`: http://pypy.readthedocs.org/en/latest/config/translation.gcrootfinder.html .. _Mercurial: http://mercurial.selenic.com/ -.. _`nightly binary builds`: http://buildbot.pypy.org/nightly/trunk/ -.. _`shadow stack`: http://pypy.readthedocs.org/en/latest/config/translation.gcrootfinder.html - Packaging --------- @@ -325,18 +322,21 @@ Checksums --------- -Here are the checksums for each of the downloads (md5 and sha1):: +Here are the checksums for each of the downloads +pypy-2.5.1 md5:: + b3cc9f8a419f9f89c3fac34b39e92e0a pypy-2.5.1-linux64.tar.bz2 + fe663500fb87d251ebf02917d25dca23 pypy-2.5.1-linux-armel.tar.bz2 + 7a0f845baec7a6ccfb57a66f0e7980e9 pypy-2.5.1-linux-armhf-raring.tar.bz2 + 27ad5e2ca3b0abd00be74ee707ef9e53 pypy-2.5.1-linux-armhf-raspbian.tar.bz2 + ca07245e27417034a786365947022eb5 pypy-2.5.1-linux.tar.bz2 + c26e06f3de54fdaaaf1830fb7ca99b70 pypy-2.5.1-osx64.tar.bz2 + de4da75efe3e3b1325861c8883504fdc pypy-2.5.1-src.tar.bz2 + 3b309573ea7ec0835dc922a5940a4bdc pypy-2.5.1-src.zip + 99a77a5610f6a4941ea310d01933e71f pypy-2.5.1-win32.zip - 0a109c96bbce20931c17dfded57a3d46 pypy-2.5.0-linux64.tar.bz2 - fc79e0ef640371fa7bdbb040c3211bcf pypy-2.5.0-linux-armel.tar.bz2 - d8d0fe8a1503a7ed4ec5ff52d0630b88 pypy-2.5.0-linux-armhf-raring.tar.bz2 - 9a4430abd36dd619c45705c3fe70f5ca pypy-2.5.0-linux-armhf-raspbian.tar.bz2 - 1c14e954fdd1d112540790ec5b0cea26 pypy-2.5.0-linux.tar.bz2 - 9916a1485f32953a46afdd36f8f01f83 pypy-2.5.0-osx64.tar.bz2 - f4700c0af45e986178b36ce91a45136e pypy-2.5.0-src.tar.bz2 - 2461a4e4fe4ea64ce7fb1a07aa310e6a pypy-2.5.0-win32.zip +pypy3-2.4.0 md5:: eadbc9790823fc0ae40c943087cd7cb3 pypy3-2.4.0-linux64.tar.bz2 7ab84727da2d5363866907f2f7921d86 pypy3-2.4.0-linux-armel.tar.bz2 @@ -348,17 +348,25 @@ 96ba72916114d16904e12562b5d84e51 pypy3-2.4.0-src.tar.bz2 c58015d0d3e08a9f24b93b8edca26d4d pypy3-2.4.0-src.zip + +pypy-1.8 sandbox md5:: + 2c9f0054f3b93a6473f10be35277825a pypy-1.8-sandbox-linux64.tar.bz2 009c970b5fa75754ae4c32a5d108a8d4 pypy-1.8-sandbox-linux.tar.bz2 - 303bdc29e80640e337f1d65c7787870738a3ebf5 pypy-2.5.0-linux64.tar.bz2 - 1bc8de2db37233051d84c9c27e1e64608283b67c pypy-2.5.0-linux-armel.tar.bz2 - 01427d7dd52e2b679c38bfad414ebe676b019ea0 pypy-2.5.0-linux-armhf-raring.tar.bz2 - e3483d52182ab3ee02f665fda02f9fac4ab40ab1 pypy-2.5.0-linux-armhf-raspbian.tar.bz2 - 7334013255670f4ae60b316a5b35499861e0714f pypy-2.5.0-linux.tar.bz2 - ad47285526b1b3c14f4eecc874bb82a133a8e551 pypy-2.5.0-osx64.tar.bz2 - 1d215a22ea16581de338700d556b21a8c02b4eff pypy-2.5.0-src.tar.bz2 - 3492aa6fe96987537b23d09d66bb0c52f1760652 pypy-2.5.0-win32.zip +pypy-2.5.1 sha1:: + + e598559cdf819707d8f89b31605118385323d8e4 pypy-2.5.1-linux64.tar.bz2 + 747d75b36960788692a64a652b6397b2ccdda227 pypy-2.5.1-linux-armel.tar.bz2 + f142697aadc5c7dbe9099331668b2f3f851f51a2 pypy-2.5.1-linux-armhf-raring.tar.bz2 + 110bd34f0a648dc0b4e3bd80d72ce953276ea54f pypy-2.5.1-linux-armhf-raspbian.tar.bz2 + 97962ccaa3d7eecff95d71abea3514491563d59f pypy-2.5.1-linux.tar.bz2 + 1daf39a6fafa757c7a96189b21dac40071db2284 pypy-2.5.1-osx64.tar.bz2 + e642ad3968e40399cf1989e7b6c70860a5675a65 pypy-2.5.1-src.tar.bz2 + df0ef936ba0e689e3ed9729f1f0569b91d8e0088 pypy-2.5.1-src.zip + 4af985fad28e4eb7d7400c7475acee65ddf3ebcc pypy-2.5.1-win32.zip + +pypy3-2.4.0 sha1:: 7d715742f6929351b310a2ca3b924cab35913089 pypy3-2.4.0-linux64.tar.bz2 b33e817f3557f91c434032c9f74e5220fe70036c pypy3-2.4.0-linux-armel.tar.bz2 @@ -370,5 +378,7 @@ 438572443ae6f54eb6122d807f104787c5247e01 pypy3-2.4.0-src.tar.bz2 bd0a91cdda965f42c382bf00f4a2fb8677b929a6 pypy3-2.4.0-src.zip +pypy-1.8 sandbox sha1:: + 895aaf7bba5787dd30adda5cc0e0e7fc297c0ca7 pypy-1.8-sandbox-linux64.tar.bz2 be94460bed8b2682880495435c309b6611ae2c31 pypy-1.8-sandbox-linux.tar.bz2 From noreply at buildbot.pypy.org Thu Mar 26 08:00:54 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 26 Mar 2015 08:00:54 +0100 (CET) Subject: [pypy-commit] stmgc default: Improve the gdb tools, design them for stmgc specifically Message-ID: <20150326070054.6977B1C1156@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1741:e3c5158f015b Date: 2015-03-26 08:01 +0100 http://bitbucket.org/pypy/stmgc/changeset/e3c5158f015b/ Log: Improve the gdb tools, design them for stmgc specifically diff --git a/c7/gdb/gdb_stm.py b/c7/gdb/old_gdb_stm.py rename from c7/gdb/gdb_stm.py rename to c7/gdb/old_gdb_stm.py diff --git a/c7/stm/core.c b/c7/stm/core.c --- a/c7/stm/core.c +++ b/c7/stm/core.c @@ -2,6 +2,11 @@ # error "must be compiled via stmgc.c" #endif +char *stm_object_pages; +long _stm_segment_nb_pages = NB_PAGES; +int _stm_nb_segments = NB_SEGMENTS; +int _stm_psegment_ofs = (int)(uintptr_t)STM_PSEGMENT; + static void teardown_core(void) { diff --git a/c7/stm/core.h b/c7/stm/core.h --- a/c7/stm/core.h +++ b/c7/stm/core.h @@ -232,7 +232,10 @@ TS_INEVITABLE, }; -static char *stm_object_pages; +extern char *stm_object_pages; +extern long _stm_segment_nb_pages; +extern int _stm_nb_segments; +extern int _stm_psegment_ofs; static int stm_object_pages_fd; static stm_thread_local_t *stm_all_thread_locals = NULL; From noreply at buildbot.pypy.org Thu Mar 26 08:01:54 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 26 Mar 2015 08:01:54 +0100 (CET) Subject: [pypy-commit] stmgc default: Forgot "hg add" Message-ID: <20150326070154.158D81C1156@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1742:84157d77ae80 Date: 2015-03-26 08:02 +0100 http://bitbucket.org/pypy/stmgc/changeset/84157d77ae80/ Log: Forgot "hg add" diff --git a/c7/gdb/gdb_stm.py b/c7/gdb/gdb_stm.py new file mode 100644 --- /dev/null +++ b/c7/gdb/gdb_stm.py @@ -0,0 +1,114 @@ +"""Adds a few built-in functions for stmgc-c7: + + $gc(p=0, thread='current') + + With an integer argument `p`, returns `(char *)(segment_base + p)`. + If `p` is a pointer type, it is assumed to be %gs-relative; the + function returns `*p` in the segment. + + The segment is by default the current one, as computed by looking + around in the debug information. You can force with the second + argment a specific segment number or a specific pthread_self() + thread address. + + $psegment(thread='current') + + Return the 'stm_priv_segment_info_t *' of the given thread + (given as a segment number or a pthread_self() thread address). + +Usage: you can for example add this line in your ~/.gdbinit: + + python exec(open('/path/to/gdb_stm.py').read()) +""" +import gdb + +def gdb_function(func): + class Func(gdb.Function): + __doc__ = func.__doc__ + invoke = staticmethod(func) + Func(func.__name__) + +# ------------------------------------------------------- + +_nb_segments = None +_segment_size = None +_psegment_ofs = None + +def get_nb_segments(): + global _nb_segments + if _nb_segments is None: + _nb_segments = int(gdb.parse_and_eval('_stm_nb_segments')) + assert 1 < _nb_segments <= 240 + return _nb_segments + +def get_segment_size(): + global _segment_size + if _segment_size is None: + nb_pages = int(gdb.parse_and_eval('_stm_segment_nb_pages')) + _segment_size = nb_pages * 4096 + return _segment_size + +def get_psegment_ofs(): + global _psegment_ofs + if _psegment_ofs is None: + _psegment_ofs = int(gdb.parse_and_eval('_stm_psegment_ofs')) + return _psegment_ofs + +def get_segment_base(segment_id): + assert 0 <= segment_id <= get_nb_segments() + base = int(gdb.parse_and_eval('stm_object_pages')) + return base + get_segment_size() * segment_id + +def get_psegment(segment_id, field=''): + assert 0 < segment_id <= get_nb_segments() + return gdb.parse_and_eval( + '((struct stm_priv_segment_info_s *)(stm_object_pages+%d))%s' + % (get_segment_size() * segment_id + get_psegment_ofs(), field)) + +def thread_to_segment_id(thread_id): + base = int(gdb.parse_and_eval('stm_object_pages')) + for j in range(1, get_nb_segments() + 1): + ts = get_psegment(j, '->transaction_state') + if int(ts) != 0: + ti = get_psegment(j, '->pub.running_thread->creating_pthread[0]') + if int(ti) == thread_id: + return j + raise Exception("thread not found: %r" % (thread_id,)) + +def interactive_segment_base(thread=None): + if thread is None: + s = gdb.execute('info threads', False, True) + i = s.find('\n* ') + assert i >= 0 + fields = s[i+2:].split() + assert fields[1] == 'Thread' + assert fields[2].startswith('0x') + thread_id = int(fields[2], 16) + segment_id = thread_to_segment_id(thread_id) + elif thread.type.code == gdb.TYPE_CODE_INT: + if 0 <= int(thread) < 256: + segment_id = int(thread) + else: + thread_id = int(thread) + segment_id = thread_to_segment_id(thread_id) + else: + raise TypeError("'thread' argument must be an int or not given") + return get_segment_base(segment_id) + + at gdb_function +def gc(p=None, thread=None): + sb = interactive_segment_base(thread) + if p is not None and p.type.code == gdb.TYPE_CODE_PTR: + return gdb.Value(sb + int(p)).cast(p.type).dereference() + elif p is None or int(p) == 0: + T = gdb.lookup_type('char').pointer() + return gdb.Value(sb).cast(T) + else: + raise TypeError("gc() first argument must be a GC pointer or 0") + + at gdb_function +def psegment(thread=None): + sb = interactive_segment_base(thread) + return gdb.parse_and_eval( + '*((struct stm_priv_segment_info_s *)%d)' + % (sb + get_psegment_ofs(),)) From noreply at buildbot.pypy.org Thu Mar 26 09:42:02 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 26 Mar 2015 09:42:02 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: A possible transaction break must reset the heapcache. Message-ID: <20150326084202.848081C0994@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76566:034b2f461fb9 Date: 2015-03-26 09:22 +0100 http://bitbucket.org/pypy/pypy/changeset/034b2f461fb9/ Log: A possible transaction break must reset the heapcache. diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -214,6 +214,7 @@ resbox = history.BoxInt(0) record_break = True + self.metainterp.heapcache.reset() rstm.possible_transaction_break(0) if record_break: From noreply at buildbot.pypy.org Thu Mar 26 09:42:03 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 26 Mar 2015 09:42:03 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: import stmgc/84157d77ae80 Message-ID: <20150326084203.BC5AC1C0994@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76567:5075882bb14f Date: 2015-03-26 09:27 +0100 http://bitbucket.org/pypy/pypy/changeset/5075882bb14f/ Log: import stmgc/84157d77ae80 diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision --- a/rpython/translator/stm/src_stm/revision +++ b/rpython/translator/stm/src_stm/revision @@ -1,1 +1,1 @@ -a0b5046a7bea +84157d77ae80 diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c --- a/rpython/translator/stm/src_stm/stm/core.c +++ b/rpython/translator/stm/src_stm/stm/core.c @@ -2,6 +2,11 @@ #ifndef _STM_CORE_H_ # error "must be compiled via stmgc.c" #endif +char *stm_object_pages; +long _stm_segment_nb_pages = NB_PAGES; +int _stm_nb_segments = NB_SEGMENTS; +int _stm_psegment_ofs = (int)(uintptr_t)STM_PSEGMENT; + static void teardown_core(void) { diff --git a/rpython/translator/stm/src_stm/stm/core.h b/rpython/translator/stm/src_stm/stm/core.h --- a/rpython/translator/stm/src_stm/stm/core.h +++ b/rpython/translator/stm/src_stm/stm/core.h @@ -232,7 +232,10 @@ TS_INEVITABLE, }; -static char *stm_object_pages; +extern char *stm_object_pages; +extern long _stm_segment_nb_pages; +extern int _stm_nb_segments; +extern int _stm_psegment_ofs; static int stm_object_pages_fd; static stm_thread_local_t *stm_all_thread_locals = NULL; From noreply at buildbot.pypy.org Thu Mar 26 09:42:04 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 26 Mar 2015 09:42:04 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Most probably, that's enough Message-ID: <20150326084204.F3DE01C0994@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76568:986a2a3e0532 Date: 2015-03-26 09:41 +0100 http://bitbucket.org/pypy/pypy/changeset/986a2a3e0532/ Log: Most probably, that's enough diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -214,7 +214,7 @@ resbox = history.BoxInt(0) record_break = True - self.metainterp.heapcache.reset() + self.metainterp.heapcache.reset_keep_likely_virtuals() rstm.possible_transaction_break(0) if record_break: From noreply at buildbot.pypy.org Thu Mar 26 10:21:42 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 26 Mar 2015 10:21:42 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: Technical-report-ize this paper. Message-ID: <20150326092142.D63181C135C@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r5514:7d300824728c Date: 2015-03-26 10:22 +0100 http://bitbucket.org/pypy/extradoc/changeset/7d300824728c/ Log: Technical-report-ize this paper. diff --git a/talk/dls2014/paper/paper.pdf b/talk/dls2014/paper/paper.pdf index 82f177e29577ffb3c99b4b6829fd6e829fe20d59..95c19801dd7b1230598296a3b5b634cb73e325e6 GIT binary patch [cut] diff --git a/talk/dls2014/paper/sigplanconf.cls b/talk/dls2014/paper/sigplanconf.cls --- a/talk/dls2014/paper/sigplanconf.cls +++ b/talk/dls2014/paper/sigplanconf.cls @@ -602,12 +602,14 @@ \def \@toappear {#1}} \toappear{% - \noindent \@permission \par - \vspace{2pt} - \noindent \textsl{\@conferencename}, \quad \@conferenceinfo. \par - \noindent Copyright \copyright\ \@copyrightyear\ ACM \@copyrightdata - \dots \@reprintprice.\par - \noindent http://dx.doi.org/10.1145/\@doi } + Technical report, June-August 2014 +% \noindent \@permission \par +% \vspace{2pt} +% \noindent \textsl{\@conferencename}, \quad \@conferenceinfo. \par +% \noindent Copyright \copyright\ \@copyrightyear\ ACM \@copyrightdata +% \dots \@reprintprice.\par +% \noindent http://dx.doi.org/10.1145/\@doi } + } \newcommand{\permission}[1]{% \gdef \@permission {#1}} From noreply at buildbot.pypy.org Thu Mar 26 10:26:25 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 26 Mar 2015 10:26:25 +0100 (CET) Subject: [pypy-commit] pypy default: Mention the technical report Message-ID: <20150326092625.AE2211C0300@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76569:5e232e841e3c Date: 2015-03-26 10:26 +0100 http://bitbucket.org/pypy/pypy/changeset/5e232e841e3c/ Log: Mention the technical report diff --git a/pypy/doc/stm.rst b/pypy/doc/stm.rst --- a/pypy/doc/stm.rst +++ b/pypy/doc/stm.rst @@ -614,6 +614,18 @@ XXX this section mostly empty for now +Technical reports +----------------- + +STMGC-C7 is described in detail in a `technical report`__. + +A separate `position paper`__ gives an overview of our position about +STM in general. + +.. __: http://bitbucket.org/pypy/extradoc/src/extradoc/talk/dls2014/paper/paper.pdf +.. __: http://bitbucket.org/pypy/extradoc/src/extradoc/talk/icooolps2014/ + + Reference to implementation details ----------------------------------- From noreply at buildbot.pypy.org Thu Mar 26 11:01:13 2015 From: noreply at buildbot.pypy.org (youknowone) Date: Thu, 26 Mar 2015 11:01:13 +0100 (CET) Subject: [pypy-commit] pypy default: issue #2005 Message-ID: <20150326100113.8101C1C0994@cobra.cs.uni-duesseldorf.de> Author: Jeong YunWon Branch: Changeset: r76570:920fce8456cb Date: 2015-03-25 04:10 +0900 http://bitbucket.org/pypy/pypy/changeset/920fce8456cb/ Log: issue #2005 After fix ``` $ ./pypy-c ../pypytest/bar.py $ ``` Before fix ``` $ pypy ../pypytest/bar.py RPython traceback: ... (See #2005 for details) ``` CPython ``` $ python ../pypytest/bar.py $ ``` diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py --- a/pypy/module/imp/importing.py +++ b/pypy/module/imp/importing.py @@ -663,7 +663,10 @@ if find_info: stream = find_info.stream if stream: - stream.close() + try: + stream.close() + except StreamErrors: + pass if tentative: return None From noreply at buildbot.pypy.org Thu Mar 26 11:01:14 2015 From: noreply at buildbot.pypy.org (youknowone) Date: Thu, 26 Mar 2015 11:01:14 +0100 (CET) Subject: [pypy-commit] pypy default: Similar cases in #2005 Message-ID: <20150326100114.B40A11C0994@cobra.cs.uni-duesseldorf.de> Author: Jeong YunWon Branch: Changeset: r76571:806f912430a0 Date: 2015-03-25 18:28 +0900 http://bitbucket.org/pypy/pypy/changeset/806f912430a0/ Log: Similar cases in #2005 diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py --- a/pypy/module/imp/importing.py +++ b/pypy/module/imp/importing.py @@ -621,7 +621,10 @@ try: load_module(space, w_modulename, find_info, reuse=True) finally: - find_info.stream.close() + try: + find_info.stream.close() + except StreamErrors: + pass # fetch the module again, in case of "substitution" w_mod = check_sys_modules(space, w_modulename) return w_mod @@ -884,7 +887,10 @@ try: code_w = read_compiled_module(space, cpathname, stream.readall()) finally: - stream.close() + try: + stream.close() + except StreamErrors: + pass space.setattr(w_mod, w('__file__'), w(cpathname)) else: code_w = parse_source_module(space, pathname, source) @@ -969,7 +975,10 @@ return stream except StreamErrors: if stream: - stream.close() + try: + stream.close() + except StreamErrors: + pass return None # XXX! must not eat all exceptions, e.g. # Out of file descriptors. From noreply at buildbot.pypy.org Thu Mar 26 11:01:15 2015 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Mar 2015 11:01:15 +0100 (CET) Subject: [pypy-commit] pypy default: Merged in youknowone/pypy (pull request #312) Message-ID: <20150326100115.D29371C0994@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r76572:5eb7a0303b1b Date: 2015-03-26 12:01 +0200 http://bitbucket.org/pypy/pypy/changeset/5eb7a0303b1b/ Log: Merged in youknowone/pypy (pull request #312) issue #2005 diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py --- a/pypy/module/imp/importing.py +++ b/pypy/module/imp/importing.py @@ -621,7 +621,10 @@ try: load_module(space, w_modulename, find_info, reuse=True) finally: - find_info.stream.close() + try: + find_info.stream.close() + except StreamErrors: + pass # fetch the module again, in case of "substitution" w_mod = check_sys_modules(space, w_modulename) return w_mod @@ -663,7 +666,10 @@ if find_info: stream = find_info.stream if stream: - stream.close() + try: + stream.close() + except StreamErrors: + pass if tentative: return None @@ -881,7 +887,10 @@ try: code_w = read_compiled_module(space, cpathname, stream.readall()) finally: - stream.close() + try: + stream.close() + except StreamErrors: + pass space.setattr(w_mod, w('__file__'), w(cpathname)) else: code_w = parse_source_module(space, pathname, source) @@ -966,7 +975,10 @@ return stream except StreamErrors: if stream: - stream.close() + try: + stream.close() + except StreamErrors: + pass return None # XXX! must not eat all exceptions, e.g. # Out of file descriptors. From noreply at buildbot.pypy.org Thu Mar 26 12:18:24 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 26 Mar 2015 12:18:24 +0100 (CET) Subject: [pypy-commit] pypy default: rst format Message-ID: <20150326111824.738E01C1156@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76573:8f539ca017fe Date: 2015-03-26 12:18 +0100 http://bitbucket.org/pypy/pypy/changeset/8f539ca017fe/ Log: rst format diff --git a/pypy/doc/whatsnew-2.5.1.rst b/pypy/doc/whatsnew-2.5.1.rst --- a/pypy/doc/whatsnew-2.5.1.rst +++ b/pypy/doc/whatsnew-2.5.1.rst @@ -14,27 +14,34 @@ .. branch: vmprof .. branch: stackroot-speedup-2 + Avoid tracing all stack roots during repeated minor collections, by ignoring the part of the stack that didn't change .. branch: stdlib-2.7.9 + Update stdlib to version 2.7.9 .. branch: fix-kqueue-error2 + Fix exception being raised by kqueue.control (CPython compatibility) .. branch: gitignore .. branch: framestate2 + Refactor rpython.flowspace.framestate.FrameState. .. branch: alt_errno + Add an alternative location to save LastError, errno around ctypes, cffi external calls so things like pdb will not overwrite it .. branch: nonquadratic-heapcache + Speed up the warmup times of the JIT by removing a quadratic algorithm in the heapcache. .. branch: online-transforms-2 + Simplify flow graphs on the fly during annotation phase. From noreply at buildbot.pypy.org Thu Mar 26 13:04:09 2015 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Mar 2015 13:04:09 +0100 (CET) Subject: [pypy-commit] pypy vmprof: deintent Message-ID: <20150326120409.7CF8E1C0F98@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76574:42b97d2e3697 Date: 2015-03-26 14:04 +0200 http://bitbucket.org/pypy/pypy/changeset/42b97d2e3697/ Log: deintent diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c --- a/pypy/module/_vmprof/src/vmprof.c +++ b/pypy/module/_vmprof/src/vmprof.c @@ -81,7 +81,7 @@ prof_word(depth); for(i=0; i Author: Carl Friedrich Bolz Branch: Changeset: r76575:04d55a9be4ce Date: 2015-03-26 13:23 +0100 http://bitbucket.org/pypy/pypy/changeset/04d55a9be4ce/ Log: the stack effect in the bytecode compiler of UNPACK_SEQUENCE was wrong (!) diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py --- a/pypy/interpreter/astcompiler/assemble.py +++ b/pypy/interpreter/astcompiler/assemble.py @@ -648,7 +648,7 @@ def _compute_UNPACK_SEQUENCE(arg): - return arg + 1 + return arg - 1 def _compute_DUP_TOPX(arg): return arg diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py --- a/pypy/interpreter/astcompiler/test/test_compiler.py +++ b/pypy/interpreter/astcompiler/test/test_compiler.py @@ -759,6 +759,19 @@ """ self.simple_test(source, 'l', [1, 2]) + def test_unpack_wrong_stackeffect(self): + source = """if 1: + l = [1, 2] + a, b = l + a, b = l + a, b = l + a, b = l + a, b = l + a, b = l + """ + code = compile_with_astcompiler(source, 'exec', self.space) + assert code.co_stacksize == 2 + def test_lambda(self): yield self.st, "y = lambda x: x", "y(4)", 4 From noreply at buildbot.pypy.org Thu Mar 26 15:52:20 2015 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 26 Mar 2015 15:52:20 +0100 (CET) Subject: [pypy-commit] pypy vmprof: close this file Message-ID: <20150326145220.0FBCD1C0484@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76576:e58a3bc4935d Date: 2015-03-26 16:52 +0200 http://bitbucket.org/pypy/pypy/changeset/e58a3bc4935d/ Log: close this file diff --git a/rpython/bin/rpython-vmprof b/rpython/bin/rpython-vmprof --- a/rpython/bin/rpython-vmprof +++ b/rpython/bin/rpython-vmprof @@ -24,3 +24,4 @@ main() finally: _vmprof.disable() + x.close() From noreply at buildbot.pypy.org Thu Mar 26 16:54:29 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 26 Mar 2015 16:54:29 +0100 (CET) Subject: [pypy-commit] pypy default: Manually remove duplicate tags (likely old versions) Message-ID: <20150326155429.4F39A1C051C@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76577:242280da8873 Date: 2015-03-26 16:54 +0100 http://bitbucket.org/pypy/pypy/changeset/242280da8873/ Log: Manually remove duplicate tags (likely old versions) diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -3,20 +3,10 @@ d8ac7d23d3ec5f9a0fa1264972f74a010dbfd07f release-1.6 ff4af8f318821f7f5ca998613a60fca09aa137da release-1.7 07e08e9c885ca67d89bcc304e45a32346daea2fa release-2.0-beta-1 -9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm -9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm ab0dd631c22015ed88e583d9fdd4c43eebf0be21 pypy-2.1-beta1-arm 20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0 394146e9bb673514c61f0150ab2013ccf78e8de7 release-2.3 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1 10f1b29a2bd21f837090286174a9ca030b8680b2 release-2.5.0 -8e24dac0b8e2db30d46d59f2c4daa3d4aaab7861 release-2.5.1 -8e24dac0b8e2db30d46d59f2c4daa3d4aaab7861 release-2.5.1 -0000000000000000000000000000000000000000 release-2.5.1 -0000000000000000000000000000000000000000 release-2.5.1 -e3d046c43451403f5969580fc1c41d5df6c4082a release-2.5.1 -e3d046c43451403f5969580fc1c41d5df6c4082a release-2.5.1 -0000000000000000000000000000000000000000 release-2.5.1 -0000000000000000000000000000000000000000 release-2.5.1 9c4588d731b7fe0b08669bd732c2b676cb0a8233 release-2.5.1 From noreply at buildbot.pypy.org Thu Mar 26 16:57:11 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 26 Mar 2015 16:57:11 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: hg merge release-2.5.x (i.e. 2.5.1) Message-ID: <20150326155711.BFA6E1C0352@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76578:a20040a72e73 Date: 2015-03-26 16:55 +0100 http://bitbucket.org/pypy/pypy/changeset/a20040a72e73/ Log: hg merge release-2.5.x (i.e. 2.5.1) diff too long, truncating to 2000 out of 2301 lines diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -11,3 +11,8 @@ 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1 10f1b29a2bd21f837090286174a9ca030b8680b2 release-2.5.0 +8e24dac0b8e2db30d46d59f2c4daa3d4aaab7861 release-2.5.1 +8e24dac0b8e2db30d46d59f2c4daa3d4aaab7861 release-2.5.1 +0000000000000000000000000000000000000000 release-2.5.1 +0000000000000000000000000000000000000000 release-2.5.1 +e3d046c43451403f5969580fc1c41d5df6c4082a release-2.5.1 diff --git a/pypy/doc/release-2.5.1.rst b/pypy/doc/release-2.5.1.rst --- a/pypy/doc/release-2.5.1.rst +++ b/pypy/doc/release-2.5.1.rst @@ -67,7 +67,19 @@ `PyPy documentation`_ and we now have seperate `RPython documentation`_. Tell us what still isn't clear, or even better help us improve the documentation. -* We merged version 2.7.9 of python's stdlib +* We merged version 2.7.9 of python's stdlib. From the python release notice: + + * The entirety of Python 3.4's `ssl module`_ has been backported. + See `PEP 466`_ for justification. + + * HTTPS certificate validation using the system's certificate store is now + enabled by default. See `PEP 476`_ for details. + + * SSLv3 has been disabled by default in httplib and its reverse dependencies + due to the `POODLE attack`_. + + * The `ensurepip module`_ has been backported, which provides the pip + package manager in every Python 2.7 installation. See `PEP 477`_. * The garbage collector now ignores parts of the stack which did not change since the last collection, another performance boost @@ -84,6 +96,12 @@ .. _`PyPy documentation`: http://doc.pypy.org .. _`RPython documentation`: http://rpython.readthedocs.org +.. _`ssl module`: https://docs.python.org/3/library/ssl.html +.. _`PEP 466`: https://www.python.org/dev/peps/pep-0466 +.. _`PEP 476`: https://www.python.org/dev/peps/pep-0476 +.. _`PEP 477`: https://www.python.org/dev/peps/pep-0477 +.. _`POODLE attack`: https://www.imperialviolet.org/2014/10/14/poodle.html +.. _`ensurepip module`: https://docs.python.org/2/library/ensurepip.html .. _resolved: http://doc.pypy.org/en/latest/whatsnew-2.5.1.html Please try it out and let us know what you think. We welcome diff --git a/pypy/doc/stm.rst b/pypy/doc/stm.rst --- a/pypy/doc/stm.rst +++ b/pypy/doc/stm.rst @@ -46,13 +46,14 @@ multiple cores. * ``pypy-stm`` provides (but does not impose) a special API to the - user in the pure Python module `transaction`_. This module is based - on the lower-level module `pypystm`_, but also provides some + user in the pure Python module ``transaction``. This module is based + on the lower-level module ``pypystm``, but also provides some compatibily with non-STM PyPy's or CPython's. * Building on top of the way the GIL is removed, we will talk - about `Atomic sections, Transactions, etc.: a better way to write - parallel programs`_. + about `How to write multithreaded programs: the 10'000-feet view`_ + and `transaction.TransactionQueue`_. + Getting Started @@ -89,7 +90,7 @@ Current status (stmgc-c7) ------------------------- -* It seems to work fine, without crashing any more. Please `report +* **NEW:** It seems to work fine, without crashing any more. Please `report any crash`_ you find (or other bugs). * It runs with an overhead as low as 20% on examples like "richards". @@ -97,33 +98,47 @@ 2x for "translate.py"-- which we are still trying to understand. One suspect is our partial GC implementation, see below. +* **NEW:** the ``PYPYSTM`` environment variable and the + ``pypy/stm/print_stm_log.py`` script let you know exactly which + "conflicts" occurred. This is described in the section + `transaction.TransactionQueue`_ below. + +* **NEW:** special transaction-friendly APIs (like ``stmdict``), + described in the section `transaction.TransactionQueue`_ below. The + old API changed again, mostly moving to different modules. Sorry + about that. I feel it's a better idea to change the API early + instead of being stuck with a bad one later... + * Currently limited to 1.5 GB of RAM (this is just a parameter in `core.h`__ -- theoretically. In practice, increase it too much and clang crashes again). Memory overflows are not correctly handled; they cause segfaults. -* The JIT warm-up time improved recently but is still bad. In order to - produce machine code, the JIT needs to enter a special single-threaded - mode for now. This means that you will get bad performance results if - your program doesn't run for several seconds, where *several* can mean - *many.* When trying benchmarks, be sure to check that you have - reached the warmed state, i.e. the performance is not improving any - more. This should be clear from the fact that as long as it's - producing more machine code, ``pypy-stm`` will run on a single core. +* **NEW:** The JIT warm-up time improved again, but is still + relatively large. In order to produce machine code, the JIT needs + to enter "inevitable" mode. This means that you will get bad + performance results if your program doesn't run for several seconds, + where *several* can mean *many.* When trying benchmarks, be sure to + check that you have reached the warmed state, i.e. the performance + is not improving any more. * The GC is new; although clearly inspired by PyPy's regular GC, it misses a number of optimizations for now. Programs allocating large numbers of small objects that don't immediately die (surely a common - situation) suffer from these missing optimizations. + situation) suffer from these missing optimizations. (The bleeding + edge ``stmgc-c8`` is better at that.) * Weakrefs might appear to work a bit strangely for now, sometimes staying alive throught ``gc.collect()``, or even dying but then - un-dying for a short time before dying again. + un-dying for a short time before dying again. A similar problem can + show up occasionally elsewhere with accesses to some external + resources, where the (apparent) serialized order doesn't match the + underlying (multithreading) order. These are bugs (partially fixed + already in ``stmgc-c8``). * The STM system is based on very efficient read/write barriers, which are mostly done (their placement could be improved a bit in - JIT-generated machine code). But the overall bookkeeping logic could - see more improvements (see `Low-level statistics`_ below). + JIT-generated machine code). * Forking the process is slow because the complete memory needs to be copied manually. A warning is printed to this effect. @@ -132,7 +147,8 @@ crash on an assertion error because of a non-implemented overflow of an internal 28-bit counter. -.. _`report bugs`: https://bugs.pypy.org/ + +.. _`report any crash`: https://bitbucket.org/pypy/pypy/issues?status=new&status=open .. __: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/rpython/translator/stm/src_stm/stm/core.h @@ -155,10 +171,41 @@ interpreter and other ones might have slightly different needs. - User Guide ========== +How to write multithreaded programs: the 10'000-feet view +--------------------------------------------------------- + +PyPy-STM offers two ways to write multithreaded programs: + +* the traditional way, using the ``thread`` or ``threading`` modules, + described first__. + +* using ``TransactionQueue``, described next__, as a way to hide the + low-level notion of threads. + +.. __: `Drop-in replacement`_ +.. __: `transaction.TransactionQueue`_ + +The issue with low-level threads are well known (particularly in other +languages that don't have GIL-based interpreters): memory corruption, +deadlocks, livelocks, and so on. There are alternative approaches to +dealing directly with threads, like OpenMP_. These approaches +typically enforce some structure on your code. ``TransactionQueue`` +is in part similar: your program needs to have "some chances" of +parallelization before you can apply it. But I believe that the scope +of applicability is much larger with ``TransactionQueue`` than with +other approaches. It usually works without forcing a complete +reorganization of your existing code, and it works on any Python +program which has got *latent* and *imperfect* parallelism. Ideally, +it only requires that the end programmer identifies where this +parallelism is likely to be found, and communicates it to the system +using a simple API. + +.. _OpenMP: http://en.wikipedia.org/wiki/OpenMP + + Drop-in replacement ------------------- @@ -181,8 +228,8 @@ order. -A better way to write parallel programs ---------------------------------------- +transaction.TransactionQueue +---------------------------- In CPU-hungry programs, we can often easily identify outermost loops over some data structure, or other repetitive algorithm, where each @@ -216,41 +263,116 @@ behavior did not change because we are using ``TransactionQueue``. All the calls still *appear* to execute in some serial order. -Now the performance should ideally be improved: if the function calls -turn out to be actually independent (most of the time), then it will -be. But if the function calls are not, then the total performance -will crawl back to the previous case, with additionally some small -penalty for the overhead. +A typical usage of ``TransactionQueue`` goes like that: at first, +the performance does not increase. +In fact, it is likely to be worse. Typically, this is +indicated by the total CPU usage, which remains low (closer to 1 than +N cores). First note that it is expected that the CPU usage should +not go much higher than 1 in the JIT warm-up phase: you must run a +program for several seconds, or for larger programs at least one +minute, to give the JIT a chance to warm up enough. But if CPU usage +remains low even afterwards, then the ``PYPYSTM`` environment variable +can be used to track what is going on. -This case occurs typically when you see the total CPU usage remaining -low (closer to 1 than N cores). Note first that it is expected that -the CPU usage should not go much higher than 1 in the JIT warm-up -phase. You must run a program for several seconds, or for larger -programs at least one minute, to give the JIT a chance to warm up -correctly. But if CPU usage remains low even though all code is -executing in a ``TransactionQueue.run()``, then the ``PYPYSTM`` -environment variable can be used to track what is going on. +Run your program with ``PYPYSTM=logfile`` to produce a log file called +``logfile``. Afterwards, use the ``pypy/stm/print_stm_log.py`` +utility to inspect the content of this log file. It produces output +like this (sorted by amount of time lost, largest first):: -Run your program with ``PYPYSTM=stmlog`` to produce a log file called -``stmlog``. Afterwards, use the ``pypy/stm/print_stm_log.py`` utility -to inspect the content of this log file. It produces output like -this:: + 10.5s lost in aborts, 1.25s paused (12412x STM_CONTENTION_WRITE_WRITE) + File "foo.py", line 10, in f + someobj.stuff = 5 + File "bar.py", line 20, in g + someobj.other = 10 - documentation in progress! +This means that 10.5 seconds were lost running transactions that were +aborted (which caused another 1.25 seconds of lost time by pausing), +because of the reason shown in the two independent single-entry +tracebacks: one thread ran the line ``someobj.stuff = 5``, whereas +another thread concurrently ran the line ``someobj.other = 10`` on the +same object. These two writes are done to the same object. This +causes a conflict, which aborts one of the two transactions. In the +example above this occurred 12412 times. +The two other conflict sources are ``STM_CONTENTION_INEVITABLE``, +which means that two transactions both tried to do an external +operation, like printing or reading from a socket or accessing an +external array of raw data; and ``STM_CONTENTION_WRITE_READ``, which +means that one transaction wrote to an object but the other one merely +read it, not wrote to it (in that case only the writing transaction is +reported; the location for the reads is not recorded because doing so +is not possible without a very large performance impact). + +Common causes of conflicts: + +* First of all, any I/O or raw manipulation of memory turns the + transaction inevitable ("must not abort"). There can be only one + inevitable transaction running at any time. A common case is if + each transaction starts with sending data to a log file. You should + refactor this case so that it occurs either near the end of the + transaction (which can then mostly run in non-inevitable mode), or + delegate it to a separate transaction or even a separate thread. + +* Writing to a list or a dictionary conflicts with any read from the + same list or dictionary, even one done with a different key. For + dictionaries and sets, you can try the types ``transaction.stmdict`` + and ``transaction.stmset``, which behave mostly like ``dict`` and + ``set`` but allow concurrent access to different keys. (What is + missing from them so far is lazy iteration: for example, + ``stmdict.iterkeys()`` is implemented as ``iter(stmdict.keys())``; + and, unlike PyPy's dictionaries and sets, the STM versions are not + ordered.) There are also experimental ``stmiddict`` and + ``stmidset`` classes using the identity of the key. + +* ``time.time()`` and ``time.clock()`` turn the transaction inevitable + in order to guarantee that a call that appears to be later will + really return a higher number. If getting slightly unordered + results is fine, use ``transaction.time()`` or + ``transaction.clock()``. + +* ``transaction.threadlocalproperty`` can be used at class-level:: + + class Foo(object): # must be a new-style class! + x = transaction.threadlocalproperty() + y = transaction.threadlocalproperty(dict) + + This declares that instances of ``Foo`` have two attributes ``x`` + and ``y`` that are thread-local: reading or writing them from + concurrently-running transactions will return independent results. + (Any other attributes of ``Foo`` instances will be globally visible + from all threads, as usual.) The optional argument to + ``threadlocalproperty()`` is the default value factory: in case no + value was assigned in the current thread yet, the factory is called + and its result becomes the value in that thread (like + ``collections.defaultdict``). If no default value factory is + specified, uninitialized reads raise ``AttributeError``. Note that + with ``TransactionQueue`` you get a pool of a fixed number of + threads, each running the transactions one after the other; such + thread-local properties will have the value last stored in them in + the same thread,, which may come from a random previous transaction. + This means that ``threadlocalproperty`` is useful mainly to avoid + conflicts from cache-like data structures. + +Note that Python is a complicated language; there are a number of less +common cases that may cause conflict (of any kind) where we might not +expect it at priori. In many of these cases it could be fixed; please +report any case that you don't understand. (For example, so far, +creating a weakref to an object requires attaching an auxiliary +internal object to that object, and so it can cause write-write +conflicts.) Atomic sections --------------- -PyPy supports *atomic sections,* which are blocks of code which you -want to execute without "releasing the GIL". In STM terms, this means -blocks of code that are executed while guaranteeing that the -transaction is not interrupted in the middle. *This is experimental -and may be removed in the future* if `lock elision`_ is ever -implemented. +The ``TransactionQueue`` class described above is based on *atomic +sections,* which are blocks of code which you want to execute without +"releasing the GIL". In STM terms, this means blocks of code that are +executed while guaranteeing that the transaction is not interrupted in +the middle. *This is experimental and may be removed in the future* +if `Software lock elision`_ is ever implemented. -Here is a usage example:: +Here is a direct usage example:: with transaction.atomic: assert len(lst1) == 10 @@ -281,8 +403,8 @@ it likely that such a piece of code will eventually block all other threads anyway. -Note that if you want to experiment with ``atomic``, you may have to add -manually a transaction break just before the atomic block. This is +Note that if you want to experiment with ``atomic``, you may have to +manually add a transaction break just before the atomic block. This is because the boundaries of the block are not guaranteed to be the boundaries of the transaction: the latter is at least as big as the block, but may be bigger. Therefore, if you run a big atomic block, it @@ -295,7 +417,8 @@ including with a ``print`` to standard output. If one thread tries to acquire a lock while running in an atomic block, and another thread has got the same lock at that point, then the former may fail with a -``thread.error``. The reason is that "waiting" for some condition to +``thread.error``. (Don't rely on it; it may also deadlock.) +The reason is that "waiting" for some condition to become true --while running in an atomic block-- does not really make sense. For now you can work around it by making sure that, say, all your prints are either in an ``atomic`` block or none of them are. @@ -354,106 +477,38 @@ .. _`software lock elision`: https://www.repository.cam.ac.uk/handle/1810/239410 -Atomic sections, Transactions, etc.: a better way to write parallel programs ----------------------------------------------------------------------------- +Miscellaneous functions +----------------------- -(This section is based on locks as we plan to implement them, but also -works with the existing atomic sections.) - -In the cases where elision works, the block of code can run in parallel -with other blocks of code *even if they are protected by the same lock.* -You still get the illusion that the blocks are run sequentially. This -works even for multiple threads that run each a series of such blocks -and nothing else, protected by one single global lock. This is -basically the Python application-level equivalent of what was done with -the interpreter in ``pypy-stm``: while you think you are writing -thread-unfriendly code because of this global lock, actually the -underlying system is able to make it run on multiple cores anyway. - -This capability can be hidden in a library or in the framework you use; -the end user's code does not need to be explicitly aware of using -threads. For a simple example of this, there is `transaction.py`_ in -``lib_pypy``. The idea is that you write, or already have, some program -where the function ``f(key, value)`` runs on every item of some big -dictionary, say:: - - for key, value in bigdict.items(): - f(key, value) - -Then you simply replace the loop with:: - - for key, value in bigdict.items(): - transaction.add(f, key, value) - transaction.run() - -This code runs the various calls to ``f(key, value)`` using a thread -pool, but every single call is executed under the protection of a unique -lock. The end result is that the behavior is exactly equivalent --- in -fact it makes little sense to do it in this way on a non-STM PyPy or on -CPython. But on ``pypy-stm``, the various locked calls to ``f(key, -value)`` can tentatively be executed in parallel, even if the observable -result is as if they were executed in some serial order. - -This approach hides the notion of threads from the end programmer, -including all the hard multithreading-related issues. This is not the -first alternative approach to explicit threads; for example, OpenMP_ is -one. However, it is one of the first ones which does not require the -code to be organized in a particular fashion. Instead, it works on any -Python program which has got latent, imperfect parallelism. Ideally, it -only requires that the end programmer identifies where this parallelism -is likely to be found, and communicates it to the system, using for -example the ``transaction.add()`` scheme. - -.. _`transaction.py`: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/lib_pypy/transaction.py -.. _OpenMP: http://en.wikipedia.org/wiki/OpenMP - - -.. _`transactional_memory`: - -API of transactional_memory ---------------------------- - -The new pure Python module ``transactional_memory`` runs on both CPython -and PyPy, both with and without STM. It contains: - -* ``getsegmentlimit()``: return the number of "segments" in +* ``transaction.getsegmentlimit()``: return the number of "segments" in this pypy-stm. This is the limit above which more threads will not be able to execute on more cores. (Right now it is limited to 4 due to inter-segment overhead, but should be increased in the future. It should also be settable, and the default value should depend on the number of actual CPUs.) If STM is not available, this returns 1. -* ``print_abort_info(minimum_time=0.0)``: debugging help. Each thread - remembers the longest abort or pause it did because of cross-thread - contention_. This function prints it to ``stderr`` if the time lost - is greater than ``minimum_time`` seconds. The record is then - cleared, to make it ready for new events. This function returns - ``True`` if it printed a report, and ``False`` otherwise. +* ``__pypy__.thread.signals_enabled``: a context manager that runs its + block of code with signals enabled. By default, signals are only + enabled in the main thread; a non-main thread will not receive + signals (this is like CPython). Enabling signals in non-main + threads is useful for libraries where threads are hidden and the end + user is not expecting his code to run elsewhere than in the main + thread. +* ``pypystm.exclusive_atomic``: a context manager similar to + ``transaction.atomic`` but which complains if it is nested. -API of __pypy__.thread ----------------------- +* ``transaction.is_atomic()``: return True if called from an atomic + context. -The ``__pypy__.thread`` submodule is a built-in module of PyPy that -contains a few internal built-in functions used by the -``transactional_memory`` module, plus the following: +* ``pypystm.count()``: return a different positive integer every time + it is called. This works without generating conflicts. The + returned integers are only roughly in increasing order; this should + not be relied upon. -* ``__pypy__.thread.atomic``: a context manager to run a block in - fully atomic mode, without "releasing the GIL". (May be eventually - removed?) -* ``__pypy__.thread.signals_enabled``: a context manager that runs its - block with signals enabled. By default, signals are only enabled in - the main thread; a non-main thread will not receive signals (this is - like CPython). Enabling signals in non-main threads is useful for - libraries where threads are hidden and the end user is not expecting - his code to run elsewhere than in the main thread. - - -.. _contention: - -Conflicts ---------- +More details about conflicts +---------------------------- Based on Software Transactional Memory, the ``pypy-stm`` solution is prone to "conflicts". To repeat the basic idea, threads execute their code @@ -469,25 +524,26 @@ the transaction). If this occurs too often, parallelization fails. How much actual parallelization a multithreaded program can see is a bit -subtle. Basically, a program not using ``__pypy__.thread.atomic`` or +subtle. Basically, a program not using ``transaction.atomic`` or eliding locks, or doing so for very short amounts of time, will parallelize almost freely (as long as it's not some artificial example where, say, all threads try to increase the same global counter and do nothing else). -However, using if the program requires longer transactions, it comes +However, if the program requires longer transactions, it comes with less obvious rules. The exact details may vary from version to version, too, until they are a bit more stabilized. Here is an overview. Parallelization works as long as two principles are respected. The -first one is that the transactions must not *conflict* with each other. -The most obvious sources of conflicts are threads that all increment a -global shared counter, or that all store the result of their -computations into the same list --- or, more subtly, that all ``pop()`` -the work to do from the same list, because that is also a mutation of -the list. (It is expected that some STM-aware library will eventually -be designed to help with conflict problems, like a STM-aware queue.) +first one is that the transactions must not *conflict* with each +other. The most obvious sources of conflicts are threads that all +increment a global shared counter, or that all store the result of +their computations into the same list --- or, more subtly, that all +``pop()`` the work to do from the same list, because that is also a +mutation of the list. (You can work around it with +``transaction.stmdict``, but for that specific example, some STM-aware +queue should eventually be designed.) A conflict occurs as follows: when a transaction commits (i.e. finishes successfully) it may cause other transactions that are still in progress @@ -503,22 +559,23 @@ Another issue is that of avoiding long-running so-called "inevitable" transactions ("inevitable" is taken in the sense of "which cannot be avoided", i.e. transactions which cannot abort any more). Transactions -like that should only occur if you use ``__pypy__.thread.atomic``, -generally become of I/O in atomic blocks. They work, but the +like that should only occur if you use ``atomic``, +generally because of I/O in atomic blocks. They work, but the transaction is turned inevitable before the I/O is performed. For all the remaining execution time of the atomic block, they will impede parallel work. The best is to organize the code so that such operations -are done completely outside ``__pypy__.thread.atomic``. +are done completely outside ``atomic``. -(This is related to the fact that blocking I/O operations are +(This is not unrelated to the fact that blocking I/O operations are discouraged with Twisted, and if you really need them, you should do them on their own separate thread.) -In case of lock elision, we don't get long-running inevitable -transactions, but a different problem can occur: doing I/O cancels lock -elision, and the lock turns into a real lock, preventing other threads -from committing if they also need this lock. (More about it when lock -elision is implemented and tested.) +In case lock elision eventually replaces atomic sections, we wouldn't +get long-running inevitable transactions, but the same problem occurs +in a different way: doing I/O cancels lock elision, and the lock turns +into a real lock. This prevents other threads from committing if they +also need this lock. (More about it when lock elision is implemented +and tested.) @@ -528,56 +585,18 @@ XXX this section mostly empty for now -Low-level statistics --------------------- - -When a non-main thread finishes, you get low-level statistics printed to -stderr, looking like that:: - - thread 0x7f73377fe600: - outside transaction 42182 0.506 s - run current 85466 0.000 s - run committed 34262 3.178 s - run aborted write write 6982 0.083 s - run aborted write read 550 0.005 s - run aborted inevitable 388 0.010 s - run aborted other 0 0.000 s - wait free segment 0 0.000 s - wait write read 78 0.027 s - wait inevitable 887 0.490 s - wait other 0 0.000 s - sync commit soon 1 0.000 s - bookkeeping 51418 0.606 s - minor gc 162970 1.135 s - major gc 1 0.019 s - sync pause 59173 1.738 s - longest recordered marker 0.000826 s - "File "x.py", line 5, in f" - -On each line, the first number is a counter, and the second number gives -the associated time --- the amount of real time that the thread was in -this state. The sum of all the times should be equal to the total time -between the thread's start and the thread's end. The most important -points are "run committed", which gives the amount of useful work, and -"outside transaction", which should give the time spent e.g. in library -calls (right now it seems to be larger than that; to investigate). The -various "run aborted" and "wait" entries are time lost due to -conflicts_. Everything else is overhead of various forms. (Short-, -medium- and long-term future work involves reducing this overhead :-) - -The last two lines are special; they are an internal marker read by -``transactional_memory.print_abort_info()``. - - Reference to implementation details ----------------------------------- -The core of the implementation is in a separate C library called stmgc_, -in the c7_ subdirectory. Please see the `README.txt`_ for more -information. In particular, the notion of segment is discussed there. +The core of the implementation is in a separate C library called +stmgc_, in the c7_ subdirectory (current version of pypy-stm) and in +the c8_ subdirectory (bleeding edge version). Please see the +`README.txt`_ for more information. In particular, the notion of +segment is discussed there. .. _stmgc: https://bitbucket.org/pypy/stmgc/src/default/ .. _c7: https://bitbucket.org/pypy/stmgc/src/default/c7/ +.. _c8: https://bitbucket.org/pypy/stmgc/src/default/c8/ .. _`README.txt`: https://bitbucket.org/pypy/stmgc/raw/default/c7/README.txt PyPy itself adds on top of it the automatic placement of read__ and write__ diff --git a/pypy/goal/getnightly.py b/pypy/goal/getnightly.py --- a/pypy/goal/getnightly.py +++ b/pypy/goal/getnightly.py @@ -7,7 +7,7 @@ if sys.platform.startswith('linux'): arch = 'linux' cmd = 'wget "%s"' - tar = "tar -x -v --wildcards --strip-components=2 -f %s '*/bin/pypy'" + tar = "tar -x -v --wildcards --strip-components=2 -f %s '*/bin/pypy' '*/bin/libpypy-c.so'" if os.uname()[-1].startswith('arm'): arch += '-armhf-raspbian' elif sys.platform.startswith('darwin'): diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -137,7 +137,9 @@ filename = filename[:-1] basename = os.path.basename(filename) lastdirname = os.path.basename(os.path.dirname(filename)) - self.co_filename = '/%s/%s' % (lastdirname, basename) + if lastdirname: + basename = '%s/%s' % (lastdirname, basename) + self.co_filename = '/%s' % (basename,) co_names = property(lambda self: [self.space.unwrap(w_name) for w_name in self.co_names_w]) # for trace diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -1626,6 +1626,13 @@ def prepare_exec(f, prog, globals, locals, compile_flags, builtin, codetype): """Manipulate parameters to exec statement to (codeobject, dict, dict). """ + if (globals is None and locals is None and + isinstance(prog, tuple) and + (len(prog) == 2 or len(prog) == 3)): + globals = prog[1] + if len(prog) == 3: + locals = prog[2] + prog = prog[0] if globals is None: globals = f.f_globals if locals is None: diff --git a/pypy/interpreter/test/test_exec.py b/pypy/interpreter/test/test_exec.py --- a/pypy/interpreter/test/test_exec.py +++ b/pypy/interpreter/test/test_exec.py @@ -262,3 +262,11 @@ """] for c in code: compile(c, "", "exec") + + def test_exec_tuple(self): + # note: this is VERY different than testing exec("a = 42", d), because + # this specific case is handled specially by the AST compiler + d = {} + x = ("a = 42", d) + exec x + assert d['a'] == 42 diff --git a/pypy/module/_csv/test/__init__.py b/pypy/module/_csv/test/__init__.py new file mode 100644 diff --git a/pypy/module/_io/test/__init__.py b/pypy/module/_io/test/__init__.py new file mode 100644 diff --git a/pypy/module/_multiprocessing/test/__init__.py b/pypy/module/_multiprocessing/test/__init__.py new file mode 100644 diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -30,7 +30,7 @@ space.wrap(addr.get_protocol()), space.wrap(addr.get_pkttype()), space.wrap(addr.get_hatype()), - space.wrap(addr.get_addr())]) + space.wrap(addr.get_haddr())]) elif rsocket.HAS_AF_UNIX and isinstance(addr, rsocket.UNIXAddress): return space.wrap(addr.get_path()) elif rsocket.HAS_AF_NETLINK and isinstance(addr, rsocket.NETLINKAddress): @@ -79,7 +79,7 @@ raise NotImplementedError # XXX Hack to seperate rpython and pypy -def addr_from_object(family, space, w_address): +def addr_from_object(family, fd, space, w_address): if family == rsocket.AF_INET: w_host, w_port = space.unpackiterable(w_address, 2) host = space.str_w(w_host) @@ -89,8 +89,9 @@ if family == rsocket.AF_INET6: pieces_w = space.unpackiterable(w_address) if not (2 <= len(pieces_w) <= 4): - raise TypeError("AF_INET6 address must be a tuple of length 2 " - "to 4, not %d" % len(pieces_w)) + raise oefmt(space.w_TypeError, + "AF_INET6 address must be a tuple of length 2 " + "to 4, not %d", len(pieces_w)) host = space.str_w(pieces_w[0]) port = space.int_w(pieces_w[1]) port = make_ushort_port(space, port) @@ -105,6 +106,28 @@ if rsocket.HAS_AF_NETLINK and family == rsocket.AF_NETLINK: w_pid, w_groups = space.unpackiterable(w_address, 2) return rsocket.NETLINKAddress(space.uint_w(w_pid), space.uint_w(w_groups)) + if rsocket.HAS_AF_PACKET and family == rsocket.AF_PACKET: + pieces_w = space.unpackiterable(w_address) + if not (2 <= len(pieces_w) <= 5): + raise oefmt(space.w_TypeError, + "AF_PACKET address must be a tuple of length 2 " + "to 5, not %d", len(pieces_w)) + ifname = space.str_w(pieces_w[0]) + ifindex = rsocket.PacketAddress.get_ifindex_from_ifname(fd, ifname) + protocol = space.int_w(pieces_w[1]) + if len(pieces_w) > 2: pkttype = space.int_w(pieces_w[2]) + else: pkttype = 0 + if len(pieces_w) > 3: hatype = space.int_w(pieces_w[3]) + else: hatype = 0 + if len(pieces_w) > 4: haddr = space.str_w(pieces_w[4]) + else: haddr = "" + if len(haddr) > 8: + raise OperationError(space.w_ValueError, space.wrap( + "Hardware address must be 8 bytes or less")) + if protocol < 0 or protocol > 0xfffff: + raise OperationError(space.w_OverflowError, space.wrap( + "protoNumber must be 0-65535.")) + return rsocket.PacketAddress(ifindex, protocol, pkttype, hatype, haddr) raise RSocketError("unknown address family") # XXX Hack to seperate rpython and pypy @@ -172,7 +195,8 @@ # convert an app-level object into an Address # based on the current socket's family def addr_from_object(self, space, w_address): - return addr_from_object(self.sock.family, space, w_address) + fd = intmask(self.sock.fd) + return addr_from_object(self.sock.family, fd, space, w_address) def bind_w(self, space, w_addr): """bind(address) diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -1,4 +1,4 @@ -import sys +import sys, os import py from pypy.tool.pytest.objspace import gettestobjspace from rpython.tool.udir import udir @@ -615,6 +615,28 @@ os.chdir(oldcwd) +class AppTestPacket: + def setup_class(cls): + if not hasattr(os, 'getuid') or os.getuid() != 0: + py.test.skip("AF_PACKET needs to be root for testing") + w_ok = space.appexec([], "(): import _socket; " + + "return hasattr(_socket, 'AF_PACKET')") + if not space.is_true(w_ok): + py.test.skip("no AF_PACKET on this platform") + cls.space = space + + def test_convert_between_tuple_and_sockaddr_ll(self): + import _socket + s = _socket.socket(_socket.AF_PACKET, _socket.SOCK_RAW) + assert s.getsockname() == ('', 0, 0, 0, '') + s.bind(('lo', 123)) + a, b, c, d, e = s.getsockname() + assert (a, b, c) == ('lo', 123, 0) + assert isinstance(d, int) + assert isinstance(e, str) + assert 0 <= len(e) <= 8 + + class AppTestSocketTCP: HOST = 'localhost' diff --git a/pypy/module/_ssl/test/__init__.py b/pypy/module/_ssl/test/__init__.py new file mode 100644 diff --git a/pypy/module/itertools/test/__init__.py b/pypy/module/itertools/test/__init__.py new file mode 100644 diff --git a/pypy/module/pwd/test/__init__.py b/pypy/module/pwd/test/__init__.py new file mode 100644 diff --git a/pypy/module/pyexpat/__init__.py b/pypy/module/pyexpat/__init__.py --- a/pypy/module/pyexpat/__init__.py +++ b/pypy/module/pyexpat/__init__.py @@ -39,8 +39,6 @@ 'error': 'space.fromcache(interp_pyexpat.Cache).w_error', '__version__': 'space.wrap("85819")', - 'EXPAT_VERSION': 'interp_pyexpat.get_expat_version(space)', - 'version_info': 'interp_pyexpat.get_expat_version_info(space)', } submodules = { @@ -53,3 +51,9 @@ 'XML_PARAM_ENTITY_PARSING_ALWAYS']: interpleveldefs[name] = 'space.wrap(interp_pyexpat.%s)' % (name,) + def startup(self, space): + from pypy.module.pyexpat import interp_pyexpat + w_ver = interp_pyexpat.get_expat_version(space) + space.setattr(self, space.wrap("EXPAT_VERSION"), w_ver) + w_ver = interp_pyexpat.get_expat_version_info(space) + space.setattr(self, space.wrap("version_info"), w_ver) diff --git a/pypy/module/select/test/__init__.py b/pypy/module/select/test/__init__.py new file mode 100644 diff --git a/pypy/module/struct/test/__init__.py b/pypy/module/struct/test/__init__.py new file mode 100644 diff --git a/pypy/module/zipimport/test/test_zipimport_deflated.py b/pypy/module/zipimport/test/test_zipimport_deflated.py --- a/pypy/module/zipimport/test/test_zipimport_deflated.py +++ b/pypy/module/zipimport/test/test_zipimport_deflated.py @@ -14,7 +14,7 @@ def setup_class(cls): try: import rpython.rlib.rzlib - except ImportError: + except CompilationError: py.test.skip("zlib not available, cannot test compressed zipfiles") cls.make_class() cls.w_BAD_ZIP = cls.space.wrap(BAD_ZIP) diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py --- a/rpython/annotator/binaryop.py +++ b/rpython/annotator/binaryop.py @@ -132,13 +132,11 @@ impl = pair(s_c1, s_o2).getitem return read_can_only_throw(impl, s_c1, s_o2) - def getitem_idx_key((s_c1, s_o2)): + def getitem_idx((s_c1, s_o2)): impl = pair(s_c1, s_o2).getitem return impl() - getitem_idx_key.can_only_throw = _getitem_can_only_throw + getitem_idx.can_only_throw = _getitem_can_only_throw - getitem_idx = getitem_idx_key - getitem_key = getitem_idx_key class __extend__(pairtype(SomeType, SomeType), @@ -565,14 +563,10 @@ return lst1.listdef.read_item() getitem.can_only_throw = [] - getitem_key = getitem - def getitem_idx((lst1, int2)): return lst1.listdef.read_item() getitem_idx.can_only_throw = [IndexError] - getitem_idx_key = getitem_idx - def setitem((lst1, int2), s_value): lst1.listdef.mutate() lst1.listdef.generalize(s_value) @@ -588,14 +582,10 @@ return SomeChar(no_nul=str1.no_nul) getitem.can_only_throw = [] - getitem_key = getitem - def getitem_idx((str1, int2)): return SomeChar(no_nul=str1.no_nul) getitem_idx.can_only_throw = [IndexError] - getitem_idx_key = getitem_idx - def mul((str1, int2)): # xxx do we want to support this return SomeString(no_nul=str1.no_nul) @@ -604,14 +594,10 @@ return SomeUnicodeCodePoint() getitem.can_only_throw = [] - getitem_key = getitem - def getitem_idx((str1, int2)): return SomeUnicodeCodePoint() getitem_idx.can_only_throw = [IndexError] - getitem_idx_key = getitem_idx - def mul((str1, int2)): # xxx do we want to support this return SomeUnicodeString() diff --git a/rpython/doc/jit/index.rst b/rpython/doc/jit/index.rst --- a/rpython/doc/jit/index.rst +++ b/rpython/doc/jit/index.rst @@ -23,11 +23,15 @@ overview pyjitpl5 + optimizer virtualizable - :doc:`Overview `: motivating our approach - :doc:`Notes ` about the current work in PyPy +- :doc:`Optimizer `: the step between tracing and writing + machine code + - :doc:`Virtulizable ` how virtualizables work and what they are (in other words how to make frames more efficient). diff --git a/rpython/doc/jit/optimizer.rst b/rpython/doc/jit/optimizer.rst new file mode 100644 --- /dev/null +++ b/rpython/doc/jit/optimizer.rst @@ -0,0 +1,196 @@ +.. _trace_optimizer: + +Trace Optimizer +=============== + +Traces of user programs are not directly translated into machine code. +The optimizer module implements several different semantic preserving +transformations that either allow operations to be swept from the trace +or convert them to operations that need less time or space. + +The optimizer is in `rpython/jit/metainterp/optimizeopt/`. +When you try to make sense of this module, this page might get you started. + +Before some optimizations are explained in more detail, it is essential to +understand how traces look like. +The optimizer comes with a test suit. It contains many trace +examples and you might want to take a look at it +(in `rpython/jit/metainterp/optimizeopt/test/*.py`). +The allowed operations can be found in `rpython/jit/metainterp/resoperation.py`. +Here is an example of a trace:: + + [p0,i0,i1] + label(p0, i0, i1) + i2 = getarray_item_raw(p0, i0, descr=) + i3 = int_add(i1,i2) + i4 = int_add(i0,1) + i5 = int_le(i4, 100) # lower-or-equal + guard_true(i5) + jump(p0, i4, i3) + +At the beginning it might be clumsy to read but it makes sense when you start +to compare the Python code that constructed the trace:: + + from array import array + a = array('i',range(101)) + sum = 0; i = 0 + while i <= 100: # can be seen as label + sum += a[i] + i += 1 + # jumps back to the while header + +There are better ways to compute the sum from ``[0..100]``, but it gives a better intuition on how +traces are constructed than ``sum(range(101))``. +Note that the trace syntax is the one used in the test suite. It is also very +similar to traces printed at runtime by PYPYLOG_. The first line gives the input variables, the +second line is a ``label`` operation, the last one is the backwards ``jump`` operation. + +.. _PYPYLOG: logging.html + +These instructions mentioned earlier are special: + +* the input defines the input parameter type and name to enter the trace. +* ``label`` is the instruction a ``jump`` can target. Label instructions have + a ``JitCellToken`` associated that uniquely identifies the label. Any jump + has a target token of a label. + +The token is saved in a so called `descriptor` of the instruction. It is +not written explicitly because it is not done in the tests either. But +the test suite creates a dummy token for each trace and adds it as descriptor +to ``label`` and ``jump``. Of course the optimizer does the same at runtime, +but using real values. +The sample trace includes a descriptor in ``getarrayitem_raw``. Here it +annotates the type of the array. It is a signed integer array. + +High level overview +------------------- + +Before the JIT backend transforms any trace into machine code, it tries to +transform the trace into an equivalent trace that executes faster. The method +`optimize_trace` in `rpython/jit/metainterp/optimizeopt/__init__.py` is the +main entry point. + +Optimizations are applied in a sequence one after another and the base +sequence is as follows:: + + intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll + +Each of the colon-separated name has a class attached, inheriting from +the `Optimization` class. The `Optimizer` class itself also +derives from the `Optimization` class and implements the control logic for +the optimization. Most of the optimizations only require a single forward pass. +The trace is 'propagated' into each optimization using the method +`propagate_forward`. Instruction by instruction, it flows from the +first optimization to the last optimization. The method `emit_operation` +is called for every operation that is passed to the next optimizer. + +A frequently encountered pattern +-------------------------------- + +To find potential optimization targets it is necessary to know the instruction +type. Simple solution is to switch using the operation number (= type):: + + for op in operations: + if op.getopnum() == rop.INT_ADD: + # handle this instruction + pass + elif op.getopnum() == rop.INT_FLOOR_DIV: + pass + # and many more + +Things get worse if you start to match the arguments +(is argument one constant and two variable or vice versa?). The pattern to tackle +this code bloat is to move it to a separate method using +`make_dispatcher_method`. It associates methods with instruction types:: + + class OptX(Optimization): + def prefix_INT_ADD(self, op): + pass # emit, transform, ... + + dispatch_opt = make_dispatcher_method(OptX, 'prefix_', + default=OptX.emit_operation) + OptX.propagate_forward = dispatch_opt + + optX = OptX() + for op in operations: + optX.propagate_forward(op) + +``propagate_forward`` searches for the method that is able to handle the instruction +type. As an example `INT_ADD` will invoke `prefix_INT_ADD`. If there is no function +for the instruction, it is routed to the default implementation (``emit_operation`` +in this example). + +Rewrite optimization +-------------------- + +The second optimization is called 'rewrite' and is commonly also known as +strength reduction. A simple example would be that an integer multiplied +by 2 is equivalent to the bits shifted to the left once +(e.g. ``x * 2 == x << 1``). Not only strength reduction is done in this +optimization but also boolean or arithmetic simplifications. Other examples +would be: ``x & 0 == 0``, ``x - 0 == x`` + +Whenever such an operation is encountered (e.g. ``y = x & 0``), no operation is +emitted. Instead the variable y is made equal to 0 +(= ``make_equal_to(op.result, 0)``). The variables found in a trace are +instances of Box classes that can be found in +`rpython/jit/metainterp/history.py`. `OptValue` wraps those variables again +and maps the boxes to the optimization values in the optimizer. When a +value is made equal, the two variable's boxes are made to point to the same +`OptValue` instance. + +**NOTE: this OptValue organization is currently being refactored in a branch.** + +Pure optimization +----------------- + +Is interwoven into the basic optimizer. It saves operations, results, +arguments to be known to have pure semantics. + +"Pure" here means the same as the ``jit.elidable`` decorator: +free of "observable" side effects and referentially transparent +(the operation can be replaced with its result without changing the program +semantics). The operations marked as ALWAYS_PURE in `resoperation.py` are a +subset of the NOSIDEEFFECT operations. Operations such as new, new array, +getfield_(raw/gc) are marked as NOSIDEEFFECT but not as ALWAYS_PURE. + +Pure operations are optimized in two different ways. If their arguments +are constants, the operation is removed and the result is turned into a +constant. If not, we can still use a memoization technique: if, later, +we see the same operation on the same arguments again, we don't need to +recompute its result, but can simply reuse the previous operation's +result. + +Unroll optimization +------------------- + +A detailed description can be found the document +`Loop-Aware Optimizations in PyPy's Tracing JIT`__ + +.. __: http://www2.maths.lth.se/matematiklth/vision/publdb/reports/pdf/ardo-bolz-etal-dls-12.pdf + +This optimization does not fall into the traditional scheme of one forward +pass only. In a nutshell it unrolls the trace _once_, connects the two +traces (by inserting parameters into the jump and label of the peeled trace) +and uses information to iron out allocations, propagate constants and +do any other optimization currently present in the 'optimizeopt' module. + +It is prepended to all optimizations and thus extends the Optimizer class +and unrolls the loop once before it proceeds. + + +What is missing from this document +---------------------------------- + +* Guards are not explained +* Several optimizations are not explained + + +Further references +------------------ + +* `Allocation Removal by Partial Evaluation in a Tracing JIT`__ +* `Loop-Aware Optimizations in PyPy's Tracing JIT`__ + +.. __: http://www.stups.uni-duesseldorf.de/mediawiki/images/b/b0/Pub-BoCuFiLePeRi2011.pdf +.. __: http://www2.maths.lth.se/matematiklth/vision/publdb/reports/pdf/ardo-bolz-etal-dls-12.pdf diff --git a/rpython/doc/rtyper.rst b/rpython/doc/rtyper.rst --- a/rpython/doc/rtyper.rst +++ b/rpython/doc/rtyper.rst @@ -118,8 +118,7 @@ given this representation. The RTyper also computes a ``concretetype`` for Constants, to match the way they are used in the low-level operations (for example, ``int_add(x, 1)`` requires a ``Constant(1)`` with -``concretetype=Signed``, but an untyped ``add(x, 1)`` works with a -``Constant(1)`` that must actually be a PyObject at run-time). +``concretetype=Signed``). In addition to ``lowleveltype``, each Repr subclass provides a set of methods called ``rtype_op_xxx()`` which define how each high-level operation ``op_xxx`` @@ -306,14 +305,14 @@ ~~~~~~~~~~~~~ As in C, pointers provide the indirection needed to make a reference modifiable -or sharable. Pointers can only point to a structure, an array, a function -(see below) or a PyObject (see below). Pointers to primitive types, if needed, -must be done by pointing to a structure with a single field of the required -type. Pointer types are declared by:: +or sharable. Pointers can only point to a structure, an array or a function +(see below). Pointers to primitive types, if needed, must be done by pointing +to a structure with a single field of the required type. Pointer types are +declared by:: Ptr(TYPE) -At run-time, pointers to GC structures (GcStruct, GcArray and PyObject) hold a +At run-time, pointers to GC structures (GcStruct, GcArray) hold a reference to what they are pointing to. Pointers to non-GC structures that can go away when their container is deallocated (Struct, Array) must be handled with care: the bigger structure of which they are part of could be freed while @@ -356,22 +355,6 @@ :graph: the flow graph of the function. -The PyObject Type -~~~~~~~~~~~~~~~~~ - -This is a special type, for compatibility with CPython: it stands for a -structure compatible with PyObject. This is also a "container" type (thinking -about C, this is ``PyObject``, not ``PyObject*``), so it is usually manipulated -via a Ptr. A typed graph can still contain generic space operations (add, -getitem, etc.) provided they are applied on objects whose low-level type is -``Ptr(PyObject)``. In fact, code generators that support this should consider -that the default type of a variable, if none is specified, is ``Ptr(PyObject)``. -In this way, they can generate the correct code for fully-untyped flow graphs. - -The testing implementation allows you to "create" PyObjects by calling -``pyobjectptr(obj)``. - - Opaque Types ~~~~~~~~~~~~ diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py --- a/rpython/flowspace/model.py +++ b/rpython/flowspace/model.py @@ -140,6 +140,12 @@ newlink.llexitcase = self.llexitcase return newlink + def replace(self, mapping): + def rename(v): + if v is not None: + return v.replace(mapping) + return self.copy(rename) + def settarget(self, targetblock): assert len(self.args) == len(targetblock.inputargs), ( "output args mismatch") @@ -215,13 +221,12 @@ return uniqueitems([w for w in result if isinstance(w, Constant)]) def renamevariables(self, mapping): - self.inputargs = [mapping.get(a, a) for a in self.inputargs] - for op in self.operations: - op.args = [mapping.get(a, a) for a in op.args] - op.result = mapping.get(op.result, op.result) - self.exitswitch = mapping.get(self.exitswitch, self.exitswitch) + self.inputargs = [a.replace(mapping) for a in self.inputargs] + self.operations = [op.replace(mapping) for op in self.operations] + if self.exitswitch is not None: + self.exitswitch = self.exitswitch.replace(mapping) for link in self.exits: - link.args = [mapping.get(a, a) for a in link.args] + link.args = [a.replace(mapping) for a in link.args] def closeblock(self, *exits): assert self.exits == [], "block already closed" @@ -327,6 +332,8 @@ newvar.concretetype = self.concretetype return newvar + def replace(self, mapping): + return mapping.get(self, self) class Constant(Hashable): @@ -356,6 +363,9 @@ # cannot count on it not mutating at runtime! return False + def replace(self, mapping): + return self + class FSException(object): def __init__(self, w_type, w_value): @@ -431,8 +441,8 @@ ", ".join(map(repr, self.args))) def replace(self, mapping): - newargs = [mapping.get(arg, arg) for arg in self.args] - newresult = mapping.get(self.result, self.result) + newargs = [arg.replace(mapping) for arg in self.args] + newresult = self.result.replace(mapping) return type(self)(self.opname, newargs, newresult, self.offset) class Atom(object): diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py --- a/rpython/flowspace/operation.py +++ b/rpython/flowspace/operation.py @@ -76,8 +76,8 @@ self.offset = -1 def replace(self, mapping): - newargs = [mapping.get(arg, arg) for arg in self.args] - newresult = mapping.get(self.result, self.result) + newargs = [arg.replace(mapping) for arg in self.args] + newresult = self.result.replace(mapping) newop = type(self)(*newargs) newop.result = newresult newop.offset = self.offset @@ -422,8 +422,6 @@ add_operator('delattr', 2, dispatch=1, pyfunc=delattr) add_operator('getitem', 2, dispatch=2, pure=True) add_operator('getitem_idx', 2, dispatch=2, pure=True) -add_operator('getitem_key', 2, dispatch=2, pure=True) -add_operator('getitem_idx_key', 2, dispatch=2, pure=True) add_operator('setitem', 3, dispatch=2) add_operator('delitem', 2, dispatch=2) add_operator('getslice', 3, dispatch=1, pyfunc=do_getslice, pure=True) @@ -686,8 +684,6 @@ # the annotator tests op.getitem.canraise = [IndexError, KeyError, Exception] op.getitem_idx.canraise = [IndexError, KeyError, Exception] -op.getitem_key.canraise = [IndexError, KeyError, Exception] -op.getitem_idx_key.canraise = [IndexError, KeyError, Exception] op.setitem.canraise = [IndexError, KeyError, Exception] op.delitem.canraise = [IndexError, KeyError, Exception] op.contains.canraise = [Exception] # from an r_dict diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py --- a/rpython/flowspace/test/test_objspace.py +++ b/rpython/flowspace/test/test_objspace.py @@ -867,7 +867,7 @@ raise graph = self.codetest(f) simplify_graph(graph) - assert self.all_operations(graph) == {'getitem_idx_key': 1} + assert self.all_operations(graph) == {'getitem_idx': 1} g = lambda: None def f(c, x): @@ -877,7 +877,7 @@ g() graph = self.codetest(f) simplify_graph(graph) - assert self.all_operations(graph) == {'getitem_idx_key': 1, + assert self.all_operations(graph) == {'getitem_idx': 1, 'simple_call': 2} def f(c, x): @@ -896,7 +896,7 @@ raise graph = self.codetest(f) simplify_graph(graph) - assert self.all_operations(graph) == {'getitem_key': 1} + assert self.all_operations(graph) == {'getitem': 1} def f(c, x): try: @@ -915,7 +915,7 @@ graph = self.codetest(f) simplify_graph(graph) self.show(graph) - assert self.all_operations(graph) == {'getitem_idx_key': 1} + assert self.all_operations(graph) == {'getitem_idx': 1} def f(c, x): try: @@ -933,7 +933,7 @@ return -1 graph = self.codetest(f) simplify_graph(graph) - assert self.all_operations(graph) == {'getitem_key': 1} + assert self.all_operations(graph) == {'getitem': 1} def f(c, x): try: diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -52,6 +52,9 @@ # XXX total addressable size. Maybe by keeping some minimarkpage arenas # XXX pre-reserved, enough for a few nursery collections? What about # XXX raw-malloced memory? + +# XXX try merging old_objects_pointing_to_pinned into +# XXX old_objects_pointing_to_young (IRC 2014-10-22, fijal and gregor_w) import sys from rpython.rtyper.lltypesystem import lltype, llmemory, llarena, llgroup from rpython.rtyper.lltypesystem.lloperation import llop @@ -63,6 +66,7 @@ from rpython.rlib.rarithmetic import LONG_BIT_SHIFT from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop from rpython.rlib.objectmodel import specialize +from rpython.memory.gc.minimarkpage import out_of_memory # # Handles the objects in 2 generations: @@ -471,10 +475,10 @@ # the start of the nursery: we actually allocate a bit more for # the nursery than really needed, to simplify pointer arithmetic # in malloc_fixedsize(). The few extra pages are never used - # anyway so it doesn't even counct. + # anyway so it doesn't even count. nursery = llarena.arena_malloc(self._nursery_memory_size(), 0) if not nursery: - raise MemoryError("cannot allocate nursery") + out_of_memory("cannot allocate nursery") return nursery def allocate_nursery(self): @@ -685,23 +689,48 @@ def collect_and_reserve(self, totalsize): """To call when nursery_free overflows nursery_top. - First check if the nursery_top is the real top, otherwise we - can just move the top of one cleanup and continue - - Do a minor collection, and possibly also a major collection, - and finally reserve 'totalsize' bytes at the start of the - now-empty nursery. + First check if pinned objects are in front of nursery_top. If so, + jump over the pinned object and try again to reserve totalsize. + Otherwise do a minor collection, and possibly a major collection, and + finally reserve totalsize bytes. """ minor_collection_count = 0 while True: self.nursery_free = llmemory.NULL # debug: don't use me + # note: no "raise MemoryError" between here and the next time + # we initialize nursery_free! if self.nursery_barriers.non_empty(): + # Pinned object in front of nursery_top. Try reserving totalsize + # by jumping into the next, yet unused, area inside the + # nursery. "Next area" in this case is the space between the + # pinned object in front of nusery_top and the pinned object + # after that. Graphically explained: + # + # |- allocating totalsize failed in this area + # | |- nursery_top + # | | |- pinned object in front of nursery_top, + # v v v jump over this + # +---------+--------+--------+--------+-----------+ } + # | used | pinned | empty | pinned | empty | }- nursery + # +---------+--------+--------+--------+-----------+ } + # ^- try reserving totalsize in here next + # + # All pinned objects are represented by entries in + # nursery_barriers (see minor_collection). The last entry is + # always the end of the nursery. Therefore if nursery_barriers + # contains only one element, we jump over a pinned object and + # the "next area" (the space where we will try to allocate + # totalsize) starts at the end of the pinned object and ends at + # nursery's end. + # + # find the size of the pinned object after nursery_top size_gc_header = self.gcheaderbuilder.size_gc_header pinned_obj_size = size_gc_header + self.get_size( self.nursery_top + size_gc_header) - + # + # update used nursery space to allocate objects self.nursery_free = self.nursery_top + pinned_obj_size self.nursery_top = self.nursery_barriers.popleft() else: @@ -729,6 +758,9 @@ "Seeing minor_collection() at least twice." "Too many pinned objects?") # + # Tried to do something about nursery_free overflowing + # nursery_top before this point. Try to reserve totalsize now. + # If this succeeds break out of loop. result = self.nursery_free if self.nursery_free + totalsize <= self.nursery_top: self.nursery_free = result + totalsize @@ -1491,7 +1523,7 @@ # being moved, not from being collected if it is not reachable anymore. self.surviving_pinned_objects = self.AddressStack() # The following counter keeps track of alive and pinned young objects - # inside the nursery. We reset it here and increace it in + # inside the nursery. We reset it here and increase it in # '_trace_drag_out()'. any_pinned_object_from_earlier = self.any_pinned_object_kept self.pinned_objects_in_nursery = 0 @@ -1625,7 +1657,9 @@ else: llarena.arena_reset(prev, self.nursery + self.nursery_size - prev, 0) # + # always add the end of the nursery to the list nursery_barriers.append(self.nursery + self.nursery_size) + # self.nursery_barriers = nursery_barriers self.surviving_pinned_objects.delete() # @@ -1950,7 +1984,7 @@ # arena = llarena.arena_malloc(raw_malloc_usage(totalsize), False) if not arena: - raise MemoryError("cannot allocate object") + out_of_memory("out of memory: couldn't allocate a few KB more") llarena.arena_reserve(arena, totalsize) # size_gc_header = self.gcheaderbuilder.size_gc_header @@ -2058,7 +2092,7 @@ # XXX A simplifying assumption that should be checked, # finalizers/weak references are rare and short which means that - # they do not need a seperate state and do not need to be + # they do not need a separate state and do not need to be # made incremental. if (not self.objects_to_trace.non_empty() and not self.more_objects_to_trace.non_empty()): @@ -2148,9 +2182,9 @@ # even higher memory consumption. To prevent it, if it's # the second time we are here, then abort the program. if self.max_heap_size_already_raised: - llop.debug_fatalerror(lltype.Void, - "Using too much memory, aborting") + out_of_memory("using too much memory, aborting") self.max_heap_size_already_raised = True + self.gc_state = STATE_SCANNING raise MemoryError self.gc_state = STATE_FINALIZING diff --git a/rpython/memory/gc/minimarkpage.py b/rpython/memory/gc/minimarkpage.py --- a/rpython/memory/gc/minimarkpage.py +++ b/rpython/memory/gc/minimarkpage.py @@ -2,7 +2,7 @@ from rpython.rtyper.lltypesystem import lltype, llmemory, llarena, rffi from rpython.rlib.rarithmetic import LONG_BIT, r_uint from rpython.rlib.objectmodel import we_are_translated -from rpython.rlib.debug import ll_assert +from rpython.rlib.debug import ll_assert, fatalerror WORD = LONG_BIT // 8 NULL = llmemory.NULL @@ -294,7 +294,7 @@ # be a page-aligned address arena_base = llarena.arena_malloc(self.arena_size, False) if not arena_base: - raise MemoryError("couldn't allocate the next arena") + out_of_memory("out of memory: couldn't allocate the next arena") arena_end = arena_base + self.arena_size # # 'firstpage' points to the first unused page @@ -593,3 +593,10 @@ if isinstance(size, int): size = llmemory.sizeof(lltype.Char) * size return size + +def out_of_memory(errmsg): + """Signal a fatal out-of-memory error and abort. For situations where + it is hard to write and test code that would handle a MemoryError + exception gracefully. + """ + fatalerror(errmsg) diff --git a/rpython/memory/gctransform/asmgcroot.py b/rpython/memory/gctransform/asmgcroot.py --- a/rpython/memory/gctransform/asmgcroot.py +++ b/rpython/memory/gctransform/asmgcroot.py @@ -368,6 +368,13 @@ if rpy_fastgil != 1: ll_assert(rpy_fastgil != 0, "walk_stack_from doesn't have the GIL") initialframedata = rffi.cast(llmemory.Address, rpy_fastgil) + # + # very rare issue: initialframedata.address[0] is uninitialized + # in this case, but "retaddr = callee.frame_address.address[0]" + # reads it. If it happens to be exactly a valid return address + # inside the C code, then bad things occur. + initialframedata.address[0] = llmemory.NULL + # self.walk_frames(curframe, otherframe, initialframedata) stackscount += 1 # @@ -519,17 +526,15 @@ from rpython.jit.backend.llsupport.jitframe import STACK_DEPTH_OFS tid = self.gc.get_possibly_forwarded_type_id(ebp_in_caller) - ll_assert(rffi.cast(lltype.Signed, tid) == - rffi.cast(lltype.Signed, self.frame_tid), - "found a stack frame that does not belong " - "anywhere I know, bug in asmgcc") - # fish the depth - extra_stack_depth = (ebp_in_caller + STACK_DEPTH_OFS).signed[0] - ll_assert((extra_stack_depth & (rffi.sizeof(lltype.Signed) - 1)) - == 0, "asmgcc: misaligned extra_stack_depth") - extra_stack_depth //= rffi.sizeof(lltype.Signed) - self._shape_decompressor.setjitframe(extra_stack_depth) - return + if (rffi.cast(lltype.Signed, tid) == + rffi.cast(lltype.Signed, self.frame_tid)): + # fish the depth + extra_stack_depth = (ebp_in_caller + STACK_DEPTH_OFS).signed[0] + ll_assert((extra_stack_depth & (rffi.sizeof(lltype.Signed) - 1)) + == 0, "asmgcc: misaligned extra_stack_depth") + extra_stack_depth //= rffi.sizeof(lltype.Signed) + self._shape_decompressor.setjitframe(extra_stack_depth) + return llop.debug_fatalerror(lltype.Void, "cannot find gc roots!") def getlocation(self, callee, ebp_in_caller, location): diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py --- a/rpython/rlib/_rsocket_rffi.py +++ b/rpython/rlib/_rsocket_rffi.py @@ -199,7 +199,7 @@ WSA_INVALID_PARAMETER WSA_NOT_ENOUGH_MEMORY WSA_OPERATION_ABORTED SIO_RCVALL SIO_KEEPALIVE_VALS -SIOCGIFNAME +SIOCGIFNAME SIOCGIFINDEX '''.split() for name in constant_names: @@ -328,7 +328,8 @@ if _HAS_AF_PACKET: CConfig.sockaddr_ll = platform.Struct('struct sockaddr_ll', - [('sll_ifindex', rffi.INT), + [('sll_family', rffi.INT), + ('sll_ifindex', rffi.INT), ('sll_protocol', rffi.INT), ('sll_pkttype', rffi.INT), ('sll_hatype', rffi.INT), diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py --- a/rpython/rlib/rsocket.py +++ b/rpython/rlib/rsocket.py @@ -5,15 +5,8 @@ a drop-in replacement for the 'socket' module. """ -# Known missing features: -# -# - address families other than AF_INET, AF_INET6, AF_UNIX, AF_PACKET -# - AF_PACKET is only supported on Linux -# - methods makefile(), -# - SSL -# -# It's unclear if makefile() and SSL support belong here or only as -# app-level code for PyPy. +# XXX this does not support yet the least common AF_xxx address families +# supported by CPython. See http://bugs.pypy.org/issue1942 from rpython.rlib import _rsocket_rffi as _c, jit, rgc from rpython.rlib.objectmodel import instantiate, keepalive_until_here @@ -200,23 +193,49 @@ family = AF_PACKET struct = _c.sockaddr_ll maxlen = minlen = sizeof(struct) + ifr_name_size = _c.ifreq.c_ifr_name.length + sll_addr_size = _c.sockaddr_ll.c_sll_addr.length + + def __init__(self, ifindex, protocol, pkttype=0, hatype=0, haddr=""): + addr = lltype.malloc(_c.sockaddr_ll, flavor='raw', zero=True, + track_allocation=False) + self.setdata(addr, PacketAddress.maxlen) + rffi.setintfield(addr, 'c_sll_family', AF_PACKET) + rffi.setintfield(addr, 'c_sll_protocol', htons(protocol)) + rffi.setintfield(addr, 'c_sll_ifindex', ifindex) + rffi.setintfield(addr, 'c_sll_pkttype', pkttype) + rffi.setintfield(addr, 'c_sll_hatype', hatype) + halen = rffi.str2chararray(haddr, + rffi.cast(rffi.CCHARP, addr.c_sll_addr), + PacketAddress.sll_addr_size) + rffi.setintfield(addr, 'c_sll_halen', halen) + + @staticmethod + def get_ifindex_from_ifname(fd, ifname): + p = lltype.malloc(_c.ifreq, flavor='raw') + iflen = rffi.str2chararray(ifname, + rffi.cast(rffi.CCHARP, p.c_ifr_name), + PacketAddress.ifr_name_size - 1) + p.c_ifr_name[iflen] = '\0' + err = _c.ioctl(fd, _c.SIOCGIFINDEX, p) + ifindex = p.c_ifr_ifindex + lltype.free(p, flavor='raw') + if err != 0: + raise RSocketError("invalid interface name") + return ifindex def get_ifname(self, fd): + ifname = "" a = self.lock(_c.sockaddr_ll) - p = lltype.malloc(_c.ifreq, flavor='raw') - rffi.setintfield(p, 'c_ifr_ifindex', - rffi.getintfield(a, 'c_sll_ifindex')) - if (_c.ioctl(fd, _c.SIOCGIFNAME, p) == 0): - # eh, the iface name is a constant length array - i = 0 - d = [] - while p.c_ifr_name[i] != '\x00' and i < len(p.c_ifr_name): - d.append(p.c_ifr_name[i]) - i += 1 - ifname = ''.join(d) - else: - ifname = "" - lltype.free(p, flavor='raw') + ifindex = rffi.getintfield(a, 'c_sll_ifindex') + if ifindex: + p = lltype.malloc(_c.ifreq, flavor='raw') + rffi.setintfield(p, 'c_ifr_ifindex', ifindex) + if (_c.ioctl(fd, _c.SIOCGIFNAME, p) == 0): + ifname = rffi.charp2strn( + rffi.cast(rffi.CCHARP, p.c_ifr_name), + PacketAddress.ifr_name_size) + lltype.free(p, flavor='raw') self.unlock() return ifname @@ -235,11 +254,11 @@ def get_hatype(self): a = self.lock(_c.sockaddr_ll) - res = bool(rffi.getintfield(a, 'c_sll_hatype')) + res = rffi.getintfield(a, 'c_sll_hatype') self.unlock() return res - def get_addr(self): + def get_haddr(self): a = self.lock(_c.sockaddr_ll) lgt = rffi.getintfield(a, 'c_sll_halen') d = [] diff --git a/rpython/rlib/rzipfile.py b/rpython/rlib/rzipfile.py --- a/rpython/rlib/rzipfile.py +++ b/rpython/rlib/rzipfile.py @@ -8,7 +8,7 @@ try: from rpython.rlib import rzlib -except (ImportError, CompilationError): +except CompilationError: rzlib = None crc_32_tab = [ diff --git a/rpython/rlib/rzlib.py b/rpython/rlib/rzlib.py --- a/rpython/rlib/rzlib.py +++ b/rpython/rlib/rzlib.py @@ -22,13 +22,10 @@ includes=['zlib.h'], testonly_libraries = testonly_libraries ) -try: - eci = rffi_platform.configure_external_library( - libname, eci, - [dict(prefix='zlib-'), - ]) -except CompilationError: - raise ImportError("Could not find a zlib library") +eci = rffi_platform.configure_external_library( + libname, eci, + [dict(prefix='zlib-'), + ]) constantnames = ''' diff --git a/rpython/rlib/test/test_rzipfile.py b/rpython/rlib/test/test_rzipfile.py --- a/rpython/rlib/test/test_rzipfile.py +++ b/rpython/rlib/test/test_rzipfile.py @@ -9,7 +9,7 @@ try: from rpython.rlib import rzlib -except ImportError, e: +except CompilationError as e: py.test.skip("zlib not installed: %s " % (e, )) class BaseTestRZipFile(BaseRtypingTest): diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py --- a/rpython/rtyper/lltypesystem/rffi.py +++ b/rpython/rtyper/lltypesystem/rffi.py @@ -835,6 +835,14 @@ else: lltype.free(cp, flavor='raw', track_allocation=False) + # str -> already-existing char[maxsize] + def str2chararray(s, array, maxsize): + length = min(len(s), maxsize) + ll_s = llstrtype(s) + copy_string_to_raw(ll_s, array, 0, length) + return length + str2chararray._annenforceargs_ = [strtype, None, int] + # char* -> str # doesn't free char* def charp2str(cp): @@ -985,19 +993,19 @@ return (str2charp, free_charp, charp2str, get_nonmovingbuffer, free_nonmovingbuffer, alloc_buffer, str_from_buffer, keep_buffer_alive_until_here, - charp2strn, charpsize2str, + charp2strn, charpsize2str, str2chararray, ) (str2charp, free_charp, charp2str, get_nonmovingbuffer, free_nonmovingbuffer, alloc_buffer, str_from_buffer, keep_buffer_alive_until_here, - charp2strn, charpsize2str, + charp2strn, charpsize2str, str2chararray, ) = make_string_mappings(str) (unicode2wcharp, free_wcharp, wcharp2unicode, get_nonmoving_unicodebuffer, free_nonmoving_unicodebuffer, alloc_unicodebuffer, unicode_from_buffer, keep_unicodebuffer_alive_until_here, - wcharp2unicoden, wcharpsize2unicode, + wcharp2unicoden, wcharpsize2unicode, unicode2wchararray, ) = make_string_mappings(unicode) # char** diff --git a/rpython/rtyper/lltypesystem/rlist.py b/rpython/rtyper/lltypesystem/rlist.py --- a/rpython/rtyper/lltypesystem/rlist.py +++ b/rpython/rtyper/lltypesystem/rlist.py @@ -1,6 +1,7 @@ from rpython.rlib import rgc, jit, types from rpython.rlib.debug import ll_assert from rpython.rlib.signature import signature +from rpython.rtyper.error import TyperError from rpython.rtyper.lltypesystem import rstr from rpython.rtyper.lltypesystem.lltype import (GcForwardReference, Ptr, GcArray, GcStruct, Void, Signed, malloc, typeOf, nullptr, typeMethod) @@ -57,7 +58,7 @@ elif variant == ("reversed",): return ReversedListIteratorRepr(self) else: - raise NotImplementedError(variant) + raise TyperError("unsupported %r iterator over a list" % (variant,)) def get_itemarray_lowleveltype(self): ITEM = self.item_repr.lowleveltype diff --git a/rpython/rtyper/lltypesystem/rrange.py b/rpython/rtyper/lltypesystem/rrange.py --- a/rpython/rtyper/lltypesystem/rrange.py +++ b/rpython/rtyper/lltypesystem/rrange.py @@ -1,5 +1,6 @@ from rpython.rtyper.lltypesystem.lltype import Ptr, GcStruct, Signed, malloc, Void from rpython.rtyper.rrange import AbstractRangeRepr, AbstractRangeIteratorRepr +from rpython.rtyper.error import TyperError # ____________________________________________________________ # @@ -59,7 +60,10 @@ self.ll_newrange = ll_newrange self.ll_newrangest = ll_newrangest - def make_iterator_repr(self): + def make_iterator_repr(self, variant=None): + if variant is not None: + raise TyperError("unsupported %r iterator over a range list" % + (variant,)) return RangeIteratorRepr(self) diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py --- a/rpython/rtyper/lltypesystem/rstr.py +++ b/rpython/rtyper/lltypesystem/rstr.py @@ -188,7 +188,10 @@ self.CACHE[value] = p return p - def make_iterator_repr(self): + def make_iterator_repr(self, variant=None): + if variant is not None: + raise TyperError("unsupported %r iterator over a str/unicode" % + (variant,)) return self.repr.iterator_repr def can_ll_be_null(self, s_value): diff --git a/rpython/rtyper/lltypesystem/test/test_rffi.py b/rpython/rtyper/lltypesystem/test/test_rffi.py --- a/rpython/rtyper/lltypesystem/test/test_rffi.py +++ b/rpython/rtyper/lltypesystem/test/test_rffi.py @@ -671,6 +671,23 @@ assert interpret(f, [], backendopt=True) == 43 + def test_str2chararray(self): + eci = ExternalCompilationInfo(includes=['string.h']) + strlen = llexternal('strlen', [CCHARP], SIZE_T, + compilation_info=eci) + def f(): + raw = str2charp("XxxZy") + n = str2chararray("abcdef", raw, 4) + assert raw[0] == 'a' + assert raw[1] == 'b' + assert raw[2] == 'c' + assert raw[3] == 'd' + assert raw[4] == 'y' + lltype.free(raw, flavor='raw') + return n + + assert interpret(f, []) == 4 + def test_around_extcall(self): if sys.platform == "win32": py.test.skip('No pipes on windows') diff --git a/rpython/rtyper/rlist.py b/rpython/rtyper/rlist.py --- a/rpython/rtyper/rlist.py +++ b/rpython/rtyper/rlist.py @@ -269,13 +269,9 @@ v_res = hop.gendirectcall(llfn, c_func_marker, c_basegetitem, v_lst, v_index) return r_lst.recast(hop.llops, v_res) - rtype_getitem_key = rtype_getitem - def rtype_getitem_idx((r_lst, r_int), hop): return pair(r_lst, r_int).rtype_getitem(hop, checkidx=True) - rtype_getitem_idx_key = rtype_getitem_idx - def rtype_setitem((r_lst, r_int), hop): if hop.has_implicit_exception(IndexError): spec = dum_checkidx diff --git a/rpython/rtyper/rmodel.py b/rpython/rtyper/rmodel.py --- a/rpython/rtyper/rmodel.py +++ b/rpython/rtyper/rmodel.py @@ -287,11 +287,9 @@ # default implementation for checked getitems - def rtype_getitem_idx_key((r_c1, r_o1), hop): + def rtype_getitem_idx((r_c1, r_o1), hop): return pair(r_c1, r_o1).rtype_getitem(hop) - rtype_getitem_idx = rtype_getitem_idx_key - rtype_getitem_key = rtype_getitem_idx_key # ____________________________________________________________ diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py --- a/rpython/rtyper/rstr.py +++ b/rpython/rtyper/rstr.py @@ -580,13 +580,9 @@ hop.exception_cannot_occur() return hop.gendirectcall(llfn, v_str, v_index) - rtype_getitem_key = rtype_getitem - def rtype_getitem_idx((r_str, r_int), hop): return pair(r_str, r_int).rtype_getitem(hop, checkidx=True) - rtype_getitem_idx_key = rtype_getitem_idx - def rtype_mul((r_str, r_int), hop): str_repr = r_str.repr v_str, v_int = hop.inputargs(str_repr, Signed) diff --git a/rpython/rtyper/rtuple.py b/rpython/rtyper/rtuple.py --- a/rpython/rtyper/rtuple.py +++ b/rpython/rtyper/rtuple.py @@ -210,7 +210,10 @@ ll_str = property(gen_str_function) - def make_iterator_repr(self): + def make_iterator_repr(self, variant=None): + if variant is not None: + raise TyperError("unsupported %r iterator over a tuple" % + (variant,)) if len(self.items_r) == 1: # subclasses are supposed to set the IteratorRepr attribute return self.IteratorRepr(self) diff --git a/rpython/rtyper/test/test_rstr.py b/rpython/rtyper/test/test_rstr.py --- a/rpython/rtyper/test/test_rstr.py +++ b/rpython/rtyper/test/test_rstr.py @@ -116,6 +116,16 @@ res = self.interpret(fn, [1]) assert res == 1 + ord('a') + 10000 + def test_str_iterator_reversed_unsupported(self): + const = self.const + def fn(): + total = 0 + t = const('foo') + for x in reversed(t): + total += ord(x) + return total + py.test.raises(TyperError, self.interpret, fn, []) + def test_char_constant(self): const = self.const def fn(s): diff --git a/rpython/rtyper/test/test_rtuple.py b/rpython/rtyper/test/test_rtuple.py --- a/rpython/rtyper/test/test_rtuple.py +++ b/rpython/rtyper/test/test_rtuple.py @@ -1,8 +1,10 @@ +import py from rpython.rtyper.rtuple import TUPLE_TYPE, TupleRepr from rpython.rtyper.lltypesystem.lltype import Signed, Bool from rpython.rtyper.rbool import bool_repr from rpython.rtyper.rint import signed_repr from rpython.rtyper.test.tool import BaseRtypingTest +from rpython.rtyper.error import TyperError from rpython.rlib.objectmodel import compute_hash from rpython.translator.translator import TranslationContext @@ -228,6 +230,15 @@ res = self.interpret(f, [93813]) assert res == 93813 + def test_tuple_iterator_reversed_unsupported(self): + def f(i): + total = 0 + t = (i,) + for x in reversed(t): + total += x + return total + py.test.raises(TyperError, self.interpret, f, [93813]) + def test_inst_tuple_iter(self): class A: pass diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py --- a/rpython/translator/c/test/test_newgc.py +++ b/rpython/translator/c/test/test_newgc.py @@ -2,6 +2,7 @@ import inspect import os import sys +import subprocess import py @@ -50,8 +51,8 @@ t.viewcg() exename = t.compile() - def run(s, i): - data = py.process.cmdexec("%s %s %d" % (exename, s, i)) + def run(s, i, runner=subprocess.check_output): + data = runner([str(exename), str(s), str(i)]) data = data.strip() if data == 'MEMORY-ERROR': raise MemoryError @@ -115,11 +116,11 @@ cls.c_allfuncs.close_isolate() cls.c_allfuncs = None - def run(self, name, *args): + def run(self, name, *args, **kwds): if not args: args = (-1, ) print 'Running %r)' % name - res = self.c_allfuncs(name, *args) + res = self.c_allfuncs(name, *args, **kwds) num = self.name_to_func[name] if self.funcsstr[num]: return res @@ -1524,6 +1525,38 @@ res = self.run("nongc_opaque_attached_to_gc") assert res == 0 From noreply at buildbot.pypy.org Thu Mar 26 17:03:19 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 26 Mar 2015 17:03:19 +0100 (CET) Subject: [pypy-commit] pypy vmprof: untabify Message-ID: <20150326160319.26DAA1C084F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof Changeset: r76579:4897a385fb65 Date: 2015-03-26 17:03 +0100 http://bitbucket.org/pypy/pypy/changeset/4897a385fb65/ Log: untabify diff --git a/rpython/bin/rpython-vmprof b/rpython/bin/rpython-vmprof --- a/rpython/bin/rpython-vmprof +++ b/rpython/bin/rpython-vmprof @@ -21,7 +21,7 @@ x = open("vmprof.log", "w") _vmprof.enable(x.fileno(), 1000) try: - main() + main() finally: - _vmprof.disable() + _vmprof.disable() x.close() From noreply at buildbot.pypy.org Thu Mar 26 20:28:46 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 26 Mar 2015 20:28:46 +0100 (CET) Subject: [pypy-commit] pypy object-dtype2: merge default into branch Message-ID: <20150326192846.CB7251C0484@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: object-dtype2 Changeset: r76580:6c9f1ab16470 Date: 2015-03-26 18:08 +0200 http://bitbucket.org/pypy/pypy/changeset/6c9f1ab16470/ Log: merge default into branch diff too long, truncating to 2000 out of 4415 lines diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -11,3 +11,12 @@ 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1 10f1b29a2bd21f837090286174a9ca030b8680b2 release-2.5.0 +8e24dac0b8e2db30d46d59f2c4daa3d4aaab7861 release-2.5.1 +8e24dac0b8e2db30d46d59f2c4daa3d4aaab7861 release-2.5.1 +0000000000000000000000000000000000000000 release-2.5.1 +0000000000000000000000000000000000000000 release-2.5.1 +e3d046c43451403f5969580fc1c41d5df6c4082a release-2.5.1 +e3d046c43451403f5969580fc1c41d5df6c4082a release-2.5.1 +0000000000000000000000000000000000000000 release-2.5.1 +0000000000000000000000000000000000000000 release-2.5.1 +9c4588d731b7fe0b08669bd732c2b676cb0a8233 release-2.5.1 diff --git a/lib_pypy/_tkinter/app.py b/lib_pypy/_tkinter/app.py --- a/lib_pypy/_tkinter/app.py +++ b/lib_pypy/_tkinter/app.py @@ -96,7 +96,7 @@ if not self.threaded: # TCL is not thread-safe, calls needs to be serialized. - self._tcl_lock = threading.Lock() + self._tcl_lock = threading.RLock() else: self._tcl_lock = _DummyLock() diff --git a/lib_pypy/cffi.egg-info b/lib_pypy/cffi.egg-info --- a/lib_pypy/cffi.egg-info +++ b/lib_pypy/cffi.egg-info @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: cffi -Version: 0.9.0 +Version: 0.9.2 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "0.9.0" -__version_info__ = (0, 9, 0) +__version__ = "0.9.2" +__version_info__ = (0, 9, 2) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -58,7 +58,7 @@ # General information about the project. project = u'PyPy' -copyright = u'2014, The PyPy Project' +copyright = u'2015, The PyPy Project' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -67,7 +67,7 @@ # The short X.Y version. version = '2.5' # The full version, including alpha/beta/rc tags. -release = '2.5.0' +release = '2.5.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-2.5.1.rst release-2.5.0.rst release-2.4.0.rst release-2.3.1.rst diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -7,6 +7,7 @@ .. toctree:: whatsnew-head.rst + whatsnew-2.5.1.rst whatsnew-2.5.0.rst whatsnew-2.4.0.rst whatsnew-2.3.1.rst diff --git a/pypy/doc/release-2.5.1.rst b/pypy/doc/release-2.5.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-2.5.1.rst @@ -0,0 +1,115 @@ +================================ +PyPy 2.5.1 - Pineapple Bromeliad +================================ + +We're pleased to announce PyPy 2.5.1, Pineapple `Bromeliad`_ following on the heels of 2.5.0 + +You can download the PyPy 2.5.1 release here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project, and for those who donate to our three sub-projects, as well as our +volunteers and contributors. +We've shown quite a bit of progress, but we're slowly running out of funds. +Please consider donating more, or even better convince your employer to donate, +so we can finish those projects! The three sub-projects are: + +* `Py3k`_ (supporting Python 3.x): We have released a Python 3.2.5 compatible version + we call PyPy3 2.4.0, and are working toward a Python 3.3 compatible version + +* `STM`_ (software transactional memory): We have released a first working version, + and continue to try out new promising paths of achieving a fast multithreaded Python + +* `NumPy`_ which requires installation of our fork of upstream numpy, + available `on bitbucket`_ + +.. _`Bromeliad`: http://xkcd.com/1498 +.. _`Py3k`: http://pypy.org/py3donate.html +.. _`STM`: http://pypy.org/tmdonate2.html +.. _`NumPy`: http://pypy.org/numpydonate.html +.. _`on bitbucket`: https://www.bitbucket.org/pypy/numpy + +We would also like to encourage new people to join the project. PyPy has many +layers and we need help with all of them: `PyPy`_ and `Rpython`_ documentation +improvements, tweaking popular `modules`_ to run on pypy, or general `help`_ with making +Rpython's JIT even better. + +.. _`PyPy`: http://doc.pypy.org +.. _`Rpython`: http://rpython.readthedocs.org +.. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly +.. _`help`: http://doc.pypy.org/en/latest/project-ideas.html + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7. It's fast (`pypy and cpython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +This release supports **x86** machines on most common operating systems +(Linux 32/64, Mac OS X 64, Windows, and OpenBSD), +as well as newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux. + +While we support 32 bit python on Windows, work on the native Windows 64 +bit python is still stalling, we would welcome a volunteer +to `handle that`_. + +.. _`pypy and cpython 2.7.x`: http://speed.pypy.org +.. _`handle that`: http://doc.pypy.org/en/latest/windows.html#what-is-missing-for-a-full-64-bit-translation + +Highlights +========== + +* The past months have seen pypy mature and grow, as rpython becomes the goto + solution for writing fast dynamic language interpreters. Our separation of + Rpython and the python interpreter PyPy is now much clearer in the + `PyPy documentation`_ and we now have seperate `RPython documentation`_. + Tell us what still isn't clear, or even better help us improve the documentation. + +* We merged version 2.7.9 of python's stdlib. From the python release notice: + + * The entirety of Python 3.4's `ssl module`_ has been backported. + See `PEP 466`_ for justification. + + * HTTPS certificate validation using the system's certificate store is now + enabled by default. See `PEP 476`_ for details. + + * SSLv3 has been disabled by default in httplib and its reverse dependencies + due to the `POODLE attack`_. + + * The `ensurepip module`_ has been backported, which provides the pip + package manager in every Python 2.7 installation. See `PEP 477`_. + +* The garbage collector now ignores parts of the stack which did not change + since the last collection, another performance boost + +* errno and LastError are saved around cffi calls so things like pdb will not + overwrite it + +* We continue to asymptotically approach a score of 7 times faster than cpython + on our benchmark suite, we now rank 6.98 on latest runs + +* Issues reported with our previous release were resolved_ after reports from users on + our issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at + #pypy. + +.. _`PyPy documentation`: http://doc.pypy.org +.. _`RPython documentation`: http://rpython.readthedocs.org +.. _`ssl module`: https://docs.python.org/3/library/ssl.html +.. _`PEP 466`: https://www.python.org/dev/peps/pep-0466 +.. _`PEP 476`: https://www.python.org/dev/peps/pep-0476 +.. _`PEP 477`: https://www.python.org/dev/peps/pep-0477 +.. _`POODLE attack`: https://www.imperialviolet.org/2014/10/14/poodle.html +.. _`ensurepip module`: https://docs.python.org/2/library/ensurepip.html +.. _resolved: http://doc.pypy.org/en/latest/whatsnew-2.5.1.html + +Please try it out and let us know what you think. We welcome +success stories, `experiments`_, or `benchmarks`_, we know you are using PyPy, please tell us about it! + +Cheers + +The PyPy Team + +.. _`experiments`: http://morepypy.blogspot.com/2015/02/experiments-in-pyrlang-with-rpython.html +.. _`benchmarks`: https://mithrandi.net/blog/2015/03/axiom-benchmark-results-on-pypy-2-5-0 diff --git a/pypy/doc/stm.rst b/pypy/doc/stm.rst --- a/pypy/doc/stm.rst +++ b/pypy/doc/stm.rst @@ -25,8 +25,8 @@ .. _`2nd call for donation`: http://pypy.org/tmdonate2.html -Introduction -============ +What pypy-stm is for +==================== ``pypy-stm`` is a variant of the regular PyPy interpreter. (This version supports Python 2.7; see below for `Python 3`_.) With caveats_ @@ -45,15 +45,36 @@ it as a drop-in replacement and multithreaded programs will run on multiple cores. -* ``pypy-stm`` does not impose any special API to the user, but it - provides a new pure Python module called `transactional_memory`_ with - features to inspect the state or debug conflicts_ that prevent - parallelization. This module can also be imported on top of a non-STM - PyPy or CPython. +* ``pypy-stm`` provides (but does not impose) a special API to the + user in the pure Python module ``transaction``. This module is based + on the lower-level module ``pypystm``, but also provides some + compatibily with non-STM PyPy's or CPython's. * Building on top of the way the GIL is removed, we will talk - about `Atomic sections, Transactions, etc.: a better way to write - parallel programs`_. + about `How to write multithreaded programs: the 10'000-feet view`_ + and `transaction.TransactionQueue`_. + + +...and what pypy-stm is not for +------------------------------- + +``pypy-stm`` gives a Python without the GIL. This means that it is +useful in situations where the GIL is the problem in the first place. +(This includes cases where the program can easily be modified to run +in multiple threads; often, we don't consider doing that precisely +because of the GIL.) + +However, there are plenty of cases where the GIL is not the problem. +Do not hope ``pypy-stm`` to be helpful in these cases! This includes +all programs that use multiple threads but don't actually spend a lot +of time running Python code. For example, it may be spending all its +time waiting for I/O to occur, or performing some long computation on +a huge matrix. These are cases where the CPU is either idle, or in +some C/Fortran library anyway; in both cases, the interpreter (either +CPython or the regular PyPy) should release the GIL around the +external calls. The threads will thus not end up fighting for the +GIL. + Getting Started @@ -63,9 +84,10 @@ Development is done in the branch `stmgc-c7`_. If you are only interested in trying it out, you can download a Ubuntu binary here__ -(``pypy-stm-2.3*.tar.bz2``, Ubuntu 12.04-14.04). The current version +(``pypy-stm-2.*.tar.bz2``, for Ubuntu 12.04-14.04). The current version supports four "segments", which means that it will run up to four -threads in parallel. +threads in parallel. (Development recently switched to `stmgc-c8`_, +but that is not ready for trying out yet.) To build a version from sources, you first need to compile a custom version of clang(!); we recommend downloading `llvm and clang like @@ -78,6 +100,7 @@ rpython/bin/rpython -Ojit --stm pypy/goal/targetpypystandalone.py .. _`stmgc-c7`: https://bitbucket.org/pypy/pypy/src/stmgc-c7/ +.. _`stmgc-c8`: https://bitbucket.org/pypy/pypy/src/stmgc-c8/ .. __: https://bitbucket.org/pypy/pypy/downloads/ .. __: http://clang.llvm.org/get_started.html .. __: https://bitbucket.org/pypy/stmgc/src/default/c7/llvmfix/ @@ -85,54 +108,72 @@ .. _caveats: -Current status --------------- +Current status (stmgc-c7) +------------------------- -* So far, small examples work fine, but there are still a few bugs. - We're busy fixing them as we find them; feel free to `report bugs`_. +* **NEW:** It seems to work fine, without crashing any more. Please `report + any crash`_ you find (or other bugs). * It runs with an overhead as low as 20% on examples like "richards". There are also other examples with higher overheads --currently up to 2x for "translate.py"-- which we are still trying to understand. One suspect is our partial GC implementation, see below. +* **NEW:** the ``PYPYSTM`` environment variable and the + ``pypy/stm/print_stm_log.py`` script let you know exactly which + "conflicts" occurred. This is described in the section + `transaction.TransactionQueue`_ below. + +* **NEW:** special transaction-friendly APIs (like ``stmdict``), + described in the section `transaction.TransactionQueue`_ below. The + old API changed again, mostly moving to different modules. Sorry + about that. I feel it's a better idea to change the API early + instead of being stuck with a bad one later... + * Currently limited to 1.5 GB of RAM (this is just a parameter in - `core.h`__). Memory overflows are not correctly handled; they cause - segfaults. + `core.h`__ -- theoretically. In practice, increase it too much and + clang crashes again). Memory overflows are not correctly handled; + they cause segfaults. -* The JIT warm-up time improved recently but is still bad. In order to - produce machine code, the JIT needs to enter a special single-threaded - mode for now. This means that you will get bad performance results if - your program doesn't run for several seconds, where *several* can mean - *many.* When trying benchmarks, be sure to check that you have - reached the warmed state, i.e. the performance is not improving any - more. This should be clear from the fact that as long as it's - producing more machine code, ``pypy-stm`` will run on a single core. +* **NEW:** The JIT warm-up time improved again, but is still + relatively large. In order to produce machine code, the JIT needs + to enter "inevitable" mode. This means that you will get bad + performance results if your program doesn't run for several seconds, + where *several* can mean *many.* When trying benchmarks, be sure to + check that you have reached the warmed state, i.e. the performance + is not improving any more. * The GC is new; although clearly inspired by PyPy's regular GC, it misses a number of optimizations for now. Programs allocating large numbers of small objects that don't immediately die (surely a common - situation) suffer from these missing optimizations. + situation) suffer from these missing optimizations. (The bleeding + edge ``stmgc-c8`` is better at that.) -* The GC has no support for destructors: the ``__del__`` method is never - called (including on file objects, which won't be closed for you). - This is of course temporary. Also, weakrefs might appear to work a - bit strangely for now (staying alive even though ``gc.collect()``, or - even dying but then un-dying for a short time before dying again). +* Weakrefs might appear to work a bit strangely for now, sometimes + staying alive throught ``gc.collect()``, or even dying but then + un-dying for a short time before dying again. A similar problem can + show up occasionally elsewhere with accesses to some external + resources, where the (apparent) serialized order doesn't match the + underlying (multithreading) order. These are bugs (partially fixed + already in ``stmgc-c8``). Also, debugging helpers like + ``weakref.getweakrefcount()`` might give wrong answers. * The STM system is based on very efficient read/write barriers, which are mostly done (their placement could be improved a bit in - JIT-generated machine code). But the overall bookkeeping logic could - see more improvements (see `Low-level statistics`_ below). + JIT-generated machine code). * Forking the process is slow because the complete memory needs to be copied manually. A warning is printed to this effect. * Very long-running processes (on the order of days) will eventually crash on an assertion error because of a non-implemented overflow of - an internal 29-bit number. + an internal 28-bit counter. -.. _`report bugs`: https://bugs.pypy.org/ +* The recursion detection code was not reimplemented. Infinite + recursion just segfaults for now. + + +.. _`report any crash`: https://bitbucket.org/pypy/pypy/issues?status=new&status=open .. __: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/rpython/translator/stm/src_stm/stm/core.h @@ -155,10 +196,41 @@ interpreter and other ones might have slightly different needs. - User Guide ========== +How to write multithreaded programs: the 10'000-feet view +--------------------------------------------------------- + +PyPy-STM offers two ways to write multithreaded programs: + +* the traditional way, using the ``thread`` or ``threading`` modules, + described first__. + +* using ``TransactionQueue``, described next__, as a way to hide the + low-level notion of threads. + +.. __: `Drop-in replacement`_ +.. __: `transaction.TransactionQueue`_ + +The issues with low-level threads are well known (particularly in other +languages that don't have GIL-based interpreters): memory corruption, +deadlocks, livelocks, and so on. There are alternative approaches to +dealing directly with threads, like OpenMP_. These approaches +typically enforce some structure on your code. ``TransactionQueue`` +is in part similar: your program needs to have "some chances" of +parallelization before you can apply it. But I believe that the scope +of applicability is much larger with ``TransactionQueue`` than with +other approaches. It usually works without forcing a complete +reorganization of your existing code, and it works on any Python +program which has got *latent* and *imperfect* parallelism. Ideally, +it only requires that the end programmer identifies where this +parallelism is likely to be found, and communicates it to the system +using a simple API. + +.. _OpenMP: http://en.wikipedia.org/wiki/OpenMP + + Drop-in replacement ------------------- @@ -175,29 +247,168 @@ This works by internally considering the points where a standard PyPy or CPython would release the GIL, and replacing them with the boundaries of -"transaction". Like their database equivalent, multiple transactions +"transactions". Like their database equivalent, multiple transactions can execute in parallel, but will commit in some serial order. They appear to behave as if they were completely run in this serialization order. +transaction.TransactionQueue +---------------------------- + +In CPU-hungry programs, we can often easily identify outermost loops +over some data structure, or other repetitive algorithm, where each +"block" consists of processing a non-trivial amount of data, and where +the blocks "have a good chance" to be independent from each other. We +don't need to prove that they are actually independent: it is enough +if they are *often independent* --- or, more precisely, if we *think +they should be* often independent. + +One typical example would look like this, where the function ``func()`` +typically invokes a large amount of code:: + + for key, value in bigdict.items(): + func(key, value) + +Then you simply replace the loop with:: + + from transaction import TransactionQueue + + tr = TransactionQueue() + for key, value in bigdict.items(): + tr.add(func, key, value) + tr.run() + +This code's behavior is equivalent. Internally, the +``TransactionQueue`` object will start N threads and try to run the +``func(key, value)`` calls on all threads in parallel. But note the +difference with a regular thread-pooling library, as found in many +lower-level languages than Python: the function calls are not randomly +interleaved with each other just because they run in parallel. The +behavior did not change because we are using ``TransactionQueue``. +All the calls still *appear* to execute in some serial order. + +A typical usage of ``TransactionQueue`` goes like that: at first, +the performance does not increase. +In fact, it is likely to be worse. Typically, this is +indicated by the total CPU usage, which remains low (closer to 1 than +N cores). First note that it is expected that the CPU usage should +not go much higher than 1 in the JIT warm-up phase: you must run a +program for several seconds, or for larger programs at least one +minute, to give the JIT a chance to warm up enough. But if CPU usage +remains low even afterwards, then the ``PYPYSTM`` environment variable +can be used to track what is going on. + +Run your program with ``PYPYSTM=logfile`` to produce a log file called +``logfile``. Afterwards, use the ``pypy/stm/print_stm_log.py`` +utility to inspect the content of this log file. It produces output +like this (sorted by amount of time lost, largest first):: + + 10.5s lost in aborts, 1.25s paused (12412x STM_CONTENTION_WRITE_WRITE) + File "foo.py", line 10, in f + someobj.stuff = 5 + File "bar.py", line 20, in g + someobj.other = 10 + +This means that 10.5 seconds were lost running transactions that were +aborted (which caused another 1.25 seconds of lost time by pausing), +because of the reason shown in the two independent single-entry +tracebacks: one thread ran the line ``someobj.stuff = 5``, whereas +another thread concurrently ran the line ``someobj.other = 10`` on the +same object. These two writes are done to the same object. This +causes a conflict, which aborts one of the two transactions. In the +example above this occurred 12412 times. + +The two other conflict sources are ``STM_CONTENTION_INEVITABLE``, +which means that two transactions both tried to do an external +operation, like printing or reading from a socket or accessing an +external array of raw data; and ``STM_CONTENTION_WRITE_READ``, which +means that one transaction wrote to an object but the other one merely +read it, not wrote to it (in that case only the writing transaction is +reported; the location for the reads is not recorded because doing so +is not possible without a very large performance impact). + +Common causes of conflicts: + +* First of all, any I/O or raw manipulation of memory turns the + transaction inevitable ("must not abort"). There can be only one + inevitable transaction running at any time. A common case is if + each transaction starts with sending data to a log file. You should + refactor this case so that it occurs either near the end of the + transaction (which can then mostly run in non-inevitable mode), or + delegate it to a separate transaction or even a separate thread. + +* Writing to a list or a dictionary conflicts with any read from the + same list or dictionary, even one done with a different key. For + dictionaries and sets, you can try the types ``transaction.stmdict`` + and ``transaction.stmset``, which behave mostly like ``dict`` and + ``set`` but allow concurrent access to different keys. (What is + missing from them so far is lazy iteration: for example, + ``stmdict.iterkeys()`` is implemented as ``iter(stmdict.keys())``; + and, unlike PyPy's dictionaries and sets, the STM versions are not + ordered.) There are also experimental ``stmiddict`` and + ``stmidset`` classes using the identity of the key. + +* ``time.time()`` and ``time.clock()`` turn the transaction inevitable + in order to guarantee that a call that appears to be later will really + return a higher number. If getting slightly unordered results is + fine, use ``transaction.time()`` or ``transaction.clock()``. The + latter operations guarantee to return increasing results only if you + can "prove" that two calls occurred in a specific order (for example + because they are both called by the same thread). In cases where no + such proof is possible, you might get randomly interleaved values. + (If you have two independent transactions, they normally behave as if + one of them was fully executed before the other; but using + ``transaction.time()`` you might see the "hidden truth" that they are + actually interleaved.) + +* ``transaction.threadlocalproperty`` can be used at class-level:: + + class Foo(object): # must be a new-style class! + x = transaction.threadlocalproperty() + y = transaction.threadlocalproperty(dict) + + This declares that instances of ``Foo`` have two attributes ``x`` + and ``y`` that are thread-local: reading or writing them from + concurrently-running transactions will return independent results. + (Any other attributes of ``Foo`` instances will be globally visible + from all threads, as usual.) The optional argument to + ``threadlocalproperty()`` is the default value factory: in case no + value was assigned in the current thread yet, the factory is called + and its result becomes the value in that thread (like + ``collections.defaultdict``). If no default value factory is + specified, uninitialized reads raise ``AttributeError``. Note that + with ``TransactionQueue`` you get a pool of a fixed number of + threads, each running the transactions one after the other; such + thread-local properties will have the value last stored in them in + the same thread,, which may come from a random previous transaction. + This means that ``threadlocalproperty`` is useful mainly to avoid + conflicts from cache-like data structures. + +Note that Python is a complicated language; there are a number of less +common cases that may cause conflict (of any kind) where we might not +expect it at priori. In many of these cases it could be fixed; please +report any case that you don't understand. + + Atomic sections --------------- -PyPy supports *atomic sections,* which are blocks of code which you want -to execute without "releasing the GIL". *This is experimental and may -be removed in the future.* In STM terms, this means blocks of code that -are executed while guaranteeing that the transaction is not interrupted -in the middle. +The ``TransactionQueue`` class described above is based on *atomic +sections,* which are blocks of code which you want to execute without +"releasing the GIL". In STM terms, this means blocks of code that are +executed while guaranteeing that the transaction is not interrupted in +the middle. *This is experimental and may be removed in the future* +if `Software lock elision`_ is ever implemented. -Here is a usage example:: +Here is a direct usage example:: - with __pypy__.thread.atomic: + with transaction.atomic: assert len(lst1) == 10 x = lst1.pop(0) lst1.append(x) -In this (bad) example, we are sure that the item popped off one end of +In this example, we are sure that the item popped off one end of the list is appened again at the other end atomically. It means that another thread can run ``len(lst1)`` or ``x in lst1`` without any particular synchronization, and always see the same results, @@ -221,25 +432,27 @@ it likely that such a piece of code will eventually block all other threads anyway. -Note that if you want to experiment with ``atomic``, you may have to add -manually a transaction break just before the atomic block. This is +Note that if you want to experiment with ``atomic``, you may have to +manually add a transaction break just before the atomic block. This is because the boundaries of the block are not guaranteed to be the boundaries of the transaction: the latter is at least as big as the -block, but maybe bigger. Therefore, if you run a big atomic block, it +block, but may be bigger. Therefore, if you run a big atomic block, it is a good idea to break the transaction just before. This can be done -e.g. by the hack of calling ``time.sleep(0)``. (This may be fixed at +by calling ``transaction.hint_commit_soon()``. (This may be fixed at some point.) -There are also issues with the interaction of locks and atomic blocks. -This can be seen if you write to files (which have locks), including -with a ``print`` to standard output. If one thread tries to acquire a -lock while running in an atomic block, and another thread has got the -same lock, then the former may fail with a ``thread.error``. The reason -is that "waiting" for some condition to become true --while running in -an atomic block-- does not really make sense. For now you can work -around it by making sure that, say, all your prints are either in an -``atomic`` block or none of them are. (This kind of issue is -theoretically hard to solve.) +There are also issues with the interaction of regular locks and atomic +blocks. This can be seen if you write to files (which have locks), +including with a ``print`` to standard output. If one thread tries to +acquire a lock while running in an atomic block, and another thread +has got the same lock at that point, then the former may fail with a +``thread.error``. (Don't rely on it; it may also deadlock.) +The reason is that "waiting" for some condition to +become true --while running in an atomic block-- does not really make +sense. For now you can work around it by making sure that, say, all +your prints are either in an ``atomic`` block or none of them are. +(This kind of issue is theoretically hard to solve and may be the +reason for atomic block support to eventually be removed.) Locks @@ -293,106 +506,38 @@ .. _`software lock elision`: https://www.repository.cam.ac.uk/handle/1810/239410 -Atomic sections, Transactions, etc.: a better way to write parallel programs ----------------------------------------------------------------------------- +Miscellaneous functions +----------------------- -(This section is based on locks as we plan to implement them, but also -works with the existing atomic sections.) - -In the cases where elision works, the block of code can run in parallel -with other blocks of code *even if they are protected by the same lock.* -You still get the illusion that the blocks are run sequentially. This -works even for multiple threads that run each a series of such blocks -and nothing else, protected by one single global lock. This is -basically the Python application-level equivalent of what was done with -the interpreter in ``pypy-stm``: while you think you are writing -thread-unfriendly code because of this global lock, actually the -underlying system is able to make it run on multiple cores anyway. - -This capability can be hidden in a library or in the framework you use; -the end user's code does not need to be explicitly aware of using -threads. For a simple example of this, there is `transaction.py`_ in -``lib_pypy``. The idea is that you write, or already have, some program -where the function ``f(key, value)`` runs on every item of some big -dictionary, say:: - - for key, value in bigdict.items(): - f(key, value) - -Then you simply replace the loop with:: - - for key, value in bigdict.items(): - transaction.add(f, key, value) - transaction.run() - -This code runs the various calls to ``f(key, value)`` using a thread -pool, but every single call is executed under the protection of a unique -lock. The end result is that the behavior is exactly equivalent --- in -fact it makes little sense to do it in this way on a non-STM PyPy or on -CPython. But on ``pypy-stm``, the various locked calls to ``f(key, -value)`` can tentatively be executed in parallel, even if the observable -result is as if they were executed in some serial order. - -This approach hides the notion of threads from the end programmer, -including all the hard multithreading-related issues. This is not the -first alternative approach to explicit threads; for example, OpenMP_ is -one. However, it is one of the first ones which does not require the -code to be organized in a particular fashion. Instead, it works on any -Python program which has got latent, imperfect parallelism. Ideally, it -only requires that the end programmer identifies where this parallelism -is likely to be found, and communicates it to the system, using for -example the ``transaction.add()`` scheme. - -.. _`transaction.py`: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/lib_pypy/transaction.py -.. _OpenMP: http://en.wikipedia.org/wiki/OpenMP - - -.. _`transactional_memory`: - -API of transactional_memory ---------------------------- - -The new pure Python module ``transactional_memory`` runs on both CPython -and PyPy, both with and without STM. It contains: - -* ``getsegmentlimit()``: return the number of "segments" in +* ``transaction.getsegmentlimit()``: return the number of "segments" in this pypy-stm. This is the limit above which more threads will not be able to execute on more cores. (Right now it is limited to 4 due to inter-segment overhead, but should be increased in the future. It should also be settable, and the default value should depend on the number of actual CPUs.) If STM is not available, this returns 1. -* ``print_abort_info(minimum_time=0.0)``: debugging help. Each thread - remembers the longest abort or pause it did because of cross-thread - contention_. This function prints it to ``stderr`` if the time lost - is greater than ``minimum_time`` seconds. The record is then - cleared, to make it ready for new events. This function returns - ``True`` if it printed a report, and ``False`` otherwise. +* ``__pypy__.thread.signals_enabled``: a context manager that runs its + block of code with signals enabled. By default, signals are only + enabled in the main thread; a non-main thread will not receive + signals (this is like CPython). Enabling signals in non-main + threads is useful for libraries where threads are hidden and the end + user is not expecting his code to run elsewhere than in the main + thread. +* ``pypystm.exclusive_atomic``: a context manager similar to + ``transaction.atomic`` but which complains if it is nested. -API of __pypy__.thread ----------------------- +* ``transaction.is_atomic()``: return True if called from an atomic + context. -The ``__pypy__.thread`` submodule is a built-in module of PyPy that -contains a few internal built-in functions used by the -``transactional_memory`` module, plus the following: +* ``pypystm.count()``: return a different positive integer every time + it is called. This works without generating conflicts. The + returned integers are only roughly in increasing order; this should + not be relied upon. -* ``__pypy__.thread.atomic``: a context manager to run a block in - fully atomic mode, without "releasing the GIL". (May be eventually - removed?) -* ``__pypy__.thread.signals_enabled``: a context manager that runs its - block with signals enabled. By default, signals are only enabled in - the main thread; a non-main thread will not receive signals (this is - like CPython). Enabling signals in non-main threads is useful for - libraries where threads are hidden and the end user is not expecting - his code to run elsewhere than in the main thread. - - -.. _contention: - -Conflicts ---------- +More details about conflicts +---------------------------- Based on Software Transactional Memory, the ``pypy-stm`` solution is prone to "conflicts". To repeat the basic idea, threads execute their code @@ -408,25 +553,26 @@ the transaction). If this occurs too often, parallelization fails. How much actual parallelization a multithreaded program can see is a bit -subtle. Basically, a program not using ``__pypy__.thread.atomic`` or +subtle. Basically, a program not using ``transaction.atomic`` or eliding locks, or doing so for very short amounts of time, will parallelize almost freely (as long as it's not some artificial example where, say, all threads try to increase the same global counter and do nothing else). -However, using if the program requires longer transactions, it comes +However, if the program requires longer transactions, it comes with less obvious rules. The exact details may vary from version to version, too, until they are a bit more stabilized. Here is an overview. Parallelization works as long as two principles are respected. The -first one is that the transactions must not *conflict* with each other. -The most obvious sources of conflicts are threads that all increment a -global shared counter, or that all store the result of their -computations into the same list --- or, more subtly, that all ``pop()`` -the work to do from the same list, because that is also a mutation of -the list. (It is expected that some STM-aware library will eventually -be designed to help with conflict problems, like a STM-aware queue.) +first one is that the transactions must not *conflict* with each +other. The most obvious sources of conflicts are threads that all +increment a global shared counter, or that all store the result of +their computations into the same list --- or, more subtly, that all +``pop()`` the work to do from the same list, because that is also a +mutation of the list. (You can work around it with +``transaction.stmdict``, but for that specific example, some STM-aware +queue should eventually be designed.) A conflict occurs as follows: when a transaction commits (i.e. finishes successfully) it may cause other transactions that are still in progress @@ -442,22 +588,23 @@ Another issue is that of avoiding long-running so-called "inevitable" transactions ("inevitable" is taken in the sense of "which cannot be avoided", i.e. transactions which cannot abort any more). Transactions -like that should only occur if you use ``__pypy__.thread.atomic``, -generally become of I/O in atomic blocks. They work, but the +like that should only occur if you use ``atomic``, +generally because of I/O in atomic blocks. They work, but the transaction is turned inevitable before the I/O is performed. For all the remaining execution time of the atomic block, they will impede parallel work. The best is to organize the code so that such operations -are done completely outside ``__pypy__.thread.atomic``. +are done completely outside ``atomic``. -(This is related to the fact that blocking I/O operations are +(This is not unrelated to the fact that blocking I/O operations are discouraged with Twisted, and if you really need them, you should do them on their own separate thread.) -In case of lock elision, we don't get long-running inevitable -transactions, but a different problem can occur: doing I/O cancels lock -elision, and the lock turns into a real lock, preventing other threads -from committing if they also need this lock. (More about it when lock -elision is implemented and tested.) +In case lock elision eventually replaces atomic sections, we wouldn't +get long-running inevitable transactions, but the same problem occurs +in a different way: doing I/O cancels lock elision, and the lock turns +into a real lock. This prevents other threads from committing if they +also need this lock. (More about it when lock elision is implemented +and tested.) @@ -467,56 +614,30 @@ XXX this section mostly empty for now -Low-level statistics --------------------- +Technical reports +----------------- -When a non-main thread finishes, you get low-level statistics printed to -stderr, looking like that:: +STMGC-C7 is described in detail in a `technical report`__. - thread 0x7f73377fe600: - outside transaction 42182 0.506 s - run current 85466 0.000 s - run committed 34262 3.178 s - run aborted write write 6982 0.083 s - run aborted write read 550 0.005 s - run aborted inevitable 388 0.010 s - run aborted other 0 0.000 s - wait free segment 0 0.000 s - wait write read 78 0.027 s - wait inevitable 887 0.490 s - wait other 0 0.000 s - sync commit soon 1 0.000 s - bookkeeping 51418 0.606 s - minor gc 162970 1.135 s - major gc 1 0.019 s - sync pause 59173 1.738 s - longest recordered marker 0.000826 s - "File "x.py", line 5, in f" +A separate `position paper`__ gives an overview of our position about +STM in general. -On each line, the first number is a counter, and the second number gives -the associated time --- the amount of real time that the thread was in -this state. The sum of all the times should be equal to the total time -between the thread's start and the thread's end. The most important -points are "run committed", which gives the amount of useful work, and -"outside transaction", which should give the time spent e.g. in library -calls (right now it seems to be larger than that; to investigate). The -various "run aborted" and "wait" entries are time lost due to -conflicts_. Everything else is overhead of various forms. (Short-, -medium- and long-term future work involves reducing this overhead :-) - -The last two lines are special; they are an internal marker read by -``transactional_memory.print_abort_info()``. +.. __: http://bitbucket.org/pypy/extradoc/src/extradoc/talk/dls2014/paper/paper.pdf +.. __: http://bitbucket.org/pypy/extradoc/src/extradoc/talk/icooolps2014/ Reference to implementation details ----------------------------------- -The core of the implementation is in a separate C library called stmgc_, -in the c7_ subdirectory. Please see the `README.txt`_ for more -information. In particular, the notion of segment is discussed there. +The core of the implementation is in a separate C library called +stmgc_, in the c7_ subdirectory (current version of pypy-stm) and in +the c8_ subdirectory (bleeding edge version). Please see the +`README.txt`_ for more information. In particular, the notion of +segment is discussed there. .. _stmgc: https://bitbucket.org/pypy/stmgc/src/default/ .. _c7: https://bitbucket.org/pypy/stmgc/src/default/c7/ +.. _c8: https://bitbucket.org/pypy/stmgc/src/default/c8/ .. _`README.txt`: https://bitbucket.org/pypy/stmgc/raw/default/c7/README.txt PyPy itself adds on top of it the automatic placement of read__ and write__ diff --git a/pypy/doc/whatsnew-2.5.0.rst b/pypy/doc/whatsnew-2.5.0.rst --- a/pypy/doc/whatsnew-2.5.0.rst +++ b/pypy/doc/whatsnew-2.5.0.rst @@ -1,6 +1,6 @@ -======================= -What's new in PyPy 2.5 -======================= +======================== +What's new in PyPy 2.5.0 +======================== .. this is a revision shortly after release-2.4.x .. startrev: 7026746cbb1b diff --git a/pypy/doc/whatsnew-2.5.1.rst b/pypy/doc/whatsnew-2.5.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-2.5.1.rst @@ -0,0 +1,47 @@ +======================== +What's new in PyPy 2.5.1 +======================== + +.. this is a revision shortly after release-2.5.0 +.. startrev: 397b96217b85 + + +Non-blocking file reads sometimes raised EAGAIN even though they +had buffered data waiting, fixed in b1c4fcb04a42 + +Fix a bug in cpyext in multithreded programs acquiring/releasing the GIL + +.. branch: vmprof + +.. branch: stackroot-speedup-2 + +Avoid tracing all stack roots during repeated minor collections, +by ignoring the part of the stack that didn't change + +.. branch: stdlib-2.7.9 + +Update stdlib to version 2.7.9 + +.. branch: fix-kqueue-error2 + +Fix exception being raised by kqueue.control (CPython compatibility) + +.. branch: gitignore + +.. branch: framestate2 + +Refactor rpython.flowspace.framestate.FrameState. + +.. branch: alt_errno + +Add an alternative location to save LastError, errno around ctypes, +cffi external calls so things like pdb will not overwrite it + +.. branch: nonquadratic-heapcache + +Speed up the warmup times of the JIT by removing a quadratic algorithm in the +heapcache. + +.. branch: online-transforms-2 + +Simplify flow graphs on the fly during annotation phase. diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -2,35 +2,6 @@ What's new in PyPy 2.5+ ======================= -.. this is a revision shortly after release-2.5.x +.. this is a revision shortly after release-2.5.1 .. startrev: 397b96217b85 - -Non-blocking file reads sometimes raised EAGAIN even though they -had buffered data waiting, fixed in b1c4fcb04a42 - - -.. branch: vmprof - -.. branch: stackroot-speedup-2 -Avoid tracing all stack roots during repeated minor collections, -by ignoring the part of the stack that didn't change - -.. branch: stdlib-2.7.9 -Update stdlib to version 2.7.9 - -.. branch: fix-kqueue-error2 -Fix exception being raised by kqueue.control (CPython compatibility) - -.. branch: gitignore - -.. branch: framestate2 -Refactor rpython.flowspace.framestate.FrameState. - -.. branch: alt_errno -Add an alternative location to save LastError, errno around ctypes, -cffi external calls so things like pdb will not overwrite it - -.. branch: nonquadratic-heapcache -Speed up the warmup times of the JIT by removing a quadratic algorithm in the -heapcache. diff --git a/pypy/goal/getnightly.py b/pypy/goal/getnightly.py --- a/pypy/goal/getnightly.py +++ b/pypy/goal/getnightly.py @@ -7,7 +7,7 @@ if sys.platform.startswith('linux'): arch = 'linux' cmd = 'wget "%s"' - tar = "tar -x -v --wildcards --strip-components=2 -f %s '*/bin/pypy'" + tar = "tar -x -v --wildcards --strip-components=2 -f %s '*/bin/pypy' '*/bin/libpypy-c.so'" if os.uname()[-1].startswith('arm'): arch += '-armhf-raspbian' elif sys.platform.startswith('darwin'): diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py --- a/pypy/interpreter/astcompiler/assemble.py +++ b/pypy/interpreter/astcompiler/assemble.py @@ -648,7 +648,7 @@ def _compute_UNPACK_SEQUENCE(arg): - return arg + 1 + return arg - 1 def _compute_DUP_TOPX(arg): return arg diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py --- a/pypy/interpreter/astcompiler/test/test_compiler.py +++ b/pypy/interpreter/astcompiler/test/test_compiler.py @@ -759,6 +759,19 @@ """ self.simple_test(source, 'l', [1, 2]) + def test_unpack_wrong_stackeffect(self): + source = """if 1: + l = [1, 2] + a, b = l + a, b = l + a, b = l + a, b = l + a, b = l + a, b = l + """ + code = compile_with_astcompiler(source, 'exec', self.space) + assert code.co_stacksize == 2 + def test_lambda(self): yield self.st, "y = lambda x: x", "y(4)", 4 diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -4,7 +4,7 @@ The bytecode interpreter itself is implemented by the PyFrame class. """ -import dis, imp, struct, types, new, sys +import dis, imp, struct, types, new, sys, os from pypy.interpreter import eval from pypy.interpreter.signature import Signature @@ -128,6 +128,17 @@ if (self.magic == cpython_magic and '__pypy__' not in sys.builtin_module_names): raise Exception("CPython host codes should not be rendered") + # When translating PyPy, freeze the file name + # /lastdirname/basename.py + # instead of freezing the complete translation-time path. + filename = self.co_filename.lstrip('<').rstrip('>') + if filename.lower().endswith('.pyc'): + filename = filename[:-1] + basename = os.path.basename(filename) + lastdirname = os.path.basename(os.path.dirname(filename)) + if lastdirname: + basename = '%s/%s' % (lastdirname, basename) + self.co_filename = '/%s' % (basename,) co_names = property(lambda self: [self.space.unwrap(w_name) for w_name in self.co_names_w]) # for trace diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -1619,6 +1619,13 @@ def prepare_exec(f, prog, globals, locals, compile_flags, builtin, codetype): """Manipulate parameters to exec statement to (codeobject, dict, dict). """ + if (globals is None and locals is None and + isinstance(prog, tuple) and + (len(prog) == 2 or len(prog) == 3)): + globals = prog[1] + if len(prog) == 3: + locals = prog[2] + prog = prog[0] if globals is None: globals = f.f_globals if locals is None: diff --git a/pypy/interpreter/test/test_exec.py b/pypy/interpreter/test/test_exec.py --- a/pypy/interpreter/test/test_exec.py +++ b/pypy/interpreter/test/test_exec.py @@ -262,3 +262,11 @@ """] for c in code: compile(c, "", "exec") + + def test_exec_tuple(self): + # note: this is VERY different than testing exec("a = 42", d), because + # this specific case is handled specially by the AST compiler + d = {} + x = ("a = 42", d) + exec x + assert d['a'] == 42 diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -2,13 +2,15 @@ from pypy.interpreter.mixedmodule import MixedModule from rpython.rlib import rdynload +VERSION = "0.9.2" + class Module(MixedModule): appleveldefs = { } interpleveldefs = { - '__version__': 'space.wrap("0.9.0")', + '__version__': 'space.wrap("%s")' % VERSION, 'load_library': 'libraryobj.load_library', diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -3247,6 +3247,88 @@ cast(p, c)[1] += 500 assert list(a) == [10000, 20500, 30000] +def test_from_buffer_not_str_unicode_bytearray(): + BChar = new_primitive_type("char") + BCharP = new_pointer_type(BChar) + BCharA = new_array_type(BCharP, None) + py.test.raises(TypeError, from_buffer, BCharA, b"foo") + py.test.raises(TypeError, from_buffer, BCharA, u"foo") + py.test.raises(TypeError, from_buffer, BCharA, bytearray(b"foo")) + try: + from __builtin__ import buffer + except ImportError: + pass + else: + py.test.raises(TypeError, from_buffer, BCharA, buffer(b"foo")) + py.test.raises(TypeError, from_buffer, BCharA, buffer(u"foo")) + py.test.raises(TypeError, from_buffer, BCharA, + buffer(bytearray(b"foo"))) + try: + from __builtin__ import memoryview + except ImportError: + pass + else: + py.test.raises(TypeError, from_buffer, BCharA, memoryview(b"foo")) + py.test.raises(TypeError, from_buffer, BCharA, + memoryview(bytearray(b"foo"))) + +def test_from_buffer_more_cases(): + try: + from _cffi_backend import _testbuff + except ImportError: + py.test.skip("not for pypy") + BChar = new_primitive_type("char") + BCharP = new_pointer_type(BChar) + BCharA = new_array_type(BCharP, None) + # + def check1(bufobj, expected): + c = from_buffer(BCharA, bufobj) + assert typeof(c) is BCharA + if sys.version_info >= (3,): + expected = [bytes(c, "ascii") for c in expected] + assert list(c) == list(expected) + # + def check(methods, expected, expected_for_memoryview=None): + if sys.version_info >= (3,): + if methods <= 7: + return + if expected_for_memoryview is not None: + expected = expected_for_memoryview + class X(object): + pass + _testbuff(X, methods) + bufobj = X() + check1(bufobj, expected) + try: + from __builtin__ import buffer + bufobjb = buffer(bufobj) + except (TypeError, ImportError): + pass + else: + check1(bufobjb, expected) + try: + bufobjm = memoryview(bufobj) + except (TypeError, NameError): + pass + else: + check1(bufobjm, expected_for_memoryview or expected) + # + check(1, "RDB") + check(2, "WRB") + check(4, "CHB") + check(8, "GTB") + check(16, "ROB") + # + check(1 | 2, "RDB") + check(1 | 4, "RDB") + check(2 | 4, "CHB") + check(1 | 8, "RDB", "GTB") + check(1 | 16, "RDB", "ROB") + check(2 | 8, "WRB", "GTB") + check(2 | 16, "WRB", "ROB") + check(4 | 8, "CHB", "GTB") + check(4 | 16, "CHB", "ROB") + def test_version(): # this test is here mostly for PyPy - assert __version__ == "0.9.0" + assert __version__ == "0.9.2" diff --git a/pypy/module/_cffi_backend/test/test_file.py b/pypy/module/_cffi_backend/test/test_file.py --- a/pypy/module/_cffi_backend/test/test_file.py +++ b/pypy/module/_cffi_backend/test/test_file.py @@ -15,3 +15,15 @@ "Update test/_backend_test_c.py by copying it from " "https://bitbucket.org/cffi/cffi/raw/default/c/test_c.py " "and killing the import lines at the start") + +def test_egginfo_version(): + from pypy.module._cffi_backend import VERSION + line = "Version: %s\n" % VERSION + eggfile = py.path.local(__file__).join('..', '..', '..', '..', '..', + 'lib_pypy', 'cffi.egg-info') + assert line in eggfile.readlines() + +def test_app_version(): + from pypy.module import _cffi_backend + from lib_pypy import cffi + assert _cffi_backend.VERSION == cffi.__version__ diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -470,7 +470,7 @@ allow_surrogates=True) return space.newtuple([space.wrap(result), space.wrap(consumed)]) - at unwrap_spec(data=str, errors='str_or_None', byteorder=int, + at unwrap_spec(data='bufferstr', errors='str_or_None', byteorder=int, w_final=WrappedDefault(False)) def utf_16_ex_decode(space, data, errors='strict', byteorder=0, w_final=None): if errors is None: @@ -491,7 +491,7 @@ return space.newtuple([space.wrap(res), space.wrap(consumed), space.wrap(byteorder)]) - at unwrap_spec(data=str, errors='str_or_None', byteorder=int, + at unwrap_spec(data='bufferstr', errors='str_or_None', byteorder=int, w_final=WrappedDefault(False)) def utf_32_ex_decode(space, data, errors='strict', byteorder=0, w_final=None): final = space.is_true(w_final) @@ -589,7 +589,7 @@ "character mapping must return integer, None or str") - at unwrap_spec(string=str, errors='str_or_None') + at unwrap_spec(string='bufferstr', errors='str_or_None') def charmap_decode(space, string, errors="strict", w_mapping=None): if errors is None: errors = 'strict' diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -727,3 +727,23 @@ _codecs.register_error("test.test_codecs_not_a_string", f) raises(TypeError, u'\u1234'.encode, 'ascii', 'test.test_codecs_not_a_string') + + def test_decode_bytearray(self): + import _codecs + b = bytearray() + assert _codecs.ascii_decode(b) == (u'', 0) + assert _codecs.latin_1_decode(b) == (u'', 0) + assert _codecs.utf_7_decode(b) == (u'', 0) + assert _codecs.utf_8_decode(b) == (u'', 0) + assert _codecs.utf_16_be_decode(b) == (u'', 0) + assert _codecs.utf_16_decode(b) == (u'', 0) + assert _codecs.utf_16_le_decode(b) == (u'', 0) + assert _codecs.utf_16_ex_decode(b) == (u'', 0, 0) + assert _codecs.utf_32_decode(b) == (u'', 0) + assert _codecs.utf_32_be_decode(b) == (u'', 0) + assert _codecs.utf_32_le_decode(b) == (u'', 0) + assert _codecs.utf_32_ex_decode(b) == (u'', 0, 0) + assert _codecs.charmap_decode(b) == (u'', 0) + assert _codecs.unicode_escape_decode(b) == (u'', 0) + assert _codecs.raw_unicode_escape_decode(b) == (u'', 0) + assert _codecs.unicode_internal_decode(b) == (u'', 0) diff --git a/pypy/module/_csv/test/__init__.py b/pypy/module/_csv/test/__init__.py new file mode 100644 diff --git a/pypy/module/_io/test/__init__.py b/pypy/module/_io/test/__init__.py new file mode 100644 diff --git a/pypy/module/_multiprocessing/test/__init__.py b/pypy/module/_multiprocessing/test/__init__.py new file mode 100644 diff --git a/pypy/module/_random/interp_random.py b/pypy/module/_random/interp_random.py --- a/pypy/module/_random/interp_random.py +++ b/pypy/module/_random/interp_random.py @@ -4,7 +4,7 @@ from pypy.interpreter.typedef import TypeDef from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.baseobjspace import W_Root -from rpython.rlib.rarithmetic import r_uint, intmask +from rpython.rlib.rarithmetic import r_uint, intmask, widen from rpython.rlib import rbigint, rrandom, rstring @@ -54,8 +54,8 @@ def getstate(self, space): state = [None] * (rrandom.N + 1) for i in range(rrandom.N): - state[i] = space.newint(intmask(self._rnd.state[i])) - state[rrandom.N] = space.newint(self._rnd.index) + state[i] = space.wrap(widen(self._rnd.state[i])) + state[rrandom.N] = space.newlong(self._rnd.index) return space.newtuple(state) def setstate(self, space, w_state): diff --git a/pypy/module/_random/test/test_random.py b/pypy/module/_random/test/test_random.py --- a/pypy/module/_random/test/test_random.py +++ b/pypy/module/_random/test/test_random.py @@ -41,6 +41,17 @@ # does not crash rnd1.setstate((-1, ) * 624 + (0, )) + def test_state_repr(self): + # since app-level jumpahead salts with repr(state), + # it is important the repr is consistent with cpython + import _random + rnd = _random.Random() + rnd.seed(1234) + state = rnd.getstate() + s = repr(state) + assert len(s) == 7956 + assert s.count('L') == 625 + def test_seed(self): import _random, sys rnd = _random.Random() @@ -102,3 +113,10 @@ self.x = x r = R(x=15) assert r.x == 15 + + def test_exact_result(self): + # this passes on CPython 2.7.9 on Linux 32 and Linux 64 + import _random + rnd = _random.Random(-3**31) + x = rnd.random() + assert x == 0.8181851342382107 diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -30,7 +30,7 @@ space.wrap(addr.get_protocol()), space.wrap(addr.get_pkttype()), space.wrap(addr.get_hatype()), - space.wrap(addr.get_addr())]) + space.wrap(addr.get_haddr())]) elif rsocket.HAS_AF_UNIX and isinstance(addr, rsocket.UNIXAddress): return space.wrap(addr.get_path()) elif rsocket.HAS_AF_NETLINK and isinstance(addr, rsocket.NETLINKAddress): @@ -79,7 +79,7 @@ raise NotImplementedError # XXX Hack to seperate rpython and pypy -def addr_from_object(family, space, w_address): +def addr_from_object(family, fd, space, w_address): if family == rsocket.AF_INET: w_host, w_port = space.unpackiterable(w_address, 2) host = space.str_w(w_host) @@ -89,8 +89,9 @@ if family == rsocket.AF_INET6: pieces_w = space.unpackiterable(w_address) if not (2 <= len(pieces_w) <= 4): - raise TypeError("AF_INET6 address must be a tuple of length 2 " - "to 4, not %d" % len(pieces_w)) + raise oefmt(space.w_TypeError, + "AF_INET6 address must be a tuple of length 2 " + "to 4, not %d", len(pieces_w)) host = space.str_w(pieces_w[0]) port = space.int_w(pieces_w[1]) port = make_ushort_port(space, port) @@ -105,6 +106,28 @@ if rsocket.HAS_AF_NETLINK and family == rsocket.AF_NETLINK: w_pid, w_groups = space.unpackiterable(w_address, 2) return rsocket.NETLINKAddress(space.uint_w(w_pid), space.uint_w(w_groups)) + if rsocket.HAS_AF_PACKET and family == rsocket.AF_PACKET: + pieces_w = space.unpackiterable(w_address) + if not (2 <= len(pieces_w) <= 5): + raise oefmt(space.w_TypeError, + "AF_PACKET address must be a tuple of length 2 " + "to 5, not %d", len(pieces_w)) + ifname = space.str_w(pieces_w[0]) + ifindex = rsocket.PacketAddress.get_ifindex_from_ifname(fd, ifname) + protocol = space.int_w(pieces_w[1]) + if len(pieces_w) > 2: pkttype = space.int_w(pieces_w[2]) + else: pkttype = 0 + if len(pieces_w) > 3: hatype = space.int_w(pieces_w[3]) + else: hatype = 0 + if len(pieces_w) > 4: haddr = space.str_w(pieces_w[4]) + else: haddr = "" + if len(haddr) > 8: + raise OperationError(space.w_ValueError, space.wrap( + "Hardware address must be 8 bytes or less")) + if protocol < 0 or protocol > 0xfffff: + raise OperationError(space.w_OverflowError, space.wrap( + "protoNumber must be 0-65535.")) + return rsocket.PacketAddress(ifindex, protocol, pkttype, hatype, haddr) raise RSocketError("unknown address family") # XXX Hack to seperate rpython and pypy @@ -172,7 +195,8 @@ # convert an app-level object into an Address # based on the current socket's family def addr_from_object(self, space, w_address): - return addr_from_object(self.sock.family, space, w_address) + fd = intmask(self.sock.fd) + return addr_from_object(self.sock.family, fd, space, w_address) def bind_w(self, space, w_addr): """bind(address) diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -1,4 +1,4 @@ -import sys +import sys, os import py from pypy.tool.pytest.objspace import gettestobjspace from rpython.tool.udir import udir @@ -615,6 +615,28 @@ os.chdir(oldcwd) +class AppTestPacket: + def setup_class(cls): + if not hasattr(os, 'getuid') or os.getuid() != 0: + py.test.skip("AF_PACKET needs to be root for testing") + w_ok = space.appexec([], "(): import _socket; " + + "return hasattr(_socket, 'AF_PACKET')") + if not space.is_true(w_ok): + py.test.skip("no AF_PACKET on this platform") + cls.space = space + + def test_convert_between_tuple_and_sockaddr_ll(self): + import _socket + s = _socket.socket(_socket.AF_PACKET, _socket.SOCK_RAW) + assert s.getsockname() == ('', 0, 0, 0, '') + s.bind(('lo', 123)) + a, b, c, d, e = s.getsockname() + assert (a, b, c) == ('lo', 123, 0) + assert isinstance(d, int) + assert isinstance(e, str) + assert 0 <= len(e) <= 8 + + class AppTestSocketTCP: HOST = 'localhost' diff --git a/pypy/module/_ssl/__init__.py b/pypy/module/_ssl/__init__.py --- a/pypy/module/_ssl/__init__.py +++ b/pypy/module/_ssl/__init__.py @@ -51,6 +51,11 @@ super(Module, cls).buildloaders() + def setup_after_space_initialization(self): + """NOT_RPYTHON""" + from pypy.module._ssl.interp_ssl import PWINFO_STORAGE + PWINFO_STORAGE.clear() + def startup(self, space): from rpython.rlib.ropenssl import init_ssl init_ssl() diff --git a/pypy/module/_ssl/test/__init__.py b/pypy/module/_ssl/test/__init__.py new file mode 100644 diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -700,7 +700,8 @@ @rgc.must_be_light_finalizer def __del__(self): - lltype.free(self.buffer, flavor='raw') + if self.buffer: + lltype.free(self.buffer, flavor='raw') def setlen(self, size, zero=False, overallocate=True): if size > 0: diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -192,7 +192,7 @@ class ApiFunction: def __init__(self, argtypes, restype, callable, error=_NOT_SPECIFIED, - c_name=None): + c_name=None, gil=None): self.argtypes = argtypes self.restype = restype self.functype = lltype.Ptr(lltype.FuncType(argtypes, restype)) @@ -208,6 +208,7 @@ assert argnames[0] == 'space' self.argnames = argnames[1:] assert len(self.argnames) == len(self.argtypes) + self.gil = gil def _freeze_(self): return True @@ -223,14 +224,15 @@ def get_wrapper(self, space): wrapper = getattr(self, '_wrapper', None) if wrapper is None: - wrapper = make_wrapper(space, self.callable) + wrapper = make_wrapper(space, self.callable, self.gil) self._wrapper = wrapper wrapper.relax_sig_check = True if self.c_name is not None: wrapper.c_name = cpyext_namespace.uniquename(self.c_name) return wrapper -def cpython_api(argtypes, restype, error=_NOT_SPECIFIED, external=True): +def cpython_api(argtypes, restype, error=_NOT_SPECIFIED, external=True, + gil=None): """ Declares a function to be exported. - `argtypes`, `restype` are lltypes and describe the function signature. @@ -240,6 +242,8 @@ SytemError. - set `external` to False to get a C function pointer, but not exported by the API headers. + - set `gil` to "acquire", "release" or "around" to acquire the GIL, + release the GIL, or both """ if isinstance(restype, lltype.Typedef): real_restype = restype.OF @@ -262,7 +266,8 @@ c_name = None else: c_name = func_name - api_function = ApiFunction(argtypes, restype, func, error, c_name=c_name) + api_function = ApiFunction(argtypes, restype, func, error, + c_name=c_name, gil=gil) func.api_func = api_function if external: @@ -594,12 +599,15 @@ pypy_debug_catch_fatal_exception = rffi.llexternal('pypy_debug_catch_fatal_exception', [], lltype.Void) # Make the wrapper for the cases (1) and (2) -def make_wrapper(space, callable): +def make_wrapper(space, callable, gil=None): "NOT_RPYTHON" names = callable.api_func.argnames argtypes_enum_ui = unrolling_iterable(enumerate(zip(callable.api_func.argtypes, [name.startswith("w_") for name in names]))) fatal_value = callable.api_func.restype._defl() + gil_acquire = (gil == "acquire" or gil == "around") + gil_release = (gil == "release" or gil == "around") + assert gil is None or gil_acquire or gil_release @specialize.ll() def wrapper(*args): @@ -607,6 +615,10 @@ from pypy.module.cpyext.pyobject import Reference # we hope that malloc removal removes the newtuple() that is # inserted exactly here by the varargs specializer + if gil_acquire: + after = rffi.aroundstate.after + if after: + after() rffi.stackcounter.stacks_counter += 1 llop.gc_stack_bottom(lltype.Void) # marker for trackgcroot.py retval = fatal_value @@ -678,6 +690,10 @@ print str(e) pypy_debug_catch_fatal_exception() rffi.stackcounter.stacks_counter -= 1 + if gil_release: + before = rffi.aroundstate.before + if before: + before() return retval callable._always_inline_ = 'try' wrapper.__name__ = "wrapper for %r" % (callable, ) diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py --- a/pypy/module/cpyext/pystate.py +++ b/pypy/module/cpyext/pystate.py @@ -19,7 +19,7 @@ class NoThreads(Exception): pass - at cpython_api([], PyThreadState, error=CANNOT_FAIL) + at cpython_api([], PyThreadState, error=CANNOT_FAIL, gil="release") def PyEval_SaveThread(space): """Release the global interpreter lock (if it has been created and thread support is enabled) and reset the thread state to NULL, returning the @@ -29,19 +29,15 @@ state = space.fromcache(InterpreterState) tstate = state.swap_thread_state( space, lltype.nullptr(PyThreadState.TO)) - if rffi.aroundstate.before: - rffi.aroundstate.before() return tstate - at cpython_api([PyThreadState], lltype.Void) + at cpython_api([PyThreadState], lltype.Void, gil="acquire") def PyEval_RestoreThread(space, tstate): """Acquire the global interpreter lock (if it has been created and thread support is enabled) and set the thread state to tstate, which must not be NULL. If the lock has been created, the current thread must not have acquired it, otherwise deadlock ensues. (This function is available even when thread support is disabled at compile time.)""" - if rffi.aroundstate.after: - rffi.aroundstate.after() state = space.fromcache(InterpreterState) state.swap_thread_state(space, tstate) @@ -182,17 +178,14 @@ state = space.fromcache(InterpreterState) return state.swap_thread_state(space, tstate) - at cpython_api([PyThreadState], lltype.Void) + at cpython_api([PyThreadState], lltype.Void, gil="acquire") def PyEval_AcquireThread(space, tstate): """Acquire the global interpreter lock and set the current thread state to tstate, which should not be NULL. The lock must have been created earlier. If this thread already has the lock, deadlock ensues. This function is not available when thread support is disabled at compile time.""" - if rffi.aroundstate.after: - # After external call is before entering Python - rffi.aroundstate.after() - at cpython_api([PyThreadState], lltype.Void) + at cpython_api([PyThreadState], lltype.Void, gil="release") def PyEval_ReleaseThread(space, tstate): """Reset the current thread state to NULL and release the global interpreter lock. The lock must have been created earlier and must be held by the current @@ -200,28 +193,20 @@ that it represents the current thread state --- if it isn't, a fatal error is reported. This function is not available when thread support is disabled at compile time.""" - if rffi.aroundstate.before: - # Before external call is after running Python - rffi.aroundstate.before() PyGILState_STATE = rffi.INT - at cpython_api([], PyGILState_STATE, error=CANNOT_FAIL) + at cpython_api([], PyGILState_STATE, error=CANNOT_FAIL, gil="acquire") def PyGILState_Ensure(space): # XXX XXX XXX THIS IS A VERY MINIMAL IMPLEMENTATION THAT WILL HAPPILY # DEADLOCK IF CALLED TWICE ON THE SAME THREAD, OR CRASH IF CALLED IN A # NEW THREAD. We should very carefully follow what CPython does instead. - if rffi.aroundstate.after: - # After external call is before entering Python - rffi.aroundstate.after() return rffi.cast(PyGILState_STATE, 0) - at cpython_api([PyGILState_STATE], lltype.Void) + at cpython_api([PyGILState_STATE], lltype.Void, gil="release") def PyGILState_Release(space, state): # XXX XXX XXX We should very carefully follow what CPython does instead. - if rffi.aroundstate.before: - # Before external call is after running Python - rffi.aroundstate.before() + pass @cpython_api([], PyInterpreterState, error=CANNOT_FAIL) def PyInterpreterState_Head(space): @@ -236,7 +221,8 @@ """ return lltype.nullptr(PyInterpreterState.TO) - at cpython_api([PyInterpreterState], PyThreadState, error=CANNOT_FAIL) + at cpython_api([PyInterpreterState], PyThreadState, error=CANNOT_FAIL, + gil="around") def PyThreadState_New(space, interp): """Create a new thread state object belonging to the given interpreter object. The global interpreter lock need not be held, but may be held if @@ -245,12 +231,8 @@ raise NoThreads # PyThreadState_Get will allocate a new execution context, # we need to protect gc and other globals with the GIL. - rffi.aroundstate.after() - try: - rthread.gc_thread_start() - return PyThreadState_Get(space) - finally: - rffi.aroundstate.before() + rthread.gc_thread_start() + return PyThreadState_Get(space) @cpython_api([PyThreadState], lltype.Void) def PyThreadState_Clear(space, tstate): diff --git a/pypy/module/cpyext/test/test_translate.py b/pypy/module/cpyext/test/test_translate.py --- a/pypy/module/cpyext/test/test_translate.py +++ b/pypy/module/cpyext/test/test_translate.py @@ -11,7 +11,7 @@ FT = lltype.FuncType([], lltype.Signed) FTPTR = lltype.Ptr(FT) - def make_wrapper(space, func): + def make_wrapper(space, func, gil=None): def wrapper(): return func(space) return wrapper diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py --- a/pypy/module/imp/importing.py +++ b/pypy/module/imp/importing.py @@ -621,7 +621,10 @@ try: load_module(space, w_modulename, find_info, reuse=True) finally: - find_info.stream.close() + try: + find_info.stream.close() + except StreamErrors: + pass # fetch the module again, in case of "substitution" w_mod = check_sys_modules(space, w_modulename) return w_mod @@ -663,7 +666,10 @@ if find_info: stream = find_info.stream if stream: - stream.close() + try: + stream.close() + except StreamErrors: + pass if tentative: return None @@ -881,7 +887,10 @@ try: code_w = read_compiled_module(space, cpathname, stream.readall()) finally: - stream.close() + try: + stream.close() + except StreamErrors: + pass space.setattr(w_mod, w('__file__'), w(cpathname)) else: code_w = parse_source_module(space, pathname, source) @@ -966,7 +975,10 @@ return stream except StreamErrors: if stream: - stream.close() + try: + stream.close() + except StreamErrors: + pass return None # XXX! must not eat all exceptions, e.g. # Out of file descriptors. diff --git a/pypy/module/itertools/test/__init__.py b/pypy/module/itertools/test/__init__.py new file mode 100644 diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -1462,6 +1462,7 @@ imag = GetSetProperty(W_NDimArray.descr_get_imag, W_NDimArray.descr_set_imag), conj = interp2app(W_NDimArray.descr_conj), + conjugate = interp2app(W_NDimArray.descr_conj), argsort = interp2app(W_NDimArray.descr_argsort), sort = interp2app(W_NDimArray.descr_sort), diff --git a/pypy/module/micronumpy/test/test_complex.py b/pypy/module/micronumpy/test/test_complex.py --- a/pypy/module/micronumpy/test/test_complex.py +++ b/pypy/module/micronumpy/test/test_complex.py @@ -382,6 +382,7 @@ assert np.conjugate(1+2j) == 1-2j eye2 = np.array([[1, 0], [0, 1]]) + assert (eye2.conjugate() == eye2).all() x = eye2 + 1j * eye2 for a, b in zip(np.conjugate(x), np.array([[ 1.-1.j, 0.-0.j], [ 0.-0.j, 1.-1.j]])): assert a[0] == b[0] diff --git a/pypy/module/pwd/test/__init__.py b/pypy/module/pwd/test/__init__.py new file mode 100644 diff --git a/pypy/module/pyexpat/__init__.py b/pypy/module/pyexpat/__init__.py --- a/pypy/module/pyexpat/__init__.py +++ b/pypy/module/pyexpat/__init__.py @@ -39,8 +39,6 @@ 'error': 'space.fromcache(interp_pyexpat.Cache).w_error', '__version__': 'space.wrap("85819")', - 'EXPAT_VERSION': 'interp_pyexpat.get_expat_version(space)', - 'version_info': 'interp_pyexpat.get_expat_version_info(space)', } submodules = { @@ -53,3 +51,9 @@ 'XML_PARAM_ENTITY_PARSING_ALWAYS']: interpleveldefs[name] = 'space.wrap(interp_pyexpat.%s)' % (name,) + def startup(self, space): + from pypy.module.pyexpat import interp_pyexpat + w_ver = interp_pyexpat.get_expat_version(space) + space.setattr(self, space.wrap("EXPAT_VERSION"), w_ver) + w_ver = interp_pyexpat.get_expat_version_info(space) + space.setattr(self, space.wrap("version_info"), w_ver) diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py --- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py +++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py @@ -62,7 +62,7 @@ stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = pipe.communicate() - if getattr(pipe, 'returncode', 0) < 0: + if pipe.wait() < 0: raise IOError("subprocess was killed by signal %d" % ( pipe.returncode,)) if stderr.startswith('SKIP:'): diff --git a/pypy/module/select/test/__init__.py b/pypy/module/select/test/__init__.py new file mode 100644 diff --git a/pypy/module/struct/test/__init__.py b/pypy/module/struct/test/__init__.py new file mode 100644 diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py b/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py --- a/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_zdistutils.py @@ -165,7 +165,8 @@ assert lib.sin(12.3) == math.sin(12.3) v = ffi.verifier ext = v.get_extension() - assert 'distutils.extension.Extension' in str(ext.__class__) + assert 'distutils.extension.Extension' in str(ext.__class__) or \ + 'setuptools.extension.Extension' in str(ext.__class__) assert ext.sources == [maybe_relative_path(v.sourcefilename)] assert ext.name == v.get_module_name() assert ext.define_macros == [('TEST_EXTENSION_OBJECT', '1')] @@ -194,7 +195,8 @@ assert lib.test1eoes(7.0) == 42.0 v = ffi.verifier ext = v.get_extension() - assert 'distutils.extension.Extension' in str(ext.__class__) + assert 'distutils.extension.Extension' in str(ext.__class__) or \ + 'setuptools.extension.Extension' in str(ext.__class__) assert ext.sources == [maybe_relative_path(v.sourcefilename), extra_source] assert ext.name == v.get_module_name() diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py b/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py --- a/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_zintegration.py @@ -4,6 +4,9 @@ import subprocess from pypy.module.test_lib_pypy.cffi_tests.udir import udir +if sys.platform == 'win32': + py.test.skip('snippets do not run on win32') + def create_venv(name): tmpdir = udir.join(name) try: @@ -13,6 +16,23 @@ except OSError as e: py.test.skip("Cannot execute virtualenv: %s" % (e,)) + try: + deepcopy = os.symlink + except: + import shutil, errno + def deepcopy(src, dst): + try: + shutil.copytree(src, dst) + except OSError as e: + if e.errno in (errno.ENOTDIR, errno.EINVAL): + shutil.copy(src, dst) + else: + print('got errno') + print(e.errno) + print('not') + print(errno.ENOTDIR) + raise + site_packages = None for dirpath, dirnames, filenames in os.walk(str(tmpdir)): if os.path.basename(dirpath) == 'site-packages': @@ -32,7 +52,7 @@ modules += ('ply',) # needed for older versions of pycparser for module in modules: target = imp.find_module(module)[1] - os.symlink(target, os.path.join(site_packages, + deepcopy(target, os.path.join(site_packages, os.path.basename(target))) return tmpdir @@ -51,7 +71,11 @@ python_f.write(py.code.Source(python_snippet)) try: os.chdir(str(SNIPPET_DIR.join(dirname))) - vp = str(venv_dir.join('bin/python')) + if os.name == 'nt': + bindir = 'Scripts' + else: + bindir = 'bin' + vp = str(venv_dir.join(bindir).join('python')) subprocess.check_call((vp, 'setup.py', 'clean')) subprocess.check_call((vp, 'setup.py', 'install')) subprocess.check_call((vp, str(python_f))) diff --git a/pypy/module/zipimport/test/test_zipimport_deflated.py b/pypy/module/zipimport/test/test_zipimport_deflated.py --- a/pypy/module/zipimport/test/test_zipimport_deflated.py +++ b/pypy/module/zipimport/test/test_zipimport_deflated.py @@ -14,7 +14,7 @@ def setup_class(cls): try: import rpython.rlib.rzlib - except ImportError: + except CompilationError: py.test.skip("zlib not available, cannot test compressed zipfiles") cls.make_class() cls.w_BAD_ZIP = cls.space.wrap(BAD_ZIP) diff --git a/pypy/objspace/fake/checkmodule.py b/pypy/objspace/fake/checkmodule.py --- a/pypy/objspace/fake/checkmodule.py +++ b/pypy/objspace/fake/checkmodule.py @@ -10,6 +10,7 @@ mod = __import__('pypy.module.%s' % modname, None, None, ['__doc__']) # force computation and record what we wrap module = mod.Module(space, W_Root()) + module.setup_after_space_initialization() module.startup(space) for name in module.loaders: seeobj_w.append(module._load_lazily(space, name)) diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py --- a/pypy/objspace/std/test/test_typeobject.py +++ b/pypy/objspace/std/test/test_typeobject.py @@ -1165,3 +1165,17 @@ return x + 1 a = A() assert a.f(1) == 2 + + def test_eq_returns_notimplemented(self): + assert type.__eq__(int, 42) is NotImplemented + assert type.__ne__(dict, 42) is NotImplemented + assert type.__eq__(int, int) is True + assert type.__eq__(int, dict) is False + + def test_cmp_on_types(self): + class X(type): + def __cmp__(self, other): + return -1 + class Y: + __metaclass__ = X + assert (Y < Y) is True diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py --- a/pypy/objspace/std/typeobject.py +++ b/pypy/objspace/std/typeobject.py @@ -645,9 +645,13 @@ "type object '%N' has no attribute %R", self, w_name) def descr_eq(self, space, w_other): + if not isinstance(w_other, W_TypeObject): + return space.w_NotImplemented return space.is_(self, w_other) def descr_ne(self, space, w_other): + if not isinstance(w_other, W_TypeObject): + return space.w_NotImplemented return space.newbool(not space.is_w(self, w_other)) From noreply at buildbot.pypy.org Thu Mar 26 20:28:48 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 26 Mar 2015 20:28:48 +0100 (CET) Subject: [pypy-commit] pypy object-dtype2: custom gc trace translates and even gets called during gc.collect, but segfaults Message-ID: <20150326192848.0781F1C0484@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: object-dtype2 Changeset: r76581:b49651372062 Date: 2015-03-26 21:27 +0200 http://bitbucket.org/pypy/pypy/changeset/b49651372062/ Log: custom gc trace translates and even gets called during gc.collect, but segfaults diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py --- a/pypy/module/micronumpy/__init__.py +++ b/pypy/module/micronumpy/__init__.py @@ -30,6 +30,9 @@ for c in ['MAXDIMS', 'CLIP', 'WRAP', 'RAISE']: interpleveldefs[c] = 'space.wrap(constants.%s)' % c + def startup(self, space): + from pypy.module.micronumpy.concrete import _setup + _setup() class UMathModule(MixedModule): appleveldefs = {} diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py --- a/pypy/module/micronumpy/concrete.py +++ b/pypy/module/micronumpy/concrete.py @@ -353,7 +353,7 @@ for i in range(length): gcref = rffi.cast(llmemory.GCREF, storage) w_obj = cast_gcref_to_instance(W_Root, gcref) - print w_obj + print 'tracing', w_obj gc._trace_callback(callback, arg, storage) storage += step @@ -363,6 +363,15 @@ print 'registering custom trace for OBJECTSTORE' rgc.register_custom_trace_hook(OBJECTSTORE, lambda_customtrace) + at jit.dont_look_inside +def _create_objectstore(storage, length, elsize): + gcstruct = lltype.malloc(OBJECTSTORE) + # JIT does not support cast_ptr_to_adr + gcstruct.storage = llmemory.cast_ptr_to_adr(storage) + print 'create gcstruct',gcstruct,'with storage',storage,'as',gcstruct.storage + gcstruct.length = length + gcstruct.step = elsize + return gcstruct class ConcreteArrayNotOwning(BaseConcreteArray): @@ -414,12 +423,7 @@ length = support.product(shape) storage = dtype.itemtype.malloc(length * dtype.elsize, zero=zero) if dtype.num == NPY.OBJECT: - _setup() # make sure gc hook is registered - gcstruct = lltype.malloc(OBJECTSTORE) - gcstruct.storage = llmemory.cast_ptr_to_adr(storage) - print 'create gcstruct',gcstruct,'with storage',storage,'as',gcstruct.storage - gcstruct.length = length - gcstruct.step = dtype.elsize + gcstruct = _create_objectstore(storage, length, dtype.elsize) ConcreteArrayNotOwning.__init__(self, shape, dtype, order, strides, backstrides, storage) self.gcstruct = gcstruct diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -1378,6 +1378,7 @@ assert res[0] == "foobar" def test_keep_object_alive(self): + # XXX how can I run this test? import numpy as np import gc class O(object): @@ -1387,5 +1388,4 @@ a = np.array(fiveOs, dtype=object) del fiveOs gc.collect() - gc.collect() assert a[2].whatami() == 'an object' From noreply at buildbot.pypy.org Fri Mar 27 00:26:04 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Fri, 27 Mar 2015 00:26:04 +0100 (CET) Subject: [pypy-commit] pypy None-consistency: Close branch None-consistency. Message-ID: <20150326232604.462BB1C09A3@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: None-consistency Changeset: r76582:1441eb71f9d9 Date: 2015-03-26 23:26 +0000 http://bitbucket.org/pypy/pypy/changeset/1441eb71f9d9/ Log: Close branch None-consistency. From noreply at buildbot.pypy.org Fri Mar 27 01:40:22 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Fri, 27 Mar 2015 01:40:22 +0100 (CET) Subject: [pypy-commit] pypy default: Do perform_normalizations() at the end of annotation Message-ID: <20150327004022.358421C0352@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r76583:56931a93bcab Date: 2014-11-03 16:00 +0000 http://bitbucket.org/pypy/pypy/changeset/56931a93bcab/ Log: Do perform_normalizations() at the end of annotation diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py --- a/rpython/annotator/annrpython.py +++ b/rpython/annotator/annrpython.py @@ -12,6 +12,7 @@ from rpython.annotator import model as annmodel, signature from rpython.annotator.argument import simple_args from rpython.annotator.bookkeeper import Bookkeeper +from rpython.rtyper.normalizecalls import perform_normalizations import py log = py.log.Producer("annrpython") @@ -317,6 +318,8 @@ graphs[graph] = True for graph in graphs: simplify.eliminate_empty_blocks(graph) + if block_subset is None: + perform_normalizations(self) #___ flowing annotations in blocks _____________________ diff --git a/rpython/rtyper/normalizecalls.py b/rpython/rtyper/normalizecalls.py --- a/rpython/rtyper/normalizecalls.py +++ b/rpython/rtyper/normalizecalls.py @@ -404,6 +404,9 @@ # ____________________________________________________________ def perform_normalizations(annotator): + from rpython.rtyper.exceptiondata import standardexceptions + for cls in standardexceptions: + annotator.bookkeeper.getuniqueclassdef(cls) create_class_constructors(annotator) annotator.frozen += 1 try: diff --git a/rpython/rtyper/rtyper.py b/rpython/rtyper/rtyper.py --- a/rpython/rtyper/rtyper.py +++ b/rpython/rtyper/rtyper.py @@ -26,7 +26,6 @@ attachRuntimeTypeInfo, Primitive) from rpython.rtyper.rmodel import Repr, inputconst, BrokenReprTyperError from rpython.rtyper.typesystem import LowLevelTypeSystem, getfunctionptr -from rpython.rtyper.normalizecalls import perform_normalizations from rpython.rtyper import rclass from rpython.rtyper.rclass import RootClassRepr from rpython.tool.pairtype import pair @@ -169,22 +168,16 @@ def specialize(self, dont_simplify_again=False): """Main entry point: specialize all annotated blocks of the program.""" # specialize depends on annotator simplifications - assert dont_simplify_again in (False, True) # safety check if not dont_simplify_again: self.annotator.simplify() - - # first make sure that all functions called in a group have exactly - # the same signature, by hacking their flow graphs if needed - perform_normalizations(self.annotator) self.exceptiondata.finish(self) # new blocks can be created as a result of specialize_block(), so # we need to be careful about the loop here. self.already_seen = {} self.specialize_more_blocks() - if self.exceptiondata is not None: - self.exceptiondata.make_helpers(self) - self.specialize_more_blocks() # for the helpers just made + self.exceptiondata.make_helpers(self) + self.specialize_more_blocks() # for the helpers just made def getannmixlevel(self): if self.annmixlevel is not None: From noreply at buildbot.pypy.org Fri Mar 27 03:19:13 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Fri, 27 Mar 2015 03:19:13 +0100 (CET) Subject: [pypy-commit] pypy default: register builtin exceptions with the annotator at the same time as the other builtins Message-ID: <20150327021913.EE75F1C051C@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r76584:fe4080908368 Date: 2014-11-03 16:48 +0000 http://bitbucket.org/pypy/pypy/changeset/fe4080908368/ Log: register builtin exceptions with the annotator at the same time as the other builtins diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py --- a/rpython/annotator/bookkeeper.py +++ b/rpython/annotator/bookkeeper.py @@ -42,7 +42,7 @@ def __setstate__(self, dic): self.__dict__.update(dic) # normal action - delayed_imports() + self.register_builtins() def __init__(self, annotator): self.annotator = annotator @@ -67,7 +67,13 @@ self.needs_generic_instantiate = {} self.thread_local_fields = set() - delayed_imports() + self.register_builtins() + + def register_builtins(self): + import rpython.annotator.builtin # for side-effects + from rpython.annotator.exception import standardexceptions + for cls in standardexceptions: + self.getuniqueclassdef(cls) def enter(self, position_key): """Start of an operation. @@ -605,6 +611,3 @@ def immutablevalue(x): return getbookkeeper().immutablevalue(x) - -def delayed_imports(): - import rpython.annotator.builtin diff --git a/rpython/annotator/exception.py b/rpython/annotator/exception.py new file mode 100644 --- /dev/null +++ b/rpython/annotator/exception.py @@ -0,0 +1,7 @@ +from rpython.rlib import rstackovf + +# the exceptions that can be implicitely raised by some operations +standardexceptions = set([TypeError, OverflowError, ValueError, + ZeroDivisionError, MemoryError, IOError, OSError, StopIteration, KeyError, + IndexError, AssertionError, RuntimeError, UnicodeDecodeError, + UnicodeEncodeError, NotImplementedError, rstackovf._StackOverflow]) diff --git a/rpython/rtyper/exceptiondata.py b/rpython/rtyper/exceptiondata.py --- a/rpython/rtyper/exceptiondata.py +++ b/rpython/rtyper/exceptiondata.py @@ -1,15 +1,9 @@ from rpython.annotator import model as annmodel +from rpython.annotator.exception import standardexceptions from rpython.rtyper.llannotation import SomePtr -from rpython.rlib import rstackovf from rpython.rtyper.rclass import ( ll_issubclass, ll_type, ll_cast_to_object, getclassrepr, getinstancerepr) -# the exceptions that can be implicitely raised by some operations -standardexceptions = set([TypeError, OverflowError, ValueError, - ZeroDivisionError, MemoryError, IOError, OSError, StopIteration, KeyError, - IndexError, AssertionError, RuntimeError, UnicodeDecodeError, - UnicodeEncodeError, NotImplementedError, rstackovf._StackOverflow]) - class UnknownException(Exception): pass @@ -20,7 +14,6 @@ standardexceptions = standardexceptions def __init__(self, rtyper): - self.make_standard_exceptions(rtyper) # (NB. rclass identifies 'Exception' and 'object') r_type = rtyper.rootclass_repr r_instance = getinstancerepr(rtyper, None) @@ -32,11 +25,6 @@ self.lltype_of_exception_value = r_instance.lowleveltype self.rtyper = rtyper - def make_standard_exceptions(self, rtyper): - bk = rtyper.annotator.bookkeeper - for cls in self.standardexceptions: - bk.getuniqueclassdef(cls) - def finish(self, rtyper): bk = rtyper.annotator.bookkeeper for cls in self.standardexceptions: diff --git a/rpython/rtyper/normalizecalls.py b/rpython/rtyper/normalizecalls.py --- a/rpython/rtyper/normalizecalls.py +++ b/rpython/rtyper/normalizecalls.py @@ -404,9 +404,6 @@ # ____________________________________________________________ def perform_normalizations(annotator): - from rpython.rtyper.exceptiondata import standardexceptions - for cls in standardexceptions: - annotator.bookkeeper.getuniqueclassdef(cls) create_class_constructors(annotator) annotator.frozen += 1 try: From noreply at buildbot.pypy.org Fri Mar 27 03:45:48 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Fri, 27 Mar 2015 03:45:48 +0100 (CET) Subject: [pypy-commit] pypy default: Kill unused rtyper.typererrors Message-ID: <20150327024548.6156E1C1D95@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r76585:7e99a5ded672 Date: 2015-03-27 02:34 +0000 http://bitbucket.org/pypy/pypy/changeset/7e99a5ded672/ Log: Kill unused rtyper.typererrors diff --git a/rpython/rtyper/rtyper.py b/rpython/rtyper/rtyper.py --- a/rpython/rtyper/rtyper.py +++ b/rpython/rtyper/rtyper.py @@ -54,7 +54,6 @@ self.concrete_calltables = {} self.cache_dummy_values = {} self.lltype2vtable = {} - self.typererrors = [] self.typererror_count = 0 # make the primitive_to_repr constant mapping self.primitive_to_repr = {} @@ -233,9 +232,6 @@ # make sure all reprs so far have had their setup() called self.call_all_setups() - if self.typererrors: - self.dump_typererrors(to_log=True) - raise TyperError("there were %d error" % len(self.typererrors)) self.log.event('-=- specialized %d%s blocks -=-' % ( blockcount, newtext)) annmixlevel = self.annmixlevel @@ -243,29 +239,6 @@ if annmixlevel is not None: annmixlevel.finish() - def dump_typererrors(self, num=None, minimize=True, to_log=False): - c = 0 - bc = 0 - for err in self.typererrors[:num]: - c += 1 - if minimize and isinstance(err, BrokenReprTyperError): - bc += 1 - continue - graph, block, position = err.where - errmsg = ("TyperError-%d: %s\n" % (c, graph) + - str(err) + - "\n") - if to_log: - self.log.ERROR(errmsg) - else: - print errmsg - if bc: - minmsg = "(minimized %d errors away for this dump)" % (bc,) - if to_log: - self.log.ERROR(minmsg) - else: - print minmsg - def call_all_setups(self): # make sure all reprs so far have had their setup() called must_setup_more = [] From noreply at buildbot.pypy.org Fri Mar 27 03:45:49 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Fri, 27 Mar 2015 03:45:49 +0100 (CET) Subject: [pypy-commit] pypy default: Clean up rtyper.gottypererror() Message-ID: <20150327024549.ABB3C1C1D95@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r76586:d8062728670a Date: 2015-03-27 02:35 +0000 http://bitbucket.org/pypy/pypy/changeset/d8062728670a/ Log: Clean up rtyper.gottypererror() diff --git a/rpython/rtyper/rtyper.py b/rpython/rtyper/rtyper.py --- a/rpython/rtyper/rtyper.py +++ b/rpython/rtyper/rtyper.py @@ -290,9 +290,9 @@ # give the best possible types to the input args try: self.setup_block_entry(block) - except TyperError, e: - self.gottypererror(e, block, "block-entry", None) - return # cannot continue this block + except TyperError as e: + self.gottypererror(e, block, "block-entry") + raise # specialize all the operations, as far as possible @@ -307,9 +307,9 @@ try: hop.setup() # this is called from here to catch TyperErrors... self.translate_hl_to_ll(hop, varmapping) - except TyperError, e: - self.gottypererror(e, block, hop.spaceop, newops) - return # cannot continue this block: no op.result.concretetype + except TyperError as e: + self.gottypererror(e, block, hop.spaceop) + raise block.operations[:] = newops block.renamevariables(varmapping) @@ -398,9 +398,9 @@ continue # no conversion needed try: new_a1 = newops.convertvar(a1, r_a1, r_a2) - except TyperError, e: - self.gottypererror(e, block, link, newops) - continue # try other args + except TyperError as e: + self.gottypererror(e, block, link) + raise if new_a1 != a1: newlinkargs[i] = new_a1 @@ -482,14 +482,10 @@ "has no return value" % op.opname) op.result.concretetype = Void - def gottypererror(self, e, block, position, llops): - """Record a TyperError without crashing immediately. - Put a 'TyperError' operation in the graph instead. - """ + def gottypererror(self, exc, block, position): + """Record information about the location of a TyperError""" graph = self.annotator.annotated.get(block) - e.where = (graph, block, position) - self.typererror_count += 1 - raise + exc.where = (graph, block, position) # __________ regular operations __________ From noreply at buildbot.pypy.org Fri Mar 27 03:45:50 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Fri, 27 Mar 2015 03:45:50 +0100 (CET) Subject: [pypy-commit] pypy default: Kill unused rtyper.typererror_count Message-ID: <20150327024550.E0FFD1C1D95@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r76587:6f053786fbd8 Date: 2015-03-27 02:38 +0000 http://bitbucket.org/pypy/pypy/changeset/6f053786fbd8/ Log: Kill unused rtyper.typererror_count diff --git a/rpython/rtyper/rtyper.py b/rpython/rtyper/rtyper.py --- a/rpython/rtyper/rtyper.py +++ b/rpython/rtyper/rtyper.py @@ -54,7 +54,6 @@ self.concrete_calltables = {} self.cache_dummy_values = {} self.lltype2vtable = {} - self.typererror_count = 0 # make the primitive_to_repr constant mapping self.primitive_to_repr = {} self.isinstance_helpers = {} @@ -223,12 +222,8 @@ percentage = 100 * n // total if percentage >= previous_percentage + 5: previous_percentage = percentage - if self.typererror_count: - error_report = " but %d errors" % self.typererror_count - else: - error_report = '' - self.log.event('specializing: %d / %d blocks (%d%%)%s' % - (n, total, percentage, error_report)) + self.log.event('specializing: %d / %d blocks (%d%%)' % + (n, total, percentage)) # make sure all reprs so far have had their setup() called self.call_all_setups() From noreply at buildbot.pypy.org Fri Mar 27 04:17:07 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Fri, 27 Mar 2015 04:17:07 +0100 (CET) Subject: [pypy-commit] pypy default: Kill unused MixLevelAnnotatorPolicy.annhelper Message-ID: <20150327031707.AEC6E1C1401@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r76588:6b6dfcc1746d Date: 2015-03-27 03:17 +0000 http://bitbucket.org/pypy/pypy/changeset/6b6dfcc1746d/ Log: Kill unused MixLevelAnnotatorPolicy.annhelper diff --git a/rpython/rtyper/annlowlevel.py b/rpython/rtyper/annlowlevel.py --- a/rpython/rtyper/annlowlevel.py +++ b/rpython/rtyper/annlowlevel.py @@ -100,7 +100,6 @@ class MixLevelAnnotatorPolicy(LowLevelAnnotatorPolicy): def __init__(pol, annhelper): - pol.annhelper = annhelper pol.rtyper = annhelper.rtyper def default_specialize(pol, funcdesc, args_s): From noreply at buildbot.pypy.org Fri Mar 27 04:52:14 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Fri, 27 Mar 2015 04:52:14 +0100 (CET) Subject: [pypy-commit] pypy default: Kill unused attribute PyPyAnnotatorPolicy.single_space Message-ID: <20150327035214.539AF1C11B5@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r76589:11e0b763be9b Date: 2015-03-27 03:52 +0000 http://bitbucket.org/pypy/pypy/changeset/11e0b763be9b/ Log: Kill unused attribute PyPyAnnotatorPolicy.single_space diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -141,7 +141,7 @@ res = _pypy_execute_source(source) before = rffi.aroundstate.before if before: before() - return rffi.cast(rffi.INT, res) + return rffi.cast(rffi.INT, res) @entrypoint('main', [], c_name='pypy_init_threads') def pypy_init_threads(): @@ -312,7 +312,7 @@ w_dict = app.getwdict(space) entry_point, _ = create_entry_point(space, w_dict) - return entry_point, None, PyPyAnnotatorPolicy(single_space = space) + return entry_point, None, PyPyAnnotatorPolicy() def interface(self, ns): for name in ['take_options', 'handle_config', 'print_help', 'target', diff --git a/pypy/tool/ann_override.py b/pypy/tool/ann_override.py --- a/pypy/tool/ann_override.py +++ b/pypy/tool/ann_override.py @@ -13,11 +13,10 @@ class PyPyAnnotatorPolicy(AnnotatorPolicy): - def __init__(pol, single_space=None): + def __init__(pol): pol.lookups = {} pol.lookups_where = {} pol.pypytypes = {} - pol.single_space = single_space def specialize__wrap(pol, funcdesc, args_s): from pypy.interpreter.baseobjspace import W_Root From noreply at buildbot.pypy.org Fri Mar 27 04:54:13 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Fri, 27 Mar 2015 04:54:13 +0100 (CET) Subject: [pypy-commit] pypy default: cleanup Message-ID: <20150327035413.353E01C1CDE@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r76590:cbcb39fe4b74 Date: 2015-03-27 03:54 +0000 http://bitbucket.org/pypy/pypy/changeset/cbcb39fe4b74/ Log: cleanup diff --git a/pypy/tool/ann_override.py b/pypy/tool/ann_override.py --- a/pypy/tool/ann_override.py +++ b/pypy/tool/ann_override.py @@ -13,12 +13,12 @@ class PyPyAnnotatorPolicy(AnnotatorPolicy): - def __init__(pol): - pol.lookups = {} - pol.lookups_where = {} - pol.pypytypes = {} + def __init__(self): + self.lookups = {} + self.lookups_where = {} + self.pypytypes = {} - def specialize__wrap(pol, funcdesc, args_s): + def specialize__wrap(self, funcdesc, args_s): from pypy.interpreter.baseobjspace import W_Root from rpython.annotator.classdef import ClassDef W_Root_def = funcdesc.bookkeeper.getuniqueclassdef(W_Root) @@ -50,102 +50,102 @@ typ = (None, str) return funcdesc.cachedgraph(typ) - def _remember_immutable(pol, t, cached): + def _remember_immutable(self, t, cached): # for jit benefit if cached not in t._immutable_fields_: # accessed this way just # for convenience t._immutable_fields_.append(cached) - def attach_lookup(pol, t, attr): + def attach_lookup(self, t, attr): cached = "cached_%s" % attr if not t.is_heaptype() and not t.is_cpytype(): - pol._remember_immutable(t, cached) + self._remember_immutable(t, cached) setattr(t, cached, t._lookup(attr)) return True return False - def attach_lookup_in_type_where(pol, t, attr): + def attach_lookup_in_type_where(self, t, attr): cached = "cached_where_%s" % attr if not t.is_heaptype() and not t.is_cpytype(): - pol._remember_immutable(t, cached) + self._remember_immutable(t, cached) setattr(t, cached, t._lookup_where(attr)) return True return False - def consider_lookup(pol, bookkeeper, attr): + def consider_lookup(self, bookkeeper, attr): from rpython.annotator.classdef import InstanceSource - assert attr not in pol.lookups + assert attr not in self.lookups from pypy.objspace.std import typeobject cached = "cached_%s" % attr clsdef = bookkeeper.getuniqueclassdef(typeobject.W_TypeObject) classdesc = clsdef.classdesc classdesc.classdict[cached] = Constant(None) clsdef.add_source_for_attribute(cached, classdesc) - for t in pol.pypytypes: - if pol.attach_lookup(t, attr): + for t in self.pypytypes: + if self.attach_lookup(t, attr): source = InstanceSource(bookkeeper, t) clsdef.add_source_for_attribute(cached, source) - pol.lookups[attr] = True + self.lookups[attr] = True - def consider_lookup_in_type_where(pol, bookkeeper, attr): + def consider_lookup_in_type_where(self, bookkeeper, attr): from rpython.annotator.classdef import InstanceSource - assert attr not in pol.lookups_where + assert attr not in self.lookups_where from pypy.objspace.std import typeobject cached = "cached_where_%s" % attr clsdef = bookkeeper.getuniqueclassdef(typeobject.W_TypeObject) classdesc = clsdef.classdesc classdesc.classdict[cached] = Constant((None, None)) clsdef.add_source_for_attribute(cached, classdesc) - for t in pol.pypytypes: - if pol.attach_lookup_in_type_where(t, attr): + for t in self.pypytypes: + if self.attach_lookup_in_type_where(t, attr): source = InstanceSource(bookkeeper, t) clsdef.add_source_for_attribute(cached, source) - pol.lookups_where[attr] = True + self.lookups_where[attr] = True - def specialize__lookup(pol, funcdesc, args_s): + def specialize__lookup(self, funcdesc, args_s): s_space, s_obj, s_name = args_s if s_name.is_constant(): attr = s_name.const def builder(translator, func): #print "LOOKUP", attr - pol.consider_lookup(funcdesc.bookkeeper, attr) + self.consider_lookup(funcdesc.bookkeeper, attr) d = {'__name__': ''} exec CACHED_LOOKUP % {'attr': attr} in d return translator.buildflowgraph(d['lookup_'+attr]) return funcdesc.cachedgraph(attr, builder=builder) else: - pol.lookups[None] = True + self.lookups[None] = True return funcdesc.cachedgraph(None) # don't specialize - def specialize__lookup_in_type_where(pol, funcdesc, args_s): + def specialize__lookup_in_type_where(self, funcdesc, args_s): s_space, s_obj, s_name = args_s if s_name.is_constant(): attr = s_name.const def builder(translator, func): #print "LOOKUP_IN_TYPE_WHERE", attr - pol.consider_lookup_in_type_where(funcdesc.bookkeeper, attr) + self.consider_lookup_in_type_where(funcdesc.bookkeeper, attr) d = {'__name__': ''} exec CACHED_LOOKUP_IN_TYPE_WHERE % {'attr': attr} in d return translator.buildflowgraph(d['lookup_in_type_where_'+attr]) return funcdesc.cachedgraph(attr, builder=builder) else: - pol.lookups_where[None] = True + self.lookups_where[None] = True return funcdesc.cachedgraph(None) - def event(pol, bookkeeper, what, x): + def event(self, bookkeeper, what, x): from pypy.objspace.std import typeobject if isinstance(x, typeobject.W_TypeObject): from rpython.annotator.classdef import InstanceSource clsdef = bookkeeper.getuniqueclassdef(typeobject.W_TypeObject) - pol.pypytypes[x] = True + self.pypytypes[x] = True #print "TYPE", x - for attr in pol.lookups: - if attr and pol.attach_lookup(x, attr): + for attr in self.lookups: + if attr and self.attach_lookup(x, attr): cached = "cached_%s" % attr source = InstanceSource(bookkeeper, x) clsdef.add_source_for_attribute(cached, source) - for attr in pol.lookups_where: - if attr and pol.attach_lookup_in_type_where(x, attr): + for attr in self.lookups_where: + if attr and self.attach_lookup_in_type_where(x, attr): cached = "cached_where_%s" % attr source = InstanceSource(bookkeeper, x) clsdef.add_source_for_attribute(cached, source) From noreply at buildbot.pypy.org Fri Mar 27 09:08:37 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 27 Mar 2015 09:08:37 +0100 (CET) Subject: [pypy-commit] pypy.org extradoc: update the values Message-ID: <20150327080837.42DEA1C1156@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r585:ea2c947a8e6f Date: 2015-03-27 09:09 +0100 http://bitbucket.org/pypy/pypy.org/changeset/ea2c947a8e6f/ Log: update the values diff --git a/don1.html b/don1.html --- a/don1.html +++ b/don1.html @@ -9,13 +9,13 @@ - $59002 of $105000 (56.2%) + $59097 of $105000 (56.3%)
    diff --git a/don3.html b/don3.html --- a/don3.html +++ b/don3.html @@ -15,7 +15,7 @@ - $51350 of $60000 (85.6%) + $51369 of $60000 (85.6%)
    diff --git a/don4.html b/don4.html --- a/don4.html +++ b/don4.html @@ -9,7 +9,7 @@ @@ -17,7 +17,7 @@ 2nd call: - $22593 of $80000 (28.2%) + $22603 of $80000 (28.3%)
    From noreply at buildbot.pypy.org Fri Mar 27 10:10:31 2015 From: noreply at buildbot.pypy.org (groggi) Date: Fri, 27 Mar 2015 10:10:31 +0100 (CET) Subject: [pypy-commit] pypy gc-incminimark-pinning-improve: Merge release-2.5.x into branch Message-ID: <20150327091031.20F861C0316@cobra.cs.uni-duesseldorf.de> Author: Gregor Wegberg Branch: gc-incminimark-pinning-improve Changeset: r76591:db8331fd62ee Date: 2015-03-27 09:54 +0100 http://bitbucket.org/pypy/pypy/changeset/db8331fd62ee/ Log: Merge release-2.5.x into branch diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -67,7 +67,7 @@ # The short X.Y version. version = '2.5' # The full version, including alpha/beta/rc tags. -release = '2.5.0' +release = '2.5.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-2.5.1.rst release-2.5.0.rst release-2.4.0.rst release-2.3.1.rst diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -7,6 +7,7 @@ .. toctree:: whatsnew-head.rst + whatsnew-2.5.1.rst whatsnew-2.5.0.rst whatsnew-2.4.0.rst whatsnew-2.3.1.rst diff --git a/pypy/doc/release-2.5.1.rst b/pypy/doc/release-2.5.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-2.5.1.rst @@ -0,0 +1,115 @@ +================================ +PyPy 2.5.1 - Pineapple Bromeliad +================================ + +We're pleased to announce PyPy 2.5.1, Pineapple `Bromeliad`_ following on the heels of 2.5.0 + +You can download the PyPy 2.5.1 release here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project, and for those who donate to our three sub-projects, as well as our +volunteers and contributors. +We've shown quite a bit of progress, but we're slowly running out of funds. +Please consider donating more, or even better convince your employer to donate, +so we can finish those projects! The three sub-projects are: + +* `Py3k`_ (supporting Python 3.x): We have released a Python 3.2.5 compatible version + we call PyPy3 2.4.0, and are working toward a Python 3.3 compatible version + +* `STM`_ (software transactional memory): We have released a first working version, + and continue to try out new promising paths of achieving a fast multithreaded Python + +* `NumPy`_ which requires installation of our fork of upstream numpy, + available `on bitbucket`_ + +.. _`Bromeliad`: http://xkcd.com/1498 +.. _`Py3k`: http://pypy.org/py3donate.html +.. _`STM`: http://pypy.org/tmdonate2.html +.. _`NumPy`: http://pypy.org/numpydonate.html +.. _`on bitbucket`: https://www.bitbucket.org/pypy/numpy + +We would also like to encourage new people to join the project. PyPy has many +layers and we need help with all of them: `PyPy`_ and `Rpython`_ documentation +improvements, tweaking popular `modules`_ to run on pypy, or general `help`_ with making +Rpython's JIT even better. + +.. _`PyPy`: http://doc.pypy.org +.. _`Rpython`: http://rpython.readthedocs.org +.. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly +.. _`help`: http://doc.pypy.org/en/latest/project-ideas.html + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7. It's fast (`pypy and cpython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +This release supports **x86** machines on most common operating systems +(Linux 32/64, Mac OS X 64, Windows, and OpenBSD), +as well as newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux. + +While we support 32 bit python on Windows, work on the native Windows 64 +bit python is still stalling, we would welcome a volunteer +to `handle that`_. + +.. _`pypy and cpython 2.7.x`: http://speed.pypy.org +.. _`handle that`: http://doc.pypy.org/en/latest/windows.html#what-is-missing-for-a-full-64-bit-translation + +Highlights +========== + +* The past months have seen pypy mature and grow, as rpython becomes the goto + solution for writing fast dynamic language interpreters. Our separation of + Rpython and the python interpreter PyPy is now much clearer in the + `PyPy documentation`_ and we now have seperate `RPython documentation`_. + Tell us what still isn't clear, or even better help us improve the documentation. + +* We merged version 2.7.9 of python's stdlib. From the python release notice: + + * The entirety of Python 3.4's `ssl module`_ has been backported. + See `PEP 466`_ for justification. + + * HTTPS certificate validation using the system's certificate store is now + enabled by default. See `PEP 476`_ for details. + + * SSLv3 has been disabled by default in httplib and its reverse dependencies + due to the `POODLE attack`_. + + * The `ensurepip module`_ has been backported, which provides the pip + package manager in every Python 2.7 installation. See `PEP 477`_. + +* The garbage collector now ignores parts of the stack which did not change + since the last collection, another performance boost + +* errno and LastError are saved around cffi calls so things like pdb will not + overwrite it + +* We continue to asymptotically approach a score of 7 times faster than cpython + on our benchmark suite, we now rank 6.98 on latest runs + +* Issues reported with our previous release were resolved_ after reports from users on + our issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at + #pypy. + +.. _`PyPy documentation`: http://doc.pypy.org +.. _`RPython documentation`: http://rpython.readthedocs.org +.. _`ssl module`: https://docs.python.org/3/library/ssl.html +.. _`PEP 466`: https://www.python.org/dev/peps/pep-0466 +.. _`PEP 476`: https://www.python.org/dev/peps/pep-0476 +.. _`PEP 477`: https://www.python.org/dev/peps/pep-0477 +.. _`POODLE attack`: https://www.imperialviolet.org/2014/10/14/poodle.html +.. _`ensurepip module`: https://docs.python.org/2/library/ensurepip.html +.. _resolved: http://doc.pypy.org/en/latest/whatsnew-2.5.1.html + +Please try it out and let us know what you think. We welcome +success stories, `experiments`_, or `benchmarks`_, we know you are using PyPy, please tell us about it! + +Cheers + +The PyPy Team + +.. _`experiments`: http://morepypy.blogspot.com/2015/02/experiments-in-pyrlang-with-rpython.html +.. _`benchmarks`: https://mithrandi.net/blog/2015/03/axiom-benchmark-results-on-pypy-2-5-0 diff --git a/pypy/doc/whatsnew-2.5.0.rst b/pypy/doc/whatsnew-2.5.0.rst --- a/pypy/doc/whatsnew-2.5.0.rst +++ b/pypy/doc/whatsnew-2.5.0.rst @@ -1,6 +1,6 @@ -======================= -What's new in PyPy 2.5 -======================= +======================== +What's new in PyPy 2.5.0 +======================== .. this is a revision shortly after release-2.4.x .. startrev: 7026746cbb1b diff --git a/pypy/doc/whatsnew-2.5.1.rst b/pypy/doc/whatsnew-2.5.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-2.5.1.rst @@ -0,0 +1,40 @@ +======================== +What's new in PyPy 2.5.1 +======================== + +.. this is a revision shortly after release-2.5.0 +.. startrev: 397b96217b85 + + +Non-blocking file reads sometimes raised EAGAIN even though they +had buffered data waiting, fixed in b1c4fcb04a42 + +Fix a bug in cpyext in multithreded programs acquiring/releasing the GIL + +.. branch: vmprof + +.. branch: stackroot-speedup-2 +Avoid tracing all stack roots during repeated minor collections, +by ignoring the part of the stack that didn't change + +.. branch: stdlib-2.7.9 +Update stdlib to version 2.7.9 + +.. branch: fix-kqueue-error2 +Fix exception being raised by kqueue.control (CPython compatibility) + +.. branch: gitignore + +.. branch: framestate2 +Refactor rpython.flowspace.framestate.FrameState. + +.. branch: alt_errno +Add an alternative location to save LastError, errno around ctypes, +cffi external calls so things like pdb will not overwrite it + +.. branch: nonquadratic-heapcache +Speed up the warmup times of the JIT by removing a quadratic algorithm in the +heapcache. + +.. branch: online-transforms-2 +Simplify flow graphs on the fly during annotation phase. diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -2,38 +2,6 @@ What's new in PyPy 2.5+ ======================= -.. this is a revision shortly after release-2.5.x +.. this is a revision shortly after release-2.5.1 .. startrev: 397b96217b85 - -Non-blocking file reads sometimes raised EAGAIN even though they -had buffered data waiting, fixed in b1c4fcb04a42 - - -.. branch: vmprof - -.. branch: stackroot-speedup-2 -Avoid tracing all stack roots during repeated minor collections, -by ignoring the part of the stack that didn't change - -.. branch: stdlib-2.7.9 -Update stdlib to version 2.7.9 - -.. branch: fix-kqueue-error2 -Fix exception being raised by kqueue.control (CPython compatibility) - -.. branch: gitignore - -.. branch: framestate2 -Refactor rpython.flowspace.framestate.FrameState. - -.. branch: alt_errno -Add an alternative location to save LastError, errno around ctypes, -cffi external calls so things like pdb will not overwrite it - -.. branch: nonquadratic-heapcache -Speed up the warmup times of the JIT by removing a quadratic algorithm in the -heapcache. - -.. branch: online-transforms-2 -Simplify flow graphs on the fly during annotation phase. diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -29,7 +29,7 @@ #define PY_VERSION "2.7.9" /* PyPy version as a string */ -#define PYPY_VERSION "2.6.0-alpha0" +#define PYPY_VERSION "2.5.1" /* Subversion Revision number of this file (not of the repository). * Empty since Mercurial migration. */ diff --git a/pypy/module/pyexpat/__init__.py b/pypy/module/pyexpat/__init__.py --- a/pypy/module/pyexpat/__init__.py +++ b/pypy/module/pyexpat/__init__.py @@ -39,8 +39,6 @@ 'error': 'space.fromcache(interp_pyexpat.Cache).w_error', '__version__': 'space.wrap("85819")', - 'EXPAT_VERSION': 'interp_pyexpat.get_expat_version(space)', - 'version_info': 'interp_pyexpat.get_expat_version_info(space)', } submodules = { @@ -53,3 +51,9 @@ 'XML_PARAM_ENTITY_PARSING_ALWAYS']: interpleveldefs[name] = 'space.wrap(interp_pyexpat.%s)' % (name,) + def startup(self, space): + from pypy.module.pyexpat import interp_pyexpat + w_ver = interp_pyexpat.get_expat_version(space) + space.setattr(self, space.wrap("EXPAT_VERSION"), w_ver) + w_ver = interp_pyexpat.get_expat_version_info(space) + space.setattr(self, space.wrap("version_info"), w_ver) diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -10,7 +10,7 @@ #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py CPYTHON_API_VERSION = 1013 #XXX # sync with include/modsupport.h -PYPY_VERSION = (2, 6, 0, "alpha", 0) #XXX # sync patchlevel.h +PYPY_VERSION = (2, 5, 1, "final", 0) #XXX # sync patchlevel.h if platform.name == 'msvc': COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600) diff --git a/rpython/doc/rtyper.rst b/rpython/doc/rtyper.rst --- a/rpython/doc/rtyper.rst +++ b/rpython/doc/rtyper.rst @@ -118,8 +118,7 @@ given this representation. The RTyper also computes a ``concretetype`` for Constants, to match the way they are used in the low-level operations (for example, ``int_add(x, 1)`` requires a ``Constant(1)`` with -``concretetype=Signed``, but an untyped ``add(x, 1)`` works with a -``Constant(1)`` that must actually be a PyObject at run-time). +``concretetype=Signed``). In addition to ``lowleveltype``, each Repr subclass provides a set of methods called ``rtype_op_xxx()`` which define how each high-level operation ``op_xxx`` @@ -306,14 +305,14 @@ ~~~~~~~~~~~~~ As in C, pointers provide the indirection needed to make a reference modifiable -or sharable. Pointers can only point to a structure, an array, a function -(see below) or a PyObject (see below). Pointers to primitive types, if needed, -must be done by pointing to a structure with a single field of the required -type. Pointer types are declared by:: +or sharable. Pointers can only point to a structure, an array or a function +(see below). Pointers to primitive types, if needed, must be done by pointing +to a structure with a single field of the required type. Pointer types are +declared by:: Ptr(TYPE) -At run-time, pointers to GC structures (GcStruct, GcArray and PyObject) hold a +At run-time, pointers to GC structures (GcStruct, GcArray) hold a reference to what they are pointing to. Pointers to non-GC structures that can go away when their container is deallocated (Struct, Array) must be handled with care: the bigger structure of which they are part of could be freed while @@ -356,22 +355,6 @@ :graph: the flow graph of the function. -The PyObject Type -~~~~~~~~~~~~~~~~~ - -This is a special type, for compatibility with CPython: it stands for a -structure compatible with PyObject. This is also a "container" type (thinking -about C, this is ``PyObject``, not ``PyObject*``), so it is usually manipulated -via a Ptr. A typed graph can still contain generic space operations (add, -getitem, etc.) provided they are applied on objects whose low-level type is -``Ptr(PyObject)``. In fact, code generators that support this should consider -that the default type of a variable, if none is specified, is ``Ptr(PyObject)``. -In this way, they can generate the correct code for fully-untyped flow graphs. - -The testing implementation allows you to "create" PyObjects by calling -``pyobjectptr(obj)``. - - Opaque Types ~~~~~~~~~~~~ diff --git a/rpython/jit/metainterp/jitprof.py b/rpython/jit/metainterp/jitprof.py --- a/rpython/jit/metainterp/jitprof.py +++ b/rpython/jit/metainterp/jitprof.py @@ -44,7 +44,10 @@ pass def get_counter(self, num): - return -1.0 + return 0 + + def get_times(self, num): + return 0.0 class Profiler(BaseProfiler): initialized = False @@ -109,6 +112,9 @@ return self.cpu.tracker.total_freed_bridges return self.counters[num] + def get_times(self, num): + return self.times[num] + def count_ops(self, opnum, kind=Counters.OPS): from rpython.jit.metainterp.resoperation import rop self.counters[kind] += 1 diff --git a/rpython/jit/metainterp/test/test_jitiface.py b/rpython/jit/metainterp/test/test_jitiface.py --- a/rpython/jit/metainterp/test/test_jitiface.py +++ b/rpython/jit/metainterp/test/test_jitiface.py @@ -5,7 +5,7 @@ from rpython.jit.codewriter.policy import JitPolicy from rpython.jit.metainterp.resoperation import rop from rpython.rtyper.annlowlevel import hlstr -from rpython.jit.metainterp.jitprof import Profiler +from rpython.jit.metainterp.jitprof import Profiler, EmptyProfiler class JitHookInterfaceTests(object): @@ -178,6 +178,20 @@ self.meta_interp(main, [], ProfilerClass=Profiler) + def test_get_stats_empty(self): + driver = JitDriver(greens = [], reds = ['i']) + def loop(i): + while i > 0: + driver.jit_merge_point(i=i) + i -= 1 + def main(): + loop(30) + assert jit_hooks.stats_get_counter_value(None, + Counters.TOTAL_COMPILED_LOOPS) == 0 + assert jit_hooks.stats_get_times_value(None, Counters.TRACING) == 0 + self.meta_interp(main, [], ProfilerClass=EmptyProfiler) + + class LLJitHookInterfaceTests(JitHookInterfaceTests): # use this for any backend, instead of the super class diff --git a/rpython/memory/gctransform/asmgcroot.py b/rpython/memory/gctransform/asmgcroot.py --- a/rpython/memory/gctransform/asmgcroot.py +++ b/rpython/memory/gctransform/asmgcroot.py @@ -368,6 +368,13 @@ if rpy_fastgil != 1: ll_assert(rpy_fastgil != 0, "walk_stack_from doesn't have the GIL") initialframedata = rffi.cast(llmemory.Address, rpy_fastgil) + # + # very rare issue: initialframedata.address[0] is uninitialized + # in this case, but "retaddr = callee.frame_address.address[0]" + # reads it. If it happens to be exactly a valid return address + # inside the C code, then bad things occur. + initialframedata.address[0] = llmemory.NULL + # self.walk_frames(curframe, otherframe, initialframedata) stackscount += 1 # @@ -519,17 +526,15 @@ from rpython.jit.backend.llsupport.jitframe import STACK_DEPTH_OFS tid = self.gc.get_possibly_forwarded_type_id(ebp_in_caller) - ll_assert(rffi.cast(lltype.Signed, tid) == - rffi.cast(lltype.Signed, self.frame_tid), - "found a stack frame that does not belong " - "anywhere I know, bug in asmgcc") - # fish the depth - extra_stack_depth = (ebp_in_caller + STACK_DEPTH_OFS).signed[0] - ll_assert((extra_stack_depth & (rffi.sizeof(lltype.Signed) - 1)) - == 0, "asmgcc: misaligned extra_stack_depth") - extra_stack_depth //= rffi.sizeof(lltype.Signed) - self._shape_decompressor.setjitframe(extra_stack_depth) - return + if (rffi.cast(lltype.Signed, tid) == + rffi.cast(lltype.Signed, self.frame_tid)): + # fish the depth + extra_stack_depth = (ebp_in_caller + STACK_DEPTH_OFS).signed[0] + ll_assert((extra_stack_depth & (rffi.sizeof(lltype.Signed) - 1)) + == 0, "asmgcc: misaligned extra_stack_depth") + extra_stack_depth //= rffi.sizeof(lltype.Signed) + self._shape_decompressor.setjitframe(extra_stack_depth) + return llop.debug_fatalerror(lltype.Void, "cannot find gc roots!") def getlocation(self, callee, ebp_in_caller, location): diff --git a/rpython/rlib/jit_hooks.py b/rpython/rlib/jit_hooks.py --- a/rpython/rlib/jit_hooks.py +++ b/rpython/rlib/jit_hooks.py @@ -130,7 +130,7 @@ @register_helper(annmodel.SomeFloat()) def stats_get_times_value(warmrunnerdesc, no): - return warmrunnerdesc.metainterp_sd.profiler.times[no] + return warmrunnerdesc.metainterp_sd.profiler.get_times(no) LOOP_RUN_CONTAINER = lltype.GcArray(lltype.Struct('elem', ('type', lltype.Char), diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py --- a/rpython/translator/simplify.py +++ b/rpython/translator/simplify.py @@ -334,25 +334,25 @@ flowcontext.py). """ for block in list(graph.iterblocks()): - for i in range(len(block.exits)-1, -1, -1): - exit = block.exits[i] - if not (exit.target is graph.exceptblock and - exit.args[0] == Constant(AssertionError)): - continue - # can we remove this exit without breaking the graph? - if len(block.exits) < 2: + for i in range(len(block.exits)-1, -1, -1): + exit = block.exits[i] + if not (exit.target is graph.exceptblock and + exit.args[0] == Constant(AssertionError)): + continue + # can we remove this exit without breaking the graph? + if len(block.exits) < 2: + break + if block.canraise: + if exit.exitcase is None: break - if block.canraise: - if exit.exitcase is None: - break - if len(block.exits) == 2: - # removing the last non-exceptional exit - block.exitswitch = None - exit.exitcase = None - # remove this exit - lst = list(block.exits) - del lst[i] - block.recloseblock(*lst) + if len(block.exits) == 2: + # removing the last non-exceptional exit + block.exitswitch = None + exit.exitcase = None + # remove this exit + lst = list(block.exits) + del lst[i] + block.recloseblock(*lst) # _____________________________________________________________________ From noreply at buildbot.pypy.org Fri Mar 27 12:39:18 2015 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Fri, 27 Mar 2015 12:39:18 +0100 (CET) Subject: [pypy-commit] pypy default: Fixed #2013 -- added constants to _ssl for TLS 1.1 and 1.2 Message-ID: <20150327113918.87F161C028C@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r76592:22eb203956f1 Date: 2015-03-27 07:39 -0400 http://bitbucket.org/pypy/pypy/changeset/22eb203956f1/ Log: Fixed #2013 -- added constants to _ssl for TLS 1.1 and 1.2 diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -33,7 +33,8 @@ PY_SSL_CLIENT, PY_SSL_SERVER = 0, 1 (PY_SSL_VERSION_SSL2, PY_SSL_VERSION_SSL3, - PY_SSL_VERSION_SSL23, PY_SSL_VERSION_TLS1) = range(4) + PY_SSL_VERSION_SSL23, PY_SSL_VERSION_TLS1, PY_SSL_VERSION_TLS1_1, + PY_SSL_VERSION_TLS1_2) = range(6) SOCKET_IS_NONBLOCKING, SOCKET_IS_BLOCKING = 0, 1 SOCKET_HAS_TIMED_OUT, SOCKET_HAS_BEEN_CLOSED = 2, 3 @@ -72,6 +73,9 @@ constants["PROTOCOL_SSLv3"] = PY_SSL_VERSION_SSL3 constants["PROTOCOL_SSLv23"] = PY_SSL_VERSION_SSL23 constants["PROTOCOL_TLSv1"] = PY_SSL_VERSION_TLS1 +if HAVE_TLSv1_2: + constants["PROTOCOL_TLSv1_1"] = PY_SSL_VERSION_TLS1_1 + constants["PROTOCOL_TLSv1_2"] = PY_SSL_VERSION_TLS1_2 constants["OP_ALL"] = SSL_OP_ALL &~SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS constants["OP_NO_SSLv2"] = SSL_OP_NO_SSLv2 @@ -140,7 +144,7 @@ def __del__(self): rffi.free_nonmovingbuffer( - self.protos, self.buf, self.pinned, self.is_raw) + self.protos, self.buf, self.pinned, self.is_raw) @staticmethod def advertiseNPN_cb(s, data_ptr, len_ptr, args): @@ -162,7 +166,7 @@ client_len = len(npn.protos) else: client = lltype.nullptr(rffi.CCHARP.TO) - client_len = 0 + client_len = 0 libssl_SSL_select_next_proto(out_ptr, outlen_ptr, server, server_len, @@ -593,14 +597,14 @@ CB_MAXLEN = 128 with lltype.scoped_alloc(rffi.CCHARP.TO, CB_MAXLEN) as buf: - if (libssl_SSL_session_reused(self.ssl) ^ + if (libssl_SSL_session_reused(self.ssl) ^ (self.socket_type == PY_SSL_CLIENT)): # if session is resumed XOR we are the client length = libssl_SSL_get_finished(self.ssl, buf, CB_MAXLEN) else: # if a new session XOR we are the server length = libssl_SSL_get_peer_finished(self.ssl, buf, CB_MAXLEN) - + if length > 0: return space.wrap(rffi.charpsize2str(buf, intmask(length))) @@ -1107,7 +1111,7 @@ except OperationError as e: if not e.match(space, space.w_TypeError): raise - raise oefmt(space.w_TypeError, + raise oefmt(space.w_TypeError, "password callback must return a string") except OperationError as e: pw_info.operationerror = e @@ -1196,6 +1200,10 @@ method = libssl_SSLv2_method() elif protocol == PY_SSL_VERSION_SSL23: method = libssl_SSLv23_method() + elif protocol == PY_SSL_VERSION_TLS1_1 and HAVE_TLSv1_2: + method = libssl_TLSv1_1_method() + elif protocol == PY_SSL_VERSION_TLS1_2 and HAVE_TLSv1_2: + method = libssl_TLSv1_2_method() else: raise oefmt(space.w_ValueError, "invalid protocol version") ctx = libssl_SSL_CTX_new(method) @@ -1348,7 +1356,7 @@ except OperationError as e: if not e.match(space, space.w_TypeError): raise - raise oefmt(space.w_TypeError, + raise oefmt(space.w_TypeError, "password should be a string or callable") libssl_SSL_CTX_set_default_passwd_cb( @@ -1452,7 +1460,7 @@ if cadata is not None: with rffi.scoped_nonmovingbuffer(cadata) as buf: self._add_ca_certs(space, buf, len(cadata), ca_file_type) - + # load cafile or capath if cafile is not None or capath is not None: ret = libssl_SSL_CTX_load_verify_locations( diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py --- a/rpython/rlib/ropenssl.py +++ b/rpython/rlib/ropenssl.py @@ -255,6 +255,8 @@ OPENSSL_VERSION_NUMBER != 0x00909000 if OPENSSL_VERSION_NUMBER < 0x0090800f and not OPENSSL_NO_ECDH: OPENSSL_NO_ECDH = True +HAVE_TLSv1_2 = OPENSSL_VERSION_NUMBER >= 0x10001000 + def external(name, argtypes, restype, **kw): kw['compilation_info'] = eci @@ -284,6 +286,9 @@ ssl_external('SSL_get_SSL_CTX', [SSL], SSL_CTX) ssl_external('SSL_set_SSL_CTX', [SSL, SSL_CTX], SSL_CTX) ssl_external('TLSv1_method', [], SSL_METHOD) +if HAVE_TLSv1_2: + ssl_external('TLSv1_1_method', [], SSL_METHOD) + ssl_external('TLSv1_2_method', [], SSL_METHOD) ssl_external('SSLv2_method', [], SSL_METHOD) ssl_external('SSLv3_method', [], SSL_METHOD) ssl_external('SSLv23_method', [], SSL_METHOD) From noreply at buildbot.pypy.org Fri Mar 27 17:45:44 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Fri, 27 Mar 2015 17:45:44 +0100 (CET) Subject: [pypy-commit] pypy default: cleanup Message-ID: <20150327164544.5E1CC1C0300@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r76593:b32fbafab401 Date: 2015-03-27 16:25 +0000 http://bitbucket.org/pypy/pypy/changeset/b32fbafab401/ Log: cleanup diff --git a/rpython/rtyper/annlowlevel.py b/rpython/rtyper/annlowlevel.py --- a/rpython/rtyper/annlowlevel.py +++ b/rpython/rtyper/annlowlevel.py @@ -41,9 +41,10 @@ __repr__ = __str__ class LowLevelAnnotatorPolicy(AnnotatorPolicy): - def __init__(pol, rtyper=None): - pol.rtyper = rtyper + def __init__(self, rtyper=None): + self.rtyper = rtyper + @staticmethod def lowlevelspecialize(funcdesc, args_s, key_for_args): args_s, key1, builder = flatten_star_args(funcdesc, args_s) key = [] @@ -73,21 +74,20 @@ flowgraph = funcdesc.cachedgraph(key, builder=builder) args_s[:] = new_args_s return flowgraph - lowlevelspecialize = staticmethod(lowlevelspecialize) + @staticmethod def default_specialize(funcdesc, args_s): return LowLevelAnnotatorPolicy.lowlevelspecialize(funcdesc, args_s, {}) - default_specialize = staticmethod(default_specialize) specialize__ll = default_specialize + @staticmethod def specialize__ll_and_arg(funcdesc, args_s, *argindices): keys = {} for i in argindices: keys[i] = args_s[i].const return LowLevelAnnotatorPolicy.lowlevelspecialize(funcdesc, args_s, keys) - specialize__ll_and_arg = staticmethod(specialize__ll_and_arg) def annotate_lowlevel_helper(annotator, ll_function, args_s, policy=None): if policy is None: @@ -99,23 +99,23 @@ class MixLevelAnnotatorPolicy(LowLevelAnnotatorPolicy): - def __init__(pol, annhelper): - pol.rtyper = annhelper.rtyper + def __init__(self, annhelper): + self.rtyper = annhelper.rtyper - def default_specialize(pol, funcdesc, args_s): + def default_specialize(self, funcdesc, args_s): name = funcdesc.name if name.startswith('ll_') or name.startswith('_ll_'): # xxx can we do better? - return super(MixLevelAnnotatorPolicy, pol).default_specialize( + return super(MixLevelAnnotatorPolicy, self).default_specialize( funcdesc, args_s) else: return AnnotatorPolicy.default_specialize(funcdesc, args_s) - def specialize__arglltype(pol, funcdesc, args_s, i): - key = pol.rtyper.getrepr(args_s[i]).lowleveltype + def specialize__arglltype(self, funcdesc, args_s, i): + key = self.rtyper.getrepr(args_s[i]).lowleveltype alt_name = funcdesc.name+"__for_%sLlT" % key._short_name() return funcdesc.cachedgraph(key, alt_name=valid_identifier(alt_name)) - def specialize__genconst(pol, funcdesc, args_s, i): + def specialize__genconst(self, funcdesc, args_s, i): # XXX this is specific to the JIT TYPE = annotation_to_lltype(args_s[i], 'genconst') args_s[i] = lltype_to_annotation(TYPE) From noreply at buildbot.pypy.org Fri Mar 27 17:45:45 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Fri, 27 Mar 2015 17:45:45 +0100 (CET) Subject: [pypy-commit] pypy default: kill timelog() thing: there are profilers for that Message-ID: <20150327164545.A05BF1C0300@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r76594:da90c30dc0dd Date: 2015-03-27 16:45 +0000 http://bitbucket.org/pypy/pypy/changeset/da90c30dc0dd/ Log: kill timelog() thing: there are profilers for that diff --git a/rpython/rtyper/test/test_llinterp.py b/rpython/rtyper/test/test_llinterp.py --- a/rpython/rtyper/test/test_llinterp.py +++ b/rpython/rtyper/test/test_llinterp.py @@ -25,22 +25,12 @@ py.log._setstate(mod.logstate) - -def timelog(prefix, call, *args, **kwds): - #import time - #print prefix, "...", - #start = time.time() - res = call(*args, **kwds) - #elapsed = time.time() - start - #print "%.2f secs" % (elapsed,) - return res - def gengraph(func, argtypes=[], viewbefore='auto', policy=None, backendopt=False, config=None, **extraconfigopts): t = TranslationContext(config=config) t.config.set(**extraconfigopts) a = t.buildannotator(policy=policy) - timelog("annotating", a.build_types, func, argtypes, main_entry_point=True) + a.build_types(func, argtypes, main_entry_point=True) a.validate() if viewbefore == 'auto': viewbefore = getattr(option, 'view', False) @@ -49,13 +39,13 @@ t.view() global typer # we need it for find_exception typer = t.buildrtyper() - timelog("rtyper-specializing", typer.specialize) + typer.specialize() #t.view() - timelog("checking graphs", t.checkgraphs) + t.checkgraphs() if backendopt: from rpython.translator.backendopt.all import backend_optimizations backend_optimizations(t) - timelog("checking graphs", t.checkgraphs) + t.checkgraphs() if viewbefore: t.view() desc = t.annotator.bookkeeper.getdesc(func) From noreply at buildbot.pypy.org Fri Mar 27 18:42:02 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 27 Mar 2015 18:42:02 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Tweaks Message-ID: <20150327174202.A297B1C0334@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76595:3723af2273db Date: 2015-03-27 18:12 +0100 http://bitbucket.org/pypy/pypy/changeset/3723af2273db/ Log: Tweaks diff --git a/pypy/module/pypystm/test_pypy_c/support.py b/pypy/module/pypystm/test_pypy_c/support.py --- a/pypy/module/pypystm/test_pypy_c/support.py +++ b/pypy/module/pypystm/test_pypy_c/support.py @@ -56,7 +56,7 @@ while len(result) != NUM_THREADS: barrier(tnum, done=True) -def run_in_threads(function, arg_thread_num=False, arg_class=None): +def _run_in_threads(function, arg_thread_num=False, arg_class=None): locks = [] result = [] for i in range(NUM_THREADS): @@ -73,6 +73,13 @@ lock._py3k_acquire(timeout=30) if len(result) < len(locks): raise Exception("not all threads completed successfully") + +def run_in_threads(*args, **kwds): + import time + try: + _run_in_threads(*args, **kwds) + finally: + time.sleep(0.5) """ def setup_class(cls): @@ -112,16 +119,17 @@ print '*', ' '.join(cmdline) env = os.environ.copy() env['PYPYSTM'] = str(self.stmfile) - env['PYPYLOG'] = str(self.logfile) + #env['PYPYLOG'] = str(self.logfile) # pipe = subprocess.Popen(cmdline, - env=env, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - stdout, stderr = pipe.communicate() + env=env) +# stdout=subprocess.PIPE, +# stderr=subprocess.PIPE) +# stdout, stderr = pipe.communicate() + pipe.wait() if pipe.returncode > 0: raise IOError("subprocess error %d:\n%s" % (pipe.returncode, - stderr)) + '(see stderr)')) #stderr)) if pipe.returncode < 0: raise IOError("subprocess was killed by signal %d" % ( pipe.returncode,)) From noreply at buildbot.pypy.org Fri Mar 27 18:42:03 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 27 Mar 2015 18:42:03 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Fix: handle optional operation 'guard_not_invalidated?' as the last one in the list Message-ID: <20150327174203.D839D1C0334@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76596:9a7d81e36d6f Date: 2015-03-27 18:42 +0100 http://bitbucket.org/pypy/pypy/changeset/9a7d81e36d6f/ Log: Fix: handle optional operation 'guard_not_invalidated?' as the last one in the list diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py --- a/pypy/module/pypyjit/test_pypy_c/model.py +++ b/pypy/module/pypyjit/test_pypy_c/model.py @@ -273,6 +273,7 @@ @classmethod def parse_ops(cls, src): ops = [cls.parse_op(line) for line in src.splitlines()] + ops.append(('--end--', None, [], '...', True)) return [op for op in ops if op is not None] @classmethod @@ -423,6 +424,10 @@ raise InvalidMatch(message, frame=sys._getframe(1)) def match_op(self, op, (exp_opname, exp_res, exp_args, exp_descr, _)): + if exp_opname == '--end--': + self._assert(op == '--end--', 'got more ops than expected') + return + self._assert(op != '--end--', 'got less ops than expected') self._assert(op.name == exp_opname, "operation mismatch") self.match_var(op.res, exp_res) if exp_args[-1:] == ['...']: # exp_args ends with '...' @@ -435,18 +440,15 @@ self.match_descr(op.descr, exp_descr) - def _next_op(self, iter_ops, assert_raises=False, ignore_ops=set()): + def _next_op(self, iter_ops, ignore_ops=set()): try: while True: op = iter_ops.next() if op.name not in ignore_ops: break except StopIteration: - self._assert(assert_raises, "not enough operations") - return - else: - self._assert(not assert_raises, "operation list too long") - return op + return '--end--' + return op def try_match(self, op, exp_op): try: @@ -513,16 +515,17 @@ continue else: op = self._next_op(iter_ops, ignore_ops=ignore_ops) - self.match_op(op, exp_op) - except InvalidMatch, e: - if type(exp_op) is not str and exp_op[4] is False: # optional operation + try: + self.match_op(op, exp_op) + except InvalidMatch: + if type(exp_op) is str or exp_op[4] is not False: + raise + #else: optional operation iter_ops.revert_one() continue # try to match with the next exp_op + except InvalidMatch, e: e.opindex = iter_ops.index - 1 raise - # - # make sure we exhausted iter_ops - self._next_op(iter_ops, assert_raises=True, ignore_ops=ignore_ops) def match(self, expected_src, ignore_ops=[]): def format(src, opindex=None): @@ -565,9 +568,9 @@ return self def next(self): index = self.index - if index == len(self.sequence): + self.index = index + 1 + if index >= len(self.sequence): raise StopIteration - self.index = index + 1 return self.sequence[index] def revert_one(self): self.index -= 1 From noreply at buildbot.pypy.org Fri Mar 27 18:51:39 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 27 Mar 2015 18:51:39 +0100 (CET) Subject: [pypy-commit] pypy default: Failing test Message-ID: <20150327175139.5F6BC1C0334@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76597:03b673428d67 Date: 2015-03-27 18:50 +0100 http://bitbucket.org/pypy/pypy/changeset/03b673428d67/ Log: Failing test diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py --- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py +++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py @@ -384,6 +384,25 @@ """ assert not self.match(loop, expected) + def test_match_optional_op(self): + loop = """ + i1 = int_add(i0, 1) + """ + expected = """ + guard_not_invalidated? + i1 = int_add(i0, 1) + """ + assert self.match(loop, expected) + # + loop = """ + i1 = int_add(i0, 1) + """ + expected = """ + i1 = int_add(i0, 1) + guard_not_invalidated? + """ + assert self.match(loop, expected) + class TestRunPyPyC(BaseTestPyPyC): def test_run_function(self): From noreply at buildbot.pypy.org Fri Mar 27 18:51:40 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 27 Mar 2015 18:51:40 +0100 (CET) Subject: [pypy-commit] pypy default: Fix: handle optional operation 'guard_not_invalidated?' as the last one in the list Message-ID: <20150327175140.830401C0334@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76598:cada63adf489 Date: 2015-03-27 18:42 +0100 http://bitbucket.org/pypy/pypy/changeset/cada63adf489/ Log: Fix: handle optional operation 'guard_not_invalidated?' as the last one in the list diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py --- a/pypy/module/pypyjit/test_pypy_c/model.py +++ b/pypy/module/pypyjit/test_pypy_c/model.py @@ -271,6 +271,7 @@ @classmethod def parse_ops(cls, src): ops = [cls.parse_op(line) for line in src.splitlines()] + ops.append(('--end--', None, [], '...', True)) return [op for op in ops if op is not None] @classmethod @@ -403,6 +404,10 @@ raise InvalidMatch(message, frame=sys._getframe(1)) def match_op(self, op, (exp_opname, exp_res, exp_args, exp_descr, _)): + if exp_opname == '--end--': + self._assert(op == '--end--', 'got more ops than expected') + return + self._assert(op != '--end--', 'got less ops than expected') self._assert(op.name == exp_opname, "operation mismatch") self.match_var(op.res, exp_res) if exp_args[-1:] == ['...']: # exp_args ends with '...' @@ -415,18 +420,15 @@ self.match_descr(op.descr, exp_descr) - def _next_op(self, iter_ops, assert_raises=False, ignore_ops=set()): + def _next_op(self, iter_ops, ignore_ops=set()): try: while True: op = iter_ops.next() if op.name not in ignore_ops: break except StopIteration: - self._assert(assert_raises, "not enough operations") - return - else: - self._assert(not assert_raises, "operation list too long") - return op + return '--end--' + return op def try_match(self, op, exp_op): try: @@ -493,16 +495,17 @@ continue else: op = self._next_op(iter_ops, ignore_ops=ignore_ops) - self.match_op(op, exp_op) - except InvalidMatch, e: - if type(exp_op) is not str and exp_op[4] is False: # optional operation + try: + self.match_op(op, exp_op) + except InvalidMatch: + if type(exp_op) is str or exp_op[4] is not False: + raise + #else: optional operation iter_ops.revert_one() continue # try to match with the next exp_op + except InvalidMatch, e: e.opindex = iter_ops.index - 1 raise - # - # make sure we exhausted iter_ops - self._next_op(iter_ops, assert_raises=True, ignore_ops=ignore_ops) def match(self, expected_src, ignore_ops=[]): def format(src, opindex=None): @@ -545,9 +548,9 @@ return self def next(self): index = self.index - if index == len(self.sequence): + self.index = index + 1 + if index >= len(self.sequence): raise StopIteration - self.index = index + 1 return self.sequence[index] def revert_one(self): self.index -= 1 From noreply at buildbot.pypy.org Fri Mar 27 18:51:41 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 27 Mar 2015 18:51:41 +0100 (CET) Subject: [pypy-commit] pypy default: merge heads Message-ID: <20150327175141.E82511C0334@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76599:79d3671d75fe Date: 2015-03-27 18:51 +0100 http://bitbucket.org/pypy/pypy/changeset/79d3671d75fe/ Log: merge heads diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -141,7 +141,7 @@ res = _pypy_execute_source(source) before = rffi.aroundstate.before if before: before() - return rffi.cast(rffi.INT, res) + return rffi.cast(rffi.INT, res) @entrypoint('main', [], c_name='pypy_init_threads') def pypy_init_threads(): @@ -312,7 +312,7 @@ w_dict = app.getwdict(space) entry_point, _ = create_entry_point(space, w_dict) - return entry_point, None, PyPyAnnotatorPolicy(single_space = space) + return entry_point, None, PyPyAnnotatorPolicy() def interface(self, ns): for name in ['take_options', 'handle_config', 'print_help', 'target', diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -33,7 +33,8 @@ PY_SSL_CLIENT, PY_SSL_SERVER = 0, 1 (PY_SSL_VERSION_SSL2, PY_SSL_VERSION_SSL3, - PY_SSL_VERSION_SSL23, PY_SSL_VERSION_TLS1) = range(4) + PY_SSL_VERSION_SSL23, PY_SSL_VERSION_TLS1, PY_SSL_VERSION_TLS1_1, + PY_SSL_VERSION_TLS1_2) = range(6) SOCKET_IS_NONBLOCKING, SOCKET_IS_BLOCKING = 0, 1 SOCKET_HAS_TIMED_OUT, SOCKET_HAS_BEEN_CLOSED = 2, 3 @@ -72,6 +73,9 @@ constants["PROTOCOL_SSLv3"] = PY_SSL_VERSION_SSL3 constants["PROTOCOL_SSLv23"] = PY_SSL_VERSION_SSL23 constants["PROTOCOL_TLSv1"] = PY_SSL_VERSION_TLS1 +if HAVE_TLSv1_2: + constants["PROTOCOL_TLSv1_1"] = PY_SSL_VERSION_TLS1_1 + constants["PROTOCOL_TLSv1_2"] = PY_SSL_VERSION_TLS1_2 constants["OP_ALL"] = SSL_OP_ALL &~SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS constants["OP_NO_SSLv2"] = SSL_OP_NO_SSLv2 @@ -140,7 +144,7 @@ def __del__(self): rffi.free_nonmovingbuffer( - self.protos, self.buf, self.pinned, self.is_raw) + self.protos, self.buf, self.pinned, self.is_raw) @staticmethod def advertiseNPN_cb(s, data_ptr, len_ptr, args): @@ -162,7 +166,7 @@ client_len = len(npn.protos) else: client = lltype.nullptr(rffi.CCHARP.TO) - client_len = 0 + client_len = 0 libssl_SSL_select_next_proto(out_ptr, outlen_ptr, server, server_len, @@ -593,14 +597,14 @@ CB_MAXLEN = 128 with lltype.scoped_alloc(rffi.CCHARP.TO, CB_MAXLEN) as buf: - if (libssl_SSL_session_reused(self.ssl) ^ + if (libssl_SSL_session_reused(self.ssl) ^ (self.socket_type == PY_SSL_CLIENT)): # if session is resumed XOR we are the client length = libssl_SSL_get_finished(self.ssl, buf, CB_MAXLEN) else: # if a new session XOR we are the server length = libssl_SSL_get_peer_finished(self.ssl, buf, CB_MAXLEN) - + if length > 0: return space.wrap(rffi.charpsize2str(buf, intmask(length))) @@ -1107,7 +1111,7 @@ except OperationError as e: if not e.match(space, space.w_TypeError): raise - raise oefmt(space.w_TypeError, + raise oefmt(space.w_TypeError, "password callback must return a string") except OperationError as e: pw_info.operationerror = e @@ -1196,6 +1200,10 @@ method = libssl_SSLv2_method() elif protocol == PY_SSL_VERSION_SSL23: method = libssl_SSLv23_method() + elif protocol == PY_SSL_VERSION_TLS1_1 and HAVE_TLSv1_2: + method = libssl_TLSv1_1_method() + elif protocol == PY_SSL_VERSION_TLS1_2 and HAVE_TLSv1_2: + method = libssl_TLSv1_2_method() else: raise oefmt(space.w_ValueError, "invalid protocol version") ctx = libssl_SSL_CTX_new(method) @@ -1348,7 +1356,7 @@ except OperationError as e: if not e.match(space, space.w_TypeError): raise - raise oefmt(space.w_TypeError, + raise oefmt(space.w_TypeError, "password should be a string or callable") libssl_SSL_CTX_set_default_passwd_cb( @@ -1452,7 +1460,7 @@ if cadata is not None: with rffi.scoped_nonmovingbuffer(cadata) as buf: self._add_ca_certs(space, buf, len(cadata), ca_file_type) - + # load cafile or capath if cafile is not None or capath is not None: ret = libssl_SSL_CTX_load_verify_locations( diff --git a/pypy/tool/ann_override.py b/pypy/tool/ann_override.py --- a/pypy/tool/ann_override.py +++ b/pypy/tool/ann_override.py @@ -13,13 +13,12 @@ class PyPyAnnotatorPolicy(AnnotatorPolicy): - def __init__(pol, single_space=None): - pol.lookups = {} - pol.lookups_where = {} - pol.pypytypes = {} - pol.single_space = single_space + def __init__(self): + self.lookups = {} + self.lookups_where = {} + self.pypytypes = {} - def specialize__wrap(pol, funcdesc, args_s): + def specialize__wrap(self, funcdesc, args_s): from pypy.interpreter.baseobjspace import W_Root from rpython.annotator.classdef import ClassDef W_Root_def = funcdesc.bookkeeper.getuniqueclassdef(W_Root) @@ -51,102 +50,102 @@ typ = (None, str) return funcdesc.cachedgraph(typ) - def _remember_immutable(pol, t, cached): + def _remember_immutable(self, t, cached): # for jit benefit if cached not in t._immutable_fields_: # accessed this way just # for convenience t._immutable_fields_.append(cached) - def attach_lookup(pol, t, attr): + def attach_lookup(self, t, attr): cached = "cached_%s" % attr if not t.is_heaptype() and not t.is_cpytype(): - pol._remember_immutable(t, cached) + self._remember_immutable(t, cached) setattr(t, cached, t._lookup(attr)) return True return False - def attach_lookup_in_type_where(pol, t, attr): + def attach_lookup_in_type_where(self, t, attr): cached = "cached_where_%s" % attr if not t.is_heaptype() and not t.is_cpytype(): - pol._remember_immutable(t, cached) + self._remember_immutable(t, cached) setattr(t, cached, t._lookup_where(attr)) return True return False - def consider_lookup(pol, bookkeeper, attr): + def consider_lookup(self, bookkeeper, attr): from rpython.annotator.classdef import InstanceSource - assert attr not in pol.lookups + assert attr not in self.lookups from pypy.objspace.std import typeobject cached = "cached_%s" % attr clsdef = bookkeeper.getuniqueclassdef(typeobject.W_TypeObject) classdesc = clsdef.classdesc classdesc.classdict[cached] = Constant(None) clsdef.add_source_for_attribute(cached, classdesc) - for t in pol.pypytypes: - if pol.attach_lookup(t, attr): + for t in self.pypytypes: + if self.attach_lookup(t, attr): source = InstanceSource(bookkeeper, t) clsdef.add_source_for_attribute(cached, source) - pol.lookups[attr] = True + self.lookups[attr] = True - def consider_lookup_in_type_where(pol, bookkeeper, attr): + def consider_lookup_in_type_where(self, bookkeeper, attr): from rpython.annotator.classdef import InstanceSource - assert attr not in pol.lookups_where + assert attr not in self.lookups_where from pypy.objspace.std import typeobject cached = "cached_where_%s" % attr clsdef = bookkeeper.getuniqueclassdef(typeobject.W_TypeObject) classdesc = clsdef.classdesc classdesc.classdict[cached] = Constant((None, None)) clsdef.add_source_for_attribute(cached, classdesc) - for t in pol.pypytypes: - if pol.attach_lookup_in_type_where(t, attr): + for t in self.pypytypes: + if self.attach_lookup_in_type_where(t, attr): source = InstanceSource(bookkeeper, t) clsdef.add_source_for_attribute(cached, source) - pol.lookups_where[attr] = True + self.lookups_where[attr] = True - def specialize__lookup(pol, funcdesc, args_s): + def specialize__lookup(self, funcdesc, args_s): s_space, s_obj, s_name = args_s if s_name.is_constant(): attr = s_name.const def builder(translator, func): #print "LOOKUP", attr - pol.consider_lookup(funcdesc.bookkeeper, attr) + self.consider_lookup(funcdesc.bookkeeper, attr) d = {'__name__': ''} exec CACHED_LOOKUP % {'attr': attr} in d return translator.buildflowgraph(d['lookup_'+attr]) return funcdesc.cachedgraph(attr, builder=builder) else: - pol.lookups[None] = True + self.lookups[None] = True return funcdesc.cachedgraph(None) # don't specialize - def specialize__lookup_in_type_where(pol, funcdesc, args_s): + def specialize__lookup_in_type_where(self, funcdesc, args_s): s_space, s_obj, s_name = args_s if s_name.is_constant(): attr = s_name.const def builder(translator, func): #print "LOOKUP_IN_TYPE_WHERE", attr - pol.consider_lookup_in_type_where(funcdesc.bookkeeper, attr) + self.consider_lookup_in_type_where(funcdesc.bookkeeper, attr) d = {'__name__': ''} exec CACHED_LOOKUP_IN_TYPE_WHERE % {'attr': attr} in d return translator.buildflowgraph(d['lookup_in_type_where_'+attr]) return funcdesc.cachedgraph(attr, builder=builder) else: - pol.lookups_where[None] = True + self.lookups_where[None] = True return funcdesc.cachedgraph(None) - def event(pol, bookkeeper, what, x): + def event(self, bookkeeper, what, x): from pypy.objspace.std import typeobject if isinstance(x, typeobject.W_TypeObject): from rpython.annotator.classdef import InstanceSource clsdef = bookkeeper.getuniqueclassdef(typeobject.W_TypeObject) - pol.pypytypes[x] = True + self.pypytypes[x] = True #print "TYPE", x - for attr in pol.lookups: - if attr and pol.attach_lookup(x, attr): + for attr in self.lookups: + if attr and self.attach_lookup(x, attr): cached = "cached_%s" % attr source = InstanceSource(bookkeeper, x) clsdef.add_source_for_attribute(cached, source) - for attr in pol.lookups_where: - if attr and pol.attach_lookup_in_type_where(x, attr): + for attr in self.lookups_where: + if attr and self.attach_lookup_in_type_where(x, attr): cached = "cached_where_%s" % attr source = InstanceSource(bookkeeper, x) clsdef.add_source_for_attribute(cached, source) diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py --- a/rpython/annotator/annrpython.py +++ b/rpython/annotator/annrpython.py @@ -12,6 +12,7 @@ from rpython.annotator import model as annmodel, signature from rpython.annotator.argument import simple_args from rpython.annotator.bookkeeper import Bookkeeper +from rpython.rtyper.normalizecalls import perform_normalizations import py log = py.log.Producer("annrpython") @@ -317,6 +318,8 @@ graphs[graph] = True for graph in graphs: simplify.eliminate_empty_blocks(graph) + if block_subset is None: + perform_normalizations(self) #___ flowing annotations in blocks _____________________ diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py --- a/rpython/annotator/bookkeeper.py +++ b/rpython/annotator/bookkeeper.py @@ -42,7 +42,7 @@ def __setstate__(self, dic): self.__dict__.update(dic) # normal action - delayed_imports() + self.register_builtins() def __init__(self, annotator): self.annotator = annotator @@ -67,7 +67,13 @@ self.needs_generic_instantiate = {} self.thread_local_fields = set() - delayed_imports() + self.register_builtins() + + def register_builtins(self): + import rpython.annotator.builtin # for side-effects + from rpython.annotator.exception import standardexceptions + for cls in standardexceptions: + self.getuniqueclassdef(cls) def enter(self, position_key): """Start of an operation. @@ -605,6 +611,3 @@ def immutablevalue(x): return getbookkeeper().immutablevalue(x) - -def delayed_imports(): - import rpython.annotator.builtin diff --git a/rpython/annotator/exception.py b/rpython/annotator/exception.py new file mode 100644 --- /dev/null +++ b/rpython/annotator/exception.py @@ -0,0 +1,7 @@ +from rpython.rlib import rstackovf + +# the exceptions that can be implicitely raised by some operations +standardexceptions = set([TypeError, OverflowError, ValueError, + ZeroDivisionError, MemoryError, IOError, OSError, StopIteration, KeyError, + IndexError, AssertionError, RuntimeError, UnicodeDecodeError, + UnicodeEncodeError, NotImplementedError, rstackovf._StackOverflow]) diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py --- a/rpython/rlib/ropenssl.py +++ b/rpython/rlib/ropenssl.py @@ -255,6 +255,8 @@ OPENSSL_VERSION_NUMBER != 0x00909000 if OPENSSL_VERSION_NUMBER < 0x0090800f and not OPENSSL_NO_ECDH: OPENSSL_NO_ECDH = True +HAVE_TLSv1_2 = OPENSSL_VERSION_NUMBER >= 0x10001000 + def external(name, argtypes, restype, **kw): kw['compilation_info'] = eci @@ -284,6 +286,9 @@ ssl_external('SSL_get_SSL_CTX', [SSL], SSL_CTX) ssl_external('SSL_set_SSL_CTX', [SSL, SSL_CTX], SSL_CTX) ssl_external('TLSv1_method', [], SSL_METHOD) +if HAVE_TLSv1_2: + ssl_external('TLSv1_1_method', [], SSL_METHOD) + ssl_external('TLSv1_2_method', [], SSL_METHOD) ssl_external('SSLv2_method', [], SSL_METHOD) ssl_external('SSLv3_method', [], SSL_METHOD) ssl_external('SSLv23_method', [], SSL_METHOD) diff --git a/rpython/rtyper/annlowlevel.py b/rpython/rtyper/annlowlevel.py --- a/rpython/rtyper/annlowlevel.py +++ b/rpython/rtyper/annlowlevel.py @@ -41,9 +41,10 @@ __repr__ = __str__ class LowLevelAnnotatorPolicy(AnnotatorPolicy): - def __init__(pol, rtyper=None): - pol.rtyper = rtyper + def __init__(self, rtyper=None): + self.rtyper = rtyper + @staticmethod def lowlevelspecialize(funcdesc, args_s, key_for_args): args_s, key1, builder = flatten_star_args(funcdesc, args_s) key = [] @@ -73,21 +74,20 @@ flowgraph = funcdesc.cachedgraph(key, builder=builder) args_s[:] = new_args_s return flowgraph - lowlevelspecialize = staticmethod(lowlevelspecialize) + @staticmethod def default_specialize(funcdesc, args_s): return LowLevelAnnotatorPolicy.lowlevelspecialize(funcdesc, args_s, {}) - default_specialize = staticmethod(default_specialize) specialize__ll = default_specialize + @staticmethod def specialize__ll_and_arg(funcdesc, args_s, *argindices): keys = {} for i in argindices: keys[i] = args_s[i].const return LowLevelAnnotatorPolicy.lowlevelspecialize(funcdesc, args_s, keys) - specialize__ll_and_arg = staticmethod(specialize__ll_and_arg) def annotate_lowlevel_helper(annotator, ll_function, args_s, policy=None): if policy is None: @@ -99,24 +99,23 @@ class MixLevelAnnotatorPolicy(LowLevelAnnotatorPolicy): - def __init__(pol, annhelper): - pol.annhelper = annhelper - pol.rtyper = annhelper.rtyper + def __init__(self, annhelper): + self.rtyper = annhelper.rtyper - def default_specialize(pol, funcdesc, args_s): + def default_specialize(self, funcdesc, args_s): name = funcdesc.name if name.startswith('ll_') or name.startswith('_ll_'): # xxx can we do better? - return super(MixLevelAnnotatorPolicy, pol).default_specialize( + return super(MixLevelAnnotatorPolicy, self).default_specialize( funcdesc, args_s) else: return AnnotatorPolicy.default_specialize(funcdesc, args_s) - def specialize__arglltype(pol, funcdesc, args_s, i): - key = pol.rtyper.getrepr(args_s[i]).lowleveltype + def specialize__arglltype(self, funcdesc, args_s, i): + key = self.rtyper.getrepr(args_s[i]).lowleveltype alt_name = funcdesc.name+"__for_%sLlT" % key._short_name() return funcdesc.cachedgraph(key, alt_name=valid_identifier(alt_name)) - def specialize__genconst(pol, funcdesc, args_s, i): + def specialize__genconst(self, funcdesc, args_s, i): # XXX this is specific to the JIT TYPE = annotation_to_lltype(args_s[i], 'genconst') args_s[i] = lltype_to_annotation(TYPE) diff --git a/rpython/rtyper/exceptiondata.py b/rpython/rtyper/exceptiondata.py --- a/rpython/rtyper/exceptiondata.py +++ b/rpython/rtyper/exceptiondata.py @@ -1,15 +1,9 @@ from rpython.annotator import model as annmodel +from rpython.annotator.exception import standardexceptions from rpython.rtyper.llannotation import SomePtr -from rpython.rlib import rstackovf from rpython.rtyper.rclass import ( ll_issubclass, ll_type, ll_cast_to_object, getclassrepr, getinstancerepr) -# the exceptions that can be implicitely raised by some operations -standardexceptions = set([TypeError, OverflowError, ValueError, - ZeroDivisionError, MemoryError, IOError, OSError, StopIteration, KeyError, - IndexError, AssertionError, RuntimeError, UnicodeDecodeError, - UnicodeEncodeError, NotImplementedError, rstackovf._StackOverflow]) - class UnknownException(Exception): pass @@ -20,7 +14,6 @@ standardexceptions = standardexceptions def __init__(self, rtyper): - self.make_standard_exceptions(rtyper) # (NB. rclass identifies 'Exception' and 'object') r_type = rtyper.rootclass_repr r_instance = getinstancerepr(rtyper, None) @@ -32,11 +25,6 @@ self.lltype_of_exception_value = r_instance.lowleveltype self.rtyper = rtyper - def make_standard_exceptions(self, rtyper): - bk = rtyper.annotator.bookkeeper - for cls in self.standardexceptions: - bk.getuniqueclassdef(cls) - def finish(self, rtyper): bk = rtyper.annotator.bookkeeper for cls in self.standardexceptions: diff --git a/rpython/rtyper/rtyper.py b/rpython/rtyper/rtyper.py --- a/rpython/rtyper/rtyper.py +++ b/rpython/rtyper/rtyper.py @@ -26,7 +26,6 @@ attachRuntimeTypeInfo, Primitive) from rpython.rtyper.rmodel import Repr, inputconst, BrokenReprTyperError from rpython.rtyper.typesystem import LowLevelTypeSystem, getfunctionptr -from rpython.rtyper.normalizecalls import perform_normalizations from rpython.rtyper import rclass from rpython.rtyper.rclass import RootClassRepr from rpython.tool.pairtype import pair @@ -55,8 +54,6 @@ self.concrete_calltables = {} self.cache_dummy_values = {} self.lltype2vtable = {} - self.typererrors = [] - self.typererror_count = 0 # make the primitive_to_repr constant mapping self.primitive_to_repr = {} self.isinstance_helpers = {} @@ -169,22 +166,16 @@ def specialize(self, dont_simplify_again=False): """Main entry point: specialize all annotated blocks of the program.""" # specialize depends on annotator simplifications - assert dont_simplify_again in (False, True) # safety check if not dont_simplify_again: self.annotator.simplify() - - # first make sure that all functions called in a group have exactly - # the same signature, by hacking their flow graphs if needed - perform_normalizations(self.annotator) self.exceptiondata.finish(self) # new blocks can be created as a result of specialize_block(), so # we need to be careful about the loop here. self.already_seen = {} self.specialize_more_blocks() - if self.exceptiondata is not None: - self.exceptiondata.make_helpers(self) - self.specialize_more_blocks() # for the helpers just made + self.exceptiondata.make_helpers(self) + self.specialize_more_blocks() # for the helpers just made def getannmixlevel(self): if self.annmixlevel is not None: @@ -231,18 +222,11 @@ percentage = 100 * n // total if percentage >= previous_percentage + 5: previous_percentage = percentage - if self.typererror_count: - error_report = " but %d errors" % self.typererror_count - else: - error_report = '' - self.log.event('specializing: %d / %d blocks (%d%%)%s' % - (n, total, percentage, error_report)) + self.log.event('specializing: %d / %d blocks (%d%%)' % + (n, total, percentage)) # make sure all reprs so far have had their setup() called self.call_all_setups() - if self.typererrors: - self.dump_typererrors(to_log=True) - raise TyperError("there were %d error" % len(self.typererrors)) self.log.event('-=- specialized %d%s blocks -=-' % ( blockcount, newtext)) annmixlevel = self.annmixlevel @@ -250,29 +234,6 @@ if annmixlevel is not None: annmixlevel.finish() - def dump_typererrors(self, num=None, minimize=True, to_log=False): - c = 0 - bc = 0 - for err in self.typererrors[:num]: - c += 1 - if minimize and isinstance(err, BrokenReprTyperError): - bc += 1 - continue - graph, block, position = err.where - errmsg = ("TyperError-%d: %s\n" % (c, graph) + - str(err) + - "\n") - if to_log: - self.log.ERROR(errmsg) - else: - print errmsg - if bc: - minmsg = "(minimized %d errors away for this dump)" % (bc,) - if to_log: - self.log.ERROR(minmsg) - else: - print minmsg - def call_all_setups(self): # make sure all reprs so far have had their setup() called must_setup_more = [] @@ -324,9 +285,9 @@ # give the best possible types to the input args try: self.setup_block_entry(block) - except TyperError, e: - self.gottypererror(e, block, "block-entry", None) - return # cannot continue this block + except TyperError as e: + self.gottypererror(e, block, "block-entry") + raise # specialize all the operations, as far as possible @@ -341,9 +302,9 @@ try: hop.setup() # this is called from here to catch TyperErrors... self.translate_hl_to_ll(hop, varmapping) - except TyperError, e: - self.gottypererror(e, block, hop.spaceop, newops) - return # cannot continue this block: no op.result.concretetype + except TyperError as e: + self.gottypererror(e, block, hop.spaceop) + raise block.operations[:] = newops block.renamevariables(varmapping) @@ -432,9 +393,9 @@ continue # no conversion needed try: new_a1 = newops.convertvar(a1, r_a1, r_a2) - except TyperError, e: - self.gottypererror(e, block, link, newops) - continue # try other args + except TyperError as e: + self.gottypererror(e, block, link) + raise if new_a1 != a1: newlinkargs[i] = new_a1 @@ -516,14 +477,10 @@ "has no return value" % op.opname) op.result.concretetype = Void - def gottypererror(self, e, block, position, llops): - """Record a TyperError without crashing immediately. - Put a 'TyperError' operation in the graph instead. - """ + def gottypererror(self, exc, block, position): + """Record information about the location of a TyperError""" graph = self.annotator.annotated.get(block) - e.where = (graph, block, position) - self.typererror_count += 1 - raise + exc.where = (graph, block, position) # __________ regular operations __________ diff --git a/rpython/rtyper/test/test_llinterp.py b/rpython/rtyper/test/test_llinterp.py --- a/rpython/rtyper/test/test_llinterp.py +++ b/rpython/rtyper/test/test_llinterp.py @@ -25,22 +25,12 @@ py.log._setstate(mod.logstate) - -def timelog(prefix, call, *args, **kwds): - #import time - #print prefix, "...", - #start = time.time() - res = call(*args, **kwds) - #elapsed = time.time() - start - #print "%.2f secs" % (elapsed,) - return res - def gengraph(func, argtypes=[], viewbefore='auto', policy=None, backendopt=False, config=None, **extraconfigopts): t = TranslationContext(config=config) t.config.set(**extraconfigopts) a = t.buildannotator(policy=policy) - timelog("annotating", a.build_types, func, argtypes, main_entry_point=True) + a.build_types(func, argtypes, main_entry_point=True) a.validate() if viewbefore == 'auto': viewbefore = getattr(option, 'view', False) @@ -49,13 +39,13 @@ t.view() global typer # we need it for find_exception typer = t.buildrtyper() - timelog("rtyper-specializing", typer.specialize) + typer.specialize() #t.view() - timelog("checking graphs", t.checkgraphs) + t.checkgraphs() if backendopt: from rpython.translator.backendopt.all import backend_optimizations backend_optimizations(t) - timelog("checking graphs", t.checkgraphs) + t.checkgraphs() if viewbefore: t.view() desc = t.annotator.bookkeeper.getdesc(func) From noreply at buildbot.pypy.org Fri Mar 27 19:51:32 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 27 Mar 2015 19:51:32 +0100 (CET) Subject: [pypy-commit] pypy default: Add a comment describing the complex reason for why the two Message-ID: <20150327185132.0D02E1C0316@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76600:b9c118ee7033 Date: 2015-03-27 19:51 +0100 http://bitbucket.org/pypy/pypy/changeset/b9c118ee7033/ Log: Add a comment describing the complex reason for why the two int_add_ovf() here cannot overflow diff --git a/pypy/module/pypyjit/test_pypy_c/test_array.py b/pypy/module/pypyjit/test_pypy_c/test_array.py --- a/pypy/module/pypyjit/test_pypy_c/test_array.py +++ b/pypy/module/pypyjit/test_pypy_c/test_array.py @@ -94,13 +94,25 @@ guard_not_invalidated(descr=...) # the bound check guard on img has been killed (thanks to the asserts) i14 = getarrayitem_raw(i10, i8, descr=) + # advanced: the following int_add cannot overflow, because: + # - i14 fits inside 32 bits + # - i9 fits inside 33 bits, because: + # - it comes from the previous iteration's i15 + # - prev i19 = prev i18 + prev i15 + # - prev i18 fits inside 32 bits + # - prev i19 is guarded to fit inside 32 bits + # - so as a consequence, prev i15 fits inside 33 bits + # the new i15 thus fits inside "33.5" bits, which is enough to + # guarantee that the next int_add(i18, i15) cannot overflow either... i15 = int_add(i9, i14) i17 = int_sub(i8, 640) # the bound check guard on intimg has been killed (thanks to the asserts) i18 = getarrayitem_raw(i11, i17, descr=) i19 = int_add(i18, i15) - # on 64bit, there is a guard checking that i19 actually fits into 32bit - ... + # guard checking that i19 actually fits into 32bit + i20 = int_signext(i19, 4) + i65 = int_ne(i20, i19) + guard_false(i65, descr=...) setarrayitem_raw(i11, i8, _, descr=) i28 = int_add(i8, 1) --TICK-- From noreply at buildbot.pypy.org Fri Mar 27 19:52:59 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Fri, 27 Mar 2015 19:52:59 +0100 (CET) Subject: [pypy-commit] pypy default: Move FORCE_ATTRIBUTES_INTO_CLASSES to the only place where it's used Message-ID: <20150327185259.A491F1C0316@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r76601:7c4eccd92fb9 Date: 2015-03-27 18:53 +0000 http://bitbucket.org/pypy/pypy/changeset/7c4eccd92fb9/ Log: Move FORCE_ATTRIBUTES_INTO_CLASSES to the only place where it's used diff --git a/rpython/annotator/classdef.py b/rpython/annotator/classdef.py --- a/rpython/annotator/classdef.py +++ b/rpython/annotator/classdef.py @@ -2,8 +2,7 @@ Type inference for user-defined classes. """ from rpython.annotator.model import ( - SomePBC, s_ImpossibleValue, unionof, s_None, SomeInteger, - SomeTuple, SomeString, AnnotatorError, SomeUnicodeString) + SomePBC, s_ImpossibleValue, unionof, s_None, AnnotatorError) from rpython.annotator import description @@ -437,18 +436,3 @@ class NoSuchAttrError(AnnotatorError): """Raised when an attribute is found on a class where __slots__ or _attrs_ forbits it.""" - -# ____________________________________________________________ - -FORCE_ATTRIBUTES_INTO_CLASSES = { - EnvironmentError: {'errno': SomeInteger(), - 'strerror': SomeString(can_be_None=True), - 'filename': SomeString(can_be_None=True)}, -} - -try: - WindowsError -except NameError: - pass -else: - FORCE_ATTRIBUTES_INTO_CLASSES[WindowsError] = {'winerror': SomeInteger()} diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py --- a/rpython/annotator/description.py +++ b/rpython/annotator/description.py @@ -7,7 +7,7 @@ from rpython.annotator.argument import rawshape, ArgErr from rpython.tool.sourcetools import valid_identifier, func_with_new_name from rpython.tool.pairtype import extendabletype -from rpython.annotator.model import AnnotatorError +from rpython.annotator.model import AnnotatorError, SomeInteger, SomeString class CallFamily(object): """A family of Desc objects that could be called from common call sites. @@ -477,8 +477,7 @@ if (self.is_builtin_exception_class() and self.all_enforced_attrs is None): - from rpython.annotator import classdef - if cls not in classdef.FORCE_ATTRIBUTES_INTO_CLASSES: + if cls not in FORCE_ATTRIBUTES_INTO_CLASSES: self.all_enforced_attrs = [] # no attribute allowed def add_source_attribute(self, name, value, mixin=False): @@ -573,8 +572,7 @@ try: return self._classdefs[key] except KeyError: - from rpython.annotator.classdef import ( - ClassDef, FORCE_ATTRIBUTES_INTO_CLASSES) + from rpython.annotator.classdef import ClassDef classdef = ClassDef(self.bookkeeper, self) self.bookkeeper.classdefs.append(classdef) self._classdefs[key] = classdef @@ -1077,3 +1075,18 @@ MemberDescriptorTypes.append(type(OSError.errno)) except AttributeError: # on CPython <= 2.4 pass + +# ____________________________________________________________ + +FORCE_ATTRIBUTES_INTO_CLASSES = { + EnvironmentError: {'errno': SomeInteger(), + 'strerror': SomeString(can_be_None=True), + 'filename': SomeString(can_be_None=True)}, +} + +try: + WindowsError +except NameError: + pass +else: + FORCE_ATTRIBUTES_INTO_CLASSES[WindowsError] = {'winerror': SomeInteger()} From noreply at buildbot.pypy.org Fri Mar 27 20:24:49 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Fri, 27 Mar 2015 20:24:49 +0100 (CET) Subject: [pypy-commit] pypy default: fix Message-ID: <20150327192449.A36261C0334@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r76602:d158f5108e0f Date: 2015-03-27 19:24 +0000 http://bitbucket.org/pypy/pypy/changeset/d158f5108e0f/ Log: fix diff --git a/pypy/module/_minimal_curses/interp_curses.py b/pypy/module/_minimal_curses/interp_curses.py --- a/pypy/module/_minimal_curses/interp_curses.py +++ b/pypy/module/_minimal_curses/interp_curses.py @@ -13,7 +13,7 @@ def __init__(self, msg): self.msg = msg -from rpython.annotator.classdef import FORCE_ATTRIBUTES_INTO_CLASSES +from rpython.annotator.description import FORCE_ATTRIBUTES_INTO_CLASSES from rpython.annotator.model import SomeString # this is necessary due to annmixlevel From noreply at buildbot.pypy.org Fri Mar 27 20:56:03 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Fri, 27 Mar 2015 20:56:03 +0100 (CET) Subject: [pypy-commit] pypy default: Prevent Repr from ever appearing in a calldesc Message-ID: <20150327195603.EEF6D1C0A29@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r76603:6274aea11083 Date: 2015-03-27 19:56 +0000 http://bitbucket.org/pypy/pypy/changeset/6274aea11083/ Log: Prevent Repr from ever appearing in a calldesc diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py --- a/rpython/annotator/description.py +++ b/rpython/annotator/description.py @@ -404,6 +404,8 @@ name=None, basedesc=None, classdict=None, specialize=None): super(ClassDesc, self).__init__(bookkeeper, cls) + if '__NOT_RPYTHON__' in cls.__dict__: + raise AnnotatorError('Bad class') if name is None: name = cls.__module__ + '.' + cls.__name__ diff --git a/rpython/rtyper/lltypesystem/rbuilder.py b/rpython/rtyper/lltypesystem/rbuilder.py --- a/rpython/rtyper/lltypesystem/rbuilder.py +++ b/rpython/rtyper/lltypesystem/rbuilder.py @@ -401,18 +401,6 @@ def empty(self): return nullptr(self.lowleveltype.TO) - @classmethod - def ll_new(cls, init_size): - # Clamp 'init_size' to be a value between 0 and 1280. - # Negative values are mapped to 1280. - init_size = intmask(min(r_uint(init_size), r_uint(1280))) - ll_builder = lltype.malloc(cls.lowleveltype.TO) - ll_builder.current_buf = ll_builder.mallocfn(init_size) - ll_builder.current_pos = 0 - ll_builder.current_end = init_size - ll_builder.total_size = init_size - return ll_builder - ll_append = staticmethod(ll_append) ll_append_char = staticmethod(ll_append_char) ll_append_slice = staticmethod(ll_append_slice) @@ -431,6 +419,19 @@ lltype.Ptr(lltype.Array(lltype.Char, hints={'nolength': True})) ) + @staticmethod + def ll_new(init_size): + # Clamp 'init_size' to be a value between 0 and 1280. + # Negative values are mapped to 1280. + init_size = intmask(min(r_uint(init_size), r_uint(1280))) + ll_builder = lltype.malloc(STRINGBUILDER) + ll_builder.current_buf = ll_builder.mallocfn(init_size) + ll_builder.current_pos = 0 + ll_builder.current_end = init_size + ll_builder.total_size = init_size + return ll_builder + + class UnicodeBuilderRepr(BaseStringBuilderRepr): lowleveltype = lltype.Ptr(UNICODEBUILDER) basetp = UNICODE @@ -440,5 +441,18 @@ lltype.Ptr(lltype.Array(lltype.UniChar, hints={'nolength': True})) ) + @staticmethod + def ll_new(init_size): + # Clamp 'init_size' to be a value between 0 and 1280. + # Negative values are mapped to 1280. + init_size = intmask(min(r_uint(init_size), r_uint(1280))) + ll_builder = lltype.malloc(UNICODEBUILDER) + ll_builder.current_buf = ll_builder.mallocfn(init_size) + ll_builder.current_pos = 0 + ll_builder.current_end = init_size + ll_builder.total_size = init_size + return ll_builder + + unicodebuilder_repr = UnicodeBuilderRepr() stringbuilder_repr = StringBuilderRepr() diff --git a/rpython/rtyper/rmodel.py b/rpython/rtyper/rmodel.py --- a/rpython/rtyper/rmodel.py +++ b/rpython/rtyper/rmodel.py @@ -25,6 +25,7 @@ """ __metaclass__ = extendabletype _initialized = setupstate.NOTINITIALIZED + __NOT_RPYTHON__ = True def __repr__(self): return '<%s %s>' % (self.__class__.__name__, self.lowleveltype) From noreply at buildbot.pypy.org Fri Mar 27 21:17:00 2015 From: noreply at buildbot.pypy.org (groggi) Date: Fri, 27 Mar 2015 21:17:00 +0100 (CET) Subject: [pypy-commit] pypy gc-incminimark-pinning-improve: add description for PYPY_GC_MAX_PINNED Message-ID: <20150327201700.29BC01C028C@cobra.cs.uni-duesseldorf.de> Author: Gregor Wegberg Branch: gc-incminimark-pinning-improve Changeset: r76604:d4677a1be92e Date: 2015-03-27 20:33 +0100 http://bitbucket.org/pypy/pypy/changeset/d4677a1be92e/ Log: add description for PYPY_GC_MAX_PINNED diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -47,6 +47,11 @@ too slow for normal use. Values are 0 (off), 1 (on major collections) or 2 (also on minor collections). + + PYPY_GC_MAX_PINNED The maximal number of pinned objects at any point + in time. Defaults to a conservative value depending + on nursery size and maximum object size inside the + nursery. Useful for debugging by setting it to 0. """ # XXX Should find a way to bound the major collection threshold by the # XXX total addressable size. Maybe by keeping some minimarkpage arenas From noreply at buildbot.pypy.org Fri Mar 27 21:17:01 2015 From: noreply at buildbot.pypy.org (groggi) Date: Fri, 27 Mar 2015 21:17:01 +0100 (CET) Subject: [pypy-commit] pypy gc-incminimark-pinning-improve: indent it like the rest Message-ID: <20150327201701.575981C028C@cobra.cs.uni-duesseldorf.de> Author: Gregor Wegberg Branch: gc-incminimark-pinning-improve Changeset: r76605:a6e1427c0f98 Date: 2015-03-27 20:34 +0100 http://bitbucket.org/pypy/pypy/changeset/a6e1427c0f98/ Log: indent it like the rest diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -48,10 +48,10 @@ 1 (on major collections) or 2 (also on minor collections). - PYPY_GC_MAX_PINNED The maximal number of pinned objects at any point - in time. Defaults to a conservative value depending - on nursery size and maximum object size inside the - nursery. Useful for debugging by setting it to 0. + PYPY_GC_MAX_PINNED The maximal number of pinned objects at any point + in time. Defaults to a conservative value depending + on nursery size and maximum object size inside the + nursery. Useful for debugging by setting it to 0. """ # XXX Should find a way to bound the major collection threshold by the # XXX total addressable size. Maybe by keeping some minimarkpage arenas From noreply at buildbot.pypy.org Fri Mar 27 21:17:02 2015 From: noreply at buildbot.pypy.org (groggi) Date: Fri, 27 Mar 2015 21:17:02 +0100 (CET) Subject: [pypy-commit] pypy gc-incminimark-pinning-improve: add whatsnew entry for branch Message-ID: <20150327201702.72E0C1C028C@cobra.cs.uni-duesseldorf.de> Author: Gregor Wegberg Branch: gc-incminimark-pinning-improve Changeset: r76606:061f5b51f7bd Date: 2015-03-27 21:16 +0100 http://bitbucket.org/pypy/pypy/changeset/061f5b51f7bd/ Log: add whatsnew entry for branch diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -5,3 +5,9 @@ .. this is a revision shortly after release-2.5.1 .. startrev: 397b96217b85 +.. branch: gc-incminimark-pinning-improve +Object Pinning is now used in `bz2` and `rzlib` (therefore also affects +Python's `zlib`). In case the data to compress/decompress is inside the nursery +(incminimark) it no longer needs to create a non-moving copy of it. This saves +one `malloc` and copying the data. Additionally a new GC environment variable +is introduced (`PYPY_GC_MAX_PINNED`) primarily for debugging purposes. From noreply at buildbot.pypy.org Fri Mar 27 22:23:16 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 27 Mar 2015 22:23:16 +0100 (CET) Subject: [pypy-commit] pypy default: Can stmgc-c7 work with CPython? Message-ID: <20150327212316.9D6301C0334@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76607:a40e7f803b85 Date: 2015-03-27 22:23 +0100 http://bitbucket.org/pypy/pypy/changeset/a40e7f803b85/ Log: Can stmgc-c7 work with CPython? diff --git a/pypy/doc/stm.rst b/pypy/doc/stm.rst --- a/pypy/doc/stm.rst +++ b/pypy/doc/stm.rst @@ -29,7 +29,8 @@ ==================== ``pypy-stm`` is a variant of the regular PyPy interpreter. (This -version supports Python 2.7; see below for `Python 3`_.) With caveats_ +version supports Python 2.7; see below for `Python 3, CPython, +and others`_.) With caveats_ listed below, it should be in theory within 20%-50% slower than a regular PyPy, comparing the JIT version in both cases (but see below!). It is called @@ -178,8 +179,8 @@ -Python 3 -======== +Python 3, CPython, and others +============================= In this document I describe "pypy-stm", which is based on PyPy's Python 2.7 interpreter. Supporting Python 3 should take about half an @@ -194,6 +195,29 @@ framework, although the amount of work to put there might vary, because the STM framework within RPython is currently targeting the PyPy interpreter and other ones might have slightly different needs. +But in general, all the tedious transformations are done by RPython +and you're only left with the (hopefully few) hard and interesting bits. + +The core of STM works as a library written in C (see `reference to +implementation details`_ below). It means that it can be used on +other interpreters than the ones produced by RPython. Duhton_ is an +early example of that. At this point, you might think about adapting +this library for CPython. You're warned, though: as far as I can +tell, it is a doomed idea. I had a hard time debugging Duhton, and +that's infinitely simpler than CPython. Even ignoring that, you can +see in the C sources of Duhton that many core design decisions are +different than in CPython: no refcounting; limited support for +prebuilt "static" objects; ``stm_read()`` and ``stm_write()`` macro +calls everywhere (and getting very rare and very obscure bugs if you +forget one); and so on. You could imagine some custom special-purpose +extension of the C language, which you would preprocess to regular C. +In my opinion that's starting to look a lot like RPython itself, but +maybe you'd prefer this approach. Of course you still have to worry +about each and every C extension module you need, but maybe you'd have +a way forward. + +.. _Duhton: https://bitbucket.org/pypy/duhton + User Guide From noreply at buildbot.pypy.org Fri Mar 27 22:26:01 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 27 Mar 2015 22:26:01 +0100 (CET) Subject: [pypy-commit] pypy.org extradoc: update the values Message-ID: <20150327212601.C1BD81C03D9@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r586:84856e0da5ef Date: 2015-03-27 22:26 +0100 http://bitbucket.org/pypy/pypy.org/changeset/84856e0da5ef/ Log: update the values diff --git a/don3.html b/don3.html --- a/don3.html +++ b/don3.html @@ -9,13 +9,13 @@ - $51369 of $60000 (85.6%) + $51891 of $60000 (86.5%)
    From noreply at buildbot.pypy.org Sat Mar 28 01:39:37 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Sat, 28 Mar 2015 01:39:37 +0100 (CET) Subject: [pypy-commit] pypy default: cleanup Message-ID: <20150328003937.78B231C028C@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r76608:ee8477ce9690 Date: 2015-03-28 00:39 +0000 http://bitbucket.org/pypy/pypy/changeset/ee8477ce9690/ Log: cleanup diff --git a/rpython/rtyper/extfunc.py b/rpython/rtyper/extfunc.py --- a/rpython/rtyper/extfunc.py +++ b/rpython/rtyper/extfunc.py @@ -157,12 +157,11 @@ r_result = rtyper.getrepr(s_result) ll_result = r_result.lowleveltype name = getattr(self, 'name', None) or self.instance.__name__ - method_name = rtyper.type_system.name[:2] + 'typeimpl' fake_method_name = rtyper.type_system.name[:2] + 'typefakeimpl' - impl = getattr(self, method_name, None) - fakeimpl = getattr(self, fake_method_name, self.instance) + impl = getattr(self, 'lltypeimpl', None) + fakeimpl = getattr(self, 'lltypefakeimpl', self.instance) if impl: - if hasattr(self, fake_method_name): + if hasattr(self, 'lltypefakeimpl'): # If we have both an llimpl and an llfakeimpl, # we need a wrapper that selects the proper one and calls it from rpython.tool.sourcetools import func_with_new_name diff --git a/rpython/translator/backendopt/all.py b/rpython/translator/backendopt/all.py --- a/rpython/translator/backendopt/all.py +++ b/rpython/translator/backendopt/all.py @@ -151,8 +151,6 @@ inline_heuristic, call_count_pred=None, inline_graph_from_anywhere=False): - - type_system = translator.rtyper.type_system.name # inline functions in each other if inline_threshold: log.inlining("phase with threshold factor: %s" % inline_threshold) @@ -171,7 +169,7 @@ # vaporize mallocs if config.mallocs: log.malloc("starting malloc removal") - remove_mallocs(translator, graphs, type_system) + remove_mallocs(translator, graphs) if config.print_statistics: print "after malloc removal:" diff --git a/rpython/translator/backendopt/malloc.py b/rpython/translator/backendopt/malloc.py --- a/rpython/translator/backendopt/malloc.py +++ b/rpython/translator/backendopt/malloc.py @@ -536,17 +536,17 @@ raise AssertionError(op.opname) -def remove_simple_mallocs(graph, type_system='lltypesystem', verbose=True): +def remove_simple_mallocs(graph, verbose=True): remover = LLTypeMallocRemover(verbose) return remover.remove_simple_mallocs(graph) -def remove_mallocs(translator, graphs=None, type_system="lltypesystem"): +def remove_mallocs(translator, graphs=None): if graphs is None: graphs = translator.graphs tot = 0 for graph in graphs: - count = remove_simple_mallocs(graph, type_system=type_system, verbose=translator.config.translation.verbose) + count = remove_simple_mallocs(graph, verbose=translator.config.translation.verbose) if count: # remove typical leftovers from malloc removal removenoops.remove_same_as(graph) diff --git a/rpython/translator/backendopt/test/test_all.py b/rpython/translator/backendopt/test/test_all.py --- a/rpython/translator/backendopt/test/test_all.py +++ b/rpython/translator/backendopt/test/test_all.py @@ -42,7 +42,6 @@ HUGE_THRESHOLD = 100*INLINE_THRESHOLD_FOR_TEST class TestLLType(object): - type_system = 'lltype' check_malloc_removed = MallocRemovalTest.check_malloc_removed def translateopt(self, func, sig, **optflags): diff --git a/rpython/translator/backendopt/test/test_inline.py b/rpython/translator/backendopt/test/test_inline.py --- a/rpython/translator/backendopt/test/test_inline.py +++ b/rpython/translator/backendopt/test/test_inline.py @@ -47,8 +47,6 @@ self.data2 = 456 class TestInline(BaseRtypingTest): - type_system = 'lltype' - def translate(self, func, argtypes): t = TranslationContext() t.buildannotator().build_types(func, argtypes) diff --git a/rpython/translator/backendopt/test/test_malloc.py b/rpython/translator/backendopt/test/test_malloc.py --- a/rpython/translator/backendopt/test/test_malloc.py +++ b/rpython/translator/backendopt/test/test_malloc.py @@ -10,7 +10,6 @@ from rpython.conftest import option class TestMallocRemoval(object): - type_system = 'lltype' MallocRemover = LLTypeMallocRemover def check_malloc_removed(cls, graph): diff --git a/rpython/translator/backendopt/test/test_mallocv.py b/rpython/translator/backendopt/test/test_mallocv.py --- a/rpython/translator/backendopt/test/test_mallocv.py +++ b/rpython/translator/backendopt/test/test_mallocv.py @@ -17,8 +17,6 @@ class TestMallocRemoval(object): - type_system = 'lltype' - def check_malloc_removed(cls, graph, expected_mallocs, expected_calls): count_mallocs = 0 count_calls = 0 diff --git a/rpython/translator/backendopt/test/test_storesink.py b/rpython/translator/backendopt/test/test_storesink.py --- a/rpython/translator/backendopt/test/test_storesink.py +++ b/rpython/translator/backendopt/test/test_storesink.py @@ -7,8 +7,6 @@ from rpython.conftest import option class TestStoreSink(object): - type_system = 'lltype' - def translate(self, func, argtypes): t = TranslationContext() t.buildannotator().build_types(func, argtypes) diff --git a/rpython/translator/backendopt/test/test_writeanalyze.py b/rpython/translator/backendopt/test/test_writeanalyze.py --- a/rpython/translator/backendopt/test/test_writeanalyze.py +++ b/rpython/translator/backendopt/test/test_writeanalyze.py @@ -7,8 +7,6 @@ class BaseTest(object): - - type_system = 'lltype' Analyzer = WriteAnalyzer def translate(self, func, sig): diff --git a/rpython/translator/test/test_exceptiontransform.py b/rpython/translator/test/test_exceptiontransform.py --- a/rpython/translator/test/test_exceptiontransform.py +++ b/rpython/translator/test/test_exceptiontransform.py @@ -27,8 +27,6 @@ return interp.eval_graph(graph, values) class TestExceptionTransform: - type_system = 'lltype' - def compile(self, fn, inputargs): from rpython.translator.c.test.test_genc import compile return compile(fn, inputargs) @@ -239,7 +237,7 @@ etrafo.create_exception_handling(g) ops = dict.fromkeys([o.opname for b, o in g.iterblockops()]) assert 'zero_gc_pointers_inside' in ops - + def test_llexternal(self): from rpython.rtyper.lltypesystem.rffi import llexternal from rpython.rtyper.lltypesystem import lltype From noreply at buildbot.pypy.org Sat Mar 28 09:38:28 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 28 Mar 2015 09:38:28 +0100 (CET) Subject: [pypy-commit] pypy default: Mention a case of easy-to-fix write-write conflicts Message-ID: <20150328083828.E1F811C0DC1@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76609:295686ab115d Date: 2015-03-28 09:23 +0100 http://bitbucket.org/pypy/pypy/changeset/295686ab115d/ Log: Mention a case of easy-to-fix write-write conflicts diff --git a/pypy/doc/stm.rst b/pypy/doc/stm.rst --- a/pypy/doc/stm.rst +++ b/pypy/doc/stm.rst @@ -409,6 +409,25 @@ This means that ``threadlocalproperty`` is useful mainly to avoid conflicts from cache-like data structures. +* In addition to all of the above, there are cases where write-write + conflicts are caused by writing the same value to an attribute again + and again. See for example ea2e519614ab_: this fixes two such + issues where we write an object field without first checking if we + already did it. The ``dont_change_any_more`` field is a flag set to + ``True`` in that part of the code, but usually this + ``rtyper_makekey()`` method will be called many times for the same + object; the code used to repeatedly set the flag to ``True``, but + now it first checks and only does the write if it is ``False``. + Similarly, in the second half of the checkin, the method + ``setup_block_entry()`` used to both assign the ``concretetype`` + fields and return a list, but its two callers were different: one + would really need the ``concretetype`` fields initialized, whereas + the other would only need to get its result list --- the + ``concretetype`` field in that case might already be set or not, but + that would not matter. + +.. _ea2e519614ab: https://bitbucket.org/pypy/pypy/commits/ea2e519614ab + Note that Python is a complicated language; there are a number of less common cases that may cause conflict (of any kind) where we might not expect it at priori. In many of these cases it could be fixed; please From noreply at buildbot.pypy.org Sat Mar 28 09:38:30 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 28 Mar 2015 09:38:30 +0100 (CET) Subject: [pypy-commit] pypy default: Expand the text on threadlocalproperty. Message-ID: <20150328083830.26F4C1C0DC1@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76610:cda2320ed4dc Date: 2015-03-28 09:38 +0100 http://bitbucket.org/pypy/pypy/changeset/cda2320ed4dc/ Log: Expand the text on threadlocalproperty. diff --git a/pypy/doc/stm.rst b/pypy/doc/stm.rst --- a/pypy/doc/stm.rst +++ b/pypy/doc/stm.rst @@ -396,18 +396,30 @@ and ``y`` that are thread-local: reading or writing them from concurrently-running transactions will return independent results. (Any other attributes of ``Foo`` instances will be globally visible - from all threads, as usual.) The optional argument to - ``threadlocalproperty()`` is the default value factory: in case no - value was assigned in the current thread yet, the factory is called - and its result becomes the value in that thread (like - ``collections.defaultdict``). If no default value factory is - specified, uninitialized reads raise ``AttributeError``. Note that - with ``TransactionQueue`` you get a pool of a fixed number of - threads, each running the transactions one after the other; such - thread-local properties will have the value last stored in them in - the same thread,, which may come from a random previous transaction. - This means that ``threadlocalproperty`` is useful mainly to avoid - conflicts from cache-like data structures. + from all threads, as usual.) This is useful together with + ``TransactionQueue`` for these two cases: + + - For attributes of long-lived objects that change during one + transaction, but should always be reset to some initial value + around transaction (for example, initialized to 0 at the start of + a transaction; or, if used for a list of pending things to do + within this transaction, it will always be empty at the end of one + transaction). + + - For general caches across transactions. With ``TransactionQueue`` + you get a pool of a fixed number N of threads, each running the + transactions serially. A thread-local property will have the + value last stored in it by the same thread, which may come from a + random previous transaction. Basically, you get N copies of the + property's value, and each transaction accesses a random copy. It + works fine for caches. + + In more details, the optional argument to ``threadlocalproperty()`` + is the default value factory: in case no value was assigned in the + current thread yet, the factory is called and its result becomes the + value in that thread (like ``collections.defaultdict``). If no + default value factory is specified, uninitialized reads raise + ``AttributeError``. * In addition to all of the above, there are cases where write-write conflicts are caused by writing the same value to an attribute again From noreply at buildbot.pypy.org Sat Mar 28 09:38:31 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 28 Mar 2015 09:38:31 +0100 (CET) Subject: [pypy-commit] pypy default: merge heads Message-ID: <20150328083831.B3C5B1C0DC1@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76611:f8ee0444befc Date: 2015-03-28 09:38 +0100 http://bitbucket.org/pypy/pypy/changeset/f8ee0444befc/ Log: merge heads diff --git a/rpython/rtyper/extfunc.py b/rpython/rtyper/extfunc.py --- a/rpython/rtyper/extfunc.py +++ b/rpython/rtyper/extfunc.py @@ -157,12 +157,11 @@ r_result = rtyper.getrepr(s_result) ll_result = r_result.lowleveltype name = getattr(self, 'name', None) or self.instance.__name__ - method_name = rtyper.type_system.name[:2] + 'typeimpl' fake_method_name = rtyper.type_system.name[:2] + 'typefakeimpl' - impl = getattr(self, method_name, None) - fakeimpl = getattr(self, fake_method_name, self.instance) + impl = getattr(self, 'lltypeimpl', None) + fakeimpl = getattr(self, 'lltypefakeimpl', self.instance) if impl: - if hasattr(self, fake_method_name): + if hasattr(self, 'lltypefakeimpl'): # If we have both an llimpl and an llfakeimpl, # we need a wrapper that selects the proper one and calls it from rpython.tool.sourcetools import func_with_new_name diff --git a/rpython/translator/backendopt/all.py b/rpython/translator/backendopt/all.py --- a/rpython/translator/backendopt/all.py +++ b/rpython/translator/backendopt/all.py @@ -151,8 +151,6 @@ inline_heuristic, call_count_pred=None, inline_graph_from_anywhere=False): - - type_system = translator.rtyper.type_system.name # inline functions in each other if inline_threshold: log.inlining("phase with threshold factor: %s" % inline_threshold) @@ -171,7 +169,7 @@ # vaporize mallocs if config.mallocs: log.malloc("starting malloc removal") - remove_mallocs(translator, graphs, type_system) + remove_mallocs(translator, graphs) if config.print_statistics: print "after malloc removal:" diff --git a/rpython/translator/backendopt/malloc.py b/rpython/translator/backendopt/malloc.py --- a/rpython/translator/backendopt/malloc.py +++ b/rpython/translator/backendopt/malloc.py @@ -536,17 +536,17 @@ raise AssertionError(op.opname) -def remove_simple_mallocs(graph, type_system='lltypesystem', verbose=True): +def remove_simple_mallocs(graph, verbose=True): remover = LLTypeMallocRemover(verbose) return remover.remove_simple_mallocs(graph) -def remove_mallocs(translator, graphs=None, type_system="lltypesystem"): +def remove_mallocs(translator, graphs=None): if graphs is None: graphs = translator.graphs tot = 0 for graph in graphs: - count = remove_simple_mallocs(graph, type_system=type_system, verbose=translator.config.translation.verbose) + count = remove_simple_mallocs(graph, verbose=translator.config.translation.verbose) if count: # remove typical leftovers from malloc removal removenoops.remove_same_as(graph) diff --git a/rpython/translator/backendopt/test/test_all.py b/rpython/translator/backendopt/test/test_all.py --- a/rpython/translator/backendopt/test/test_all.py +++ b/rpython/translator/backendopt/test/test_all.py @@ -42,7 +42,6 @@ HUGE_THRESHOLD = 100*INLINE_THRESHOLD_FOR_TEST class TestLLType(object): - type_system = 'lltype' check_malloc_removed = MallocRemovalTest.check_malloc_removed def translateopt(self, func, sig, **optflags): diff --git a/rpython/translator/backendopt/test/test_inline.py b/rpython/translator/backendopt/test/test_inline.py --- a/rpython/translator/backendopt/test/test_inline.py +++ b/rpython/translator/backendopt/test/test_inline.py @@ -47,8 +47,6 @@ self.data2 = 456 class TestInline(BaseRtypingTest): - type_system = 'lltype' - def translate(self, func, argtypes): t = TranslationContext() t.buildannotator().build_types(func, argtypes) diff --git a/rpython/translator/backendopt/test/test_malloc.py b/rpython/translator/backendopt/test/test_malloc.py --- a/rpython/translator/backendopt/test/test_malloc.py +++ b/rpython/translator/backendopt/test/test_malloc.py @@ -10,7 +10,6 @@ from rpython.conftest import option class TestMallocRemoval(object): - type_system = 'lltype' MallocRemover = LLTypeMallocRemover def check_malloc_removed(cls, graph): diff --git a/rpython/translator/backendopt/test/test_mallocv.py b/rpython/translator/backendopt/test/test_mallocv.py --- a/rpython/translator/backendopt/test/test_mallocv.py +++ b/rpython/translator/backendopt/test/test_mallocv.py @@ -17,8 +17,6 @@ class TestMallocRemoval(object): - type_system = 'lltype' - def check_malloc_removed(cls, graph, expected_mallocs, expected_calls): count_mallocs = 0 count_calls = 0 diff --git a/rpython/translator/backendopt/test/test_storesink.py b/rpython/translator/backendopt/test/test_storesink.py --- a/rpython/translator/backendopt/test/test_storesink.py +++ b/rpython/translator/backendopt/test/test_storesink.py @@ -7,8 +7,6 @@ from rpython.conftest import option class TestStoreSink(object): - type_system = 'lltype' - def translate(self, func, argtypes): t = TranslationContext() t.buildannotator().build_types(func, argtypes) diff --git a/rpython/translator/backendopt/test/test_writeanalyze.py b/rpython/translator/backendopt/test/test_writeanalyze.py --- a/rpython/translator/backendopt/test/test_writeanalyze.py +++ b/rpython/translator/backendopt/test/test_writeanalyze.py @@ -7,8 +7,6 @@ class BaseTest(object): - - type_system = 'lltype' Analyzer = WriteAnalyzer def translate(self, func, sig): diff --git a/rpython/translator/test/test_exceptiontransform.py b/rpython/translator/test/test_exceptiontransform.py --- a/rpython/translator/test/test_exceptiontransform.py +++ b/rpython/translator/test/test_exceptiontransform.py @@ -27,8 +27,6 @@ return interp.eval_graph(graph, values) class TestExceptionTransform: - type_system = 'lltype' - def compile(self, fn, inputargs): from rpython.translator.c.test.test_genc import compile return compile(fn, inputargs) @@ -239,7 +237,7 @@ etrafo.create_exception_handling(g) ops = dict.fromkeys([o.opname for b, o in g.iterblockops()]) assert 'zero_gc_pointers_inside' in ops - + def test_llexternal(self): from rpython.rtyper.lltypesystem.rffi import llexternal from rpython.rtyper.lltypesystem import lltype From noreply at buildbot.pypy.org Sat Mar 28 20:07:31 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 28 Mar 2015 20:07:31 +0100 (CET) Subject: [pypy-commit] pypy default: This place, at least, needs a @rgc.no_collect to make sure none of the Message-ID: <20150328190731.C68181C054A@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76612:994b78f6c6eb Date: 2015-03-28 20:07 +0100 http://bitbucket.org/pypy/pypy/changeset/994b78f6c6eb/ Log: This place, at least, needs a @rgc.no_collect to make sure none of the custom tracers from the program being translated can actually collect diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -521,6 +521,7 @@ custom_trace_funcs_unrolled = unrolling_iterable( [(self.get_type_id(TP), func) for TP, func in custom_trace_funcs]) + @rgc.no_collect @specialize.arg(2) def custom_trace_dispatcher(obj, typeid, callback, arg): for type_id_exp, func in custom_trace_funcs_unrolled: From noreply at buildbot.pypy.org Sat Mar 28 21:29:57 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Sat, 28 Mar 2015 21:29:57 +0100 (CET) Subject: [pypy-commit] pypy default: Simplify Desc.consider_call_site() signature Message-ID: <20150328202957.EBCDF1C054A@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r76613:22b3272ea735 Date: 2015-03-28 20:29 +0000 http://bitbucket.org/pypy/pypy/changeset/22b3272ea735/ Log: Simplify Desc.consider_call_site() signature diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py --- a/rpython/annotator/bookkeeper.py +++ b/rpython/annotator/bookkeeper.py @@ -169,9 +169,7 @@ def consider_call_site_for_pbc(self, s_callable, args, s_result, call_op): descs = list(s_callable.descriptions) - family = descs[0].getcallfamily() - s_callable.getKind().consider_call_site(self, family, descs, args, - s_result, call_op) + s_callable.getKind().consider_call_site(descs, args, s_result, call_op) def getuniqueclassdef(self, cls): """Get the ClassDef associated with the given user cls. diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py --- a/rpython/annotator/description.py +++ b/rpython/annotator/description.py @@ -329,9 +329,10 @@ name) @staticmethod - def consider_call_site(bookkeeper, family, descs, args, s_result, op): + def consider_call_site(descs, args, s_result, op): shape = rawshape(args) row = FunctionDesc.row_to_consider(descs, args, op) + family = descs[0].getcallfamily() family.calltable_add_row(shape, row) @staticmethod @@ -760,7 +761,7 @@ return s_result # common case @staticmethod - def consider_call_site(bookkeeper, family, descs, args, s_result, op): + def consider_call_site(descs, args, s_result, op): from rpython.annotator.model import SomeInstance, SomePBC, s_None if len(descs) == 1: # call to a single class, look at the result annotation @@ -795,17 +796,14 @@ "unexpected dynamic __init__?") initfuncdesc, = s_init.descriptions if isinstance(initfuncdesc, FunctionDesc): - initmethdesc = bookkeeper.getmethoddesc(initfuncdesc, - classdef, - classdef, - '__init__') + from rpython.annotator.bookkeeper import getbookkeeper + initmethdesc = getbookkeeper().getmethoddesc( + initfuncdesc, classdef, classdef, '__init__') initdescs.append(initmethdesc) # register a call to exactly these __init__ methods if initdescs: initdescs[0].mergecallfamilies(*initdescs[1:]) - initfamily = initdescs[0].getcallfamily() - MethodDesc.consider_call_site(bookkeeper, initfamily, initdescs, - args, s_None, op) + MethodDesc.consider_call_site(initdescs, args, s_None, op) def getallbases(self): desc = self @@ -897,10 +895,11 @@ flags) @staticmethod - def consider_call_site(bookkeeper, family, descs, args, s_result, op): + def consider_call_site(descs, args, s_result, op): cnt, keys, star = rawshape(args) shape = cnt + 1, keys, star # account for the extra 'self' row = FunctionDesc.row_to_consider(descs, args, op) + family = descs[0].getcallfamily() family.calltable_add_row(shape, row) def rowkey(self): @@ -1058,10 +1057,11 @@ return self.funcdesc.pycall(schedule, args, s_previous_result, op) @staticmethod - def consider_call_site(bookkeeper, family, descs, args, s_result, op): + def consider_call_site(descs, args, s_result, op): cnt, keys, star = rawshape(args) shape = cnt + 1, keys, star # account for the extra 'self' row = FunctionDesc.row_to_consider(descs, args, op) + family = descs[0].getcallfamily() family.calltable_add_row(shape, row) def rowkey(self): From noreply at buildbot.pypy.org Sat Mar 28 22:39:14 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Sat, 28 Mar 2015 22:39:14 +0100 (CET) Subject: [pypy-commit] pypy default: kill consider_call_site_for_pbc() Message-ID: <20150328213914.7439C1C01D0@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r76614:e420df7b3011 Date: 2015-03-28 21:39 +0000 http://bitbucket.org/pypy/pypy/changeset/e420df7b3011/ Log: kill consider_call_site_for_pbc() diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py --- a/rpython/annotator/bookkeeper.py +++ b/rpython/annotator/bookkeeper.py @@ -111,8 +111,7 @@ for pbc, args_s in self.emulated_pbc_calls.itervalues(): args = simple_args(args_s) - self.consider_call_site_for_pbc(pbc, args, - s_ImpossibleValue, None) + pbc.consider_call_site(args, s_ImpossibleValue, None) self.emulated_pbc_calls = {} finally: self.leave() @@ -163,13 +162,7 @@ if s_result is None: s_result = s_ImpossibleValue args = call_op.build_args(args_s) - self.consider_call_site_for_pbc(s_callable, args, - s_result, call_op) - - def consider_call_site_for_pbc(self, s_callable, args, s_result, - call_op): - descs = list(s_callable.descriptions) - s_callable.getKind().consider_call_site(descs, args, s_result, call_op) + s_callable.consider_call_site(args, s_result, call_op) def getuniqueclassdef(self, cls): """Get the ClassDef associated with the given user cls. diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py --- a/rpython/annotator/model.py +++ b/rpython/annotator/model.py @@ -495,6 +495,10 @@ if len(self.descriptions) > 1: kind.simplify_desc_set(self.descriptions) + def consider_call_site(self, args, s_result, call_op): + descs = list(self.descriptions) + self.getKind().consider_call_site(descs, args, s_result, call_op) + def can_be_none(self): return self.can_be_None @@ -588,7 +592,7 @@ class SomeProperty(SomeObject): - # used for union error only + # used for union error only immutable = True knowntype = type(property) From noreply at buildbot.pypy.org Sun Mar 29 15:32:10 2015 From: noreply at buildbot.pypy.org (mjacob) Date: Sun, 29 Mar 2015 15:32:10 +0200 (CEST) Subject: [pypy-commit] pypy llvm-translation-backend: Fix no_links_to_startblock(). Message-ID: <20150329133210.12BD41C11B5@cobra.cs.uni-duesseldorf.de> Author: Manuel Jacob Branch: llvm-translation-backend Changeset: r76615:7264b1d99a55 Date: 2015-03-28 15:32 +0100 http://bitbucket.org/pypy/pypy/changeset/7264b1d99a55/ Log: Fix no_links_to_startblock(). diff --git a/rpython/translator/unsimplify.py b/rpython/translator/unsimplify.py --- a/rpython/translator/unsimplify.py +++ b/rpython/translator/unsimplify.py @@ -147,7 +147,7 @@ links_to_start_block = True break if links_to_start_block: - insert_empty_startblock(None, graph) + insert_empty_startblock(graph) def call_initial_function(translator, initial_func, annhelper=None): """Before the program starts, call 'initial_func()'.""" From noreply at buildbot.pypy.org Sun Mar 29 15:32:11 2015 From: noreply at buildbot.pypy.org (mjacob) Date: Sun, 29 Mar 2015 15:32:11 +0200 (CEST) Subject: [pypy-commit] pypy llvm-translation-backend: Add errno.h include to threadlocal.c. Message-ID: <20150329133211.3A9E21C11B5@cobra.cs.uni-duesseldorf.de> Author: Manuel Jacob Branch: llvm-translation-backend Changeset: r76616:9960bc589d0f Date: 2015-03-28 16:12 +0100 http://bitbucket.org/pypy/pypy/changeset/9960bc589d0f/ Log: Add errno.h include to threadlocal.c. diff --git a/rpython/translator/c/src/threadlocal.c b/rpython/translator/c/src/threadlocal.c --- a/rpython/translator/c/src/threadlocal.c +++ b/rpython/translator/c/src/threadlocal.c @@ -1,5 +1,6 @@ #include "common_header.h" #include "structdef.h" /* for struct pypy_threadlocal_s */ +#include #include #include #include From noreply at buildbot.pypy.org Sun Mar 29 15:32:12 2015 From: noreply at buildbot.pypy.org (mjacob) Date: Sun, 29 Mar 2015 15:32:12 +0200 (CEST) Subject: [pypy-commit] pypy llvm-translation-backend: Fix compatibility with LLVM 3.6 (and drop support for older versions). Message-ID: <20150329133212.579D81C11B5@cobra.cs.uni-duesseldorf.de> Author: Manuel Jacob Branch: llvm-translation-backend Changeset: r76617:12306f051e3b Date: 2015-03-28 19:04 +0100 http://bitbucket.org/pypy/pypy/changeset/12306f051e3b/ Log: Fix compatibility with LLVM 3.6 (and drop support for older versions). diff --git a/rpython/translator/llvm/genllvm.py b/rpython/translator/llvm/genllvm.py --- a/rpython/translator/llvm/genllvm.py +++ b/rpython/translator/llvm/genllvm.py @@ -1839,7 +1839,7 @@ '}}\n'.format(raise_=get_repr(exctrans.rpyexc_raise_ptr), type=get_repr(self.ovf_err[0]), inst=get_repr(self.ovf_err[1]))) - f.write('!0 = metadata !{ }\n') + f.write('!0 = !{ }\n') def gen_source(self): global database From noreply at buildbot.pypy.org Sun Mar 29 15:32:23 2015 From: noreply at buildbot.pypy.org (mjacob) Date: Sun, 29 Mar 2015 15:32:23 +0200 (CEST) Subject: [pypy-commit] pypy llvm-translation-backend: hg merge default Message-ID: <20150329133223.DB3EB1C11B5@cobra.cs.uni-duesseldorf.de> Author: Manuel Jacob Branch: llvm-translation-backend Changeset: r76618:2caf25f2cf22 Date: 2015-03-29 15:31 +0200 http://bitbucket.org/pypy/pypy/changeset/2caf25f2cf22/ Log: hg merge default diff too long, truncating to 2000 out of 44092 lines diff --git a/.gitignore b/.gitignore --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,10 @@ bin/pypy-c include/*.h +include/numpy/ lib_pypy/ctypes_config_cache/_[^_]*_*.py +libpypy-c.* +pypy-c pypy/_cache pypy/doc/*.html pypy/doc/config/*.html @@ -18,4 +21,5 @@ pypy/translator/c/src/dtoa.o pypy/translator/goal/pypy-c pypy/translator/goal/target*-c -release/ \ No newline at end of file +release/ +rpython/_cache/ diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -3,11 +3,10 @@ d8ac7d23d3ec5f9a0fa1264972f74a010dbfd07f release-1.6 ff4af8f318821f7f5ca998613a60fca09aa137da release-1.7 07e08e9c885ca67d89bcc304e45a32346daea2fa release-2.0-beta-1 -9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm -9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm ab0dd631c22015ed88e583d9fdd4c43eebf0be21 pypy-2.1-beta1-arm 20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0 394146e9bb673514c61f0150ab2013ccf78e8de7 release-2.3 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1 10f1b29a2bd21f837090286174a9ca030b8680b2 release-2.5.0 +9c4588d731b7fe0b08669bd732c2b676cb0a8233 release-2.5.1 diff --git a/lib-python/2.7/CGIHTTPServer.py b/lib-python/2.7/CGIHTTPServer.py --- a/lib-python/2.7/CGIHTTPServer.py +++ b/lib-python/2.7/CGIHTTPServer.py @@ -106,16 +106,16 @@ def run_cgi(self): """Execute a CGI script.""" dir, rest = self.cgi_info - - i = rest.find('/') + path = dir + '/' + rest + i = path.find('/', len(dir)+1) while i >= 0: - nextdir = rest[:i] - nextrest = rest[i+1:] + nextdir = path[:i] + nextrest = path[i+1:] scriptdir = self.translate_path(nextdir) if os.path.isdir(scriptdir): dir, rest = nextdir, nextrest - i = rest.find('/') + i = path.find('/', len(dir)+1) else: break diff --git a/lib-python/2.7/Cookie.py b/lib-python/2.7/Cookie.py --- a/lib-python/2.7/Cookie.py +++ b/lib-python/2.7/Cookie.py @@ -56,7 +56,7 @@ >>> C = Cookie.SmartCookie() [Note: Long-time users of Cookie.py will remember using -Cookie.Cookie() to create an Cookie object. Although deprecated, it +Cookie.Cookie() to create a Cookie object. Although deprecated, it is still supported by the code. See the Backward Compatibility notes for more information.] @@ -426,6 +426,8 @@ "version" : "Version", } + _flags = {'secure', 'httponly'} + def __init__(self): # Set defaults self.key = self.value = self.coded_value = None @@ -529,9 +531,11 @@ _LegalCharsPatt = r"[\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=]" _CookiePattern = re.compile( r"(?x)" # This is a Verbose pattern + r"\s*" # Optional whitespace at start of cookie r"(?P" # Start of group 'key' ""+ _LegalCharsPatt +"+?" # Any word of at least one letter, nongreedy r")" # End of group 'key' + r"(" # Optional group: there may not be a value. r"\s*=\s*" # Equal Sign r"(?P" # Start of group 'val' r'"(?:[^\\"]|\\.)*"' # Any doublequoted string @@ -540,7 +544,9 @@ r"|" # or ""+ _LegalCharsPatt +"*" # Any word or empty string r")" # End of group 'val' - r"\s*;?" # Probably ending in a semi-colon + r")?" # End of optional value group + r"\s*" # Any number of spaces. + r"(\s+|;|$)" # Ending either at space, semicolon, or EOS. ) @@ -585,8 +591,12 @@ def __setitem__(self, key, value): """Dictionary style assignment.""" - rval, cval = self.value_encode(value) - self.__set(key, rval, cval) + if isinstance(value, Morsel): + # allow assignment of constructed Morsels (e.g. for pickling) + dict.__setitem__(self, key, value) + else: + rval, cval = self.value_encode(value) + self.__set(key, rval, cval) # end __setitem__ def output(self, attrs=None, header="Set-Cookie:", sep="\015\012"): @@ -641,7 +651,7 @@ while 0 <= i < n: # Start looking for a cookie - match = patt.search(str, i) + match = patt.match(str, i) if not match: break # No more cookies K,V = match.group("key"), match.group("val") @@ -656,8 +666,12 @@ M[ K[1:] ] = V elif K.lower() in Morsel._reserved: if M: - M[ K ] = _unquote(V) - else: + if V is None: + if K.lower() in Morsel._flags: + M[K] = True + else: + M[K] = _unquote(V) + elif V is not None: rval, cval = self.value_decode(V) self.__set(K, rval, cval) M = self[K] diff --git a/lib-python/2.7/SocketServer.py b/lib-python/2.7/SocketServer.py --- a/lib-python/2.7/SocketServer.py +++ b/lib-python/2.7/SocketServer.py @@ -416,8 +416,12 @@ self.socket = socket.socket(self.address_family, self.socket_type) if bind_and_activate: - self.server_bind() - self.server_activate() + try: + self.server_bind() + self.server_activate() + except: + self.server_close() + raise def server_bind(self): """Called by constructor to bind the socket. diff --git a/lib-python/2.7/_abcoll.py b/lib-python/2.7/_abcoll.py --- a/lib-python/2.7/_abcoll.py +++ b/lib-python/2.7/_abcoll.py @@ -143,7 +143,7 @@ methods except for __contains__, __iter__ and __len__. To override the comparisons (presumably for speed, as the - semantics are fixed), all you have to do is redefine __le__ and + semantics are fixed), redefine __le__ and __ge__, then the other operations will automatically follow suit. """ diff --git a/lib-python/2.7/argparse.py b/lib-python/2.7/argparse.py --- a/lib-python/2.7/argparse.py +++ b/lib-python/2.7/argparse.py @@ -1089,7 +1089,14 @@ # parse all the remaining options into the namespace # store any unrecognized options on the object, so that the top # level parser can decide what to do with them - namespace, arg_strings = parser.parse_known_args(arg_strings, namespace) + + # In case this subparser defines new defaults, we parse them + # in a new namespace object and then update the original + # namespace for the relevant parts. + subnamespace, arg_strings = parser.parse_known_args(arg_strings, None) + for key, value in vars(subnamespace).items(): + setattr(namespace, key, value) + if arg_strings: vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, []) getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings) diff --git a/lib-python/2.7/asynchat.py b/lib-python/2.7/asynchat.py --- a/lib-python/2.7/asynchat.py +++ b/lib-python/2.7/asynchat.py @@ -46,12 +46,17 @@ you - by calling your self.found_terminator() method. """ +import asyncore +import errno import socket -import asyncore from collections import deque from sys import py3kwarning from warnings import filterwarnings, catch_warnings +_BLOCKING_IO_ERRORS = (errno.EAGAIN, errno.EALREADY, errno.EINPROGRESS, + errno.EWOULDBLOCK) + + class async_chat (asyncore.dispatcher): """This is an abstract class. You must derive from this class, and add the two methods collect_incoming_data() and found_terminator()""" @@ -109,6 +114,8 @@ try: data = self.recv (self.ac_in_buffer_size) except socket.error, why: + if why.args[0] in _BLOCKING_IO_ERRORS: + return self.handle_error() return diff --git a/lib-python/2.7/bsddb/test/test_queue.py b/lib-python/2.7/bsddb/test/test_queue.py --- a/lib-python/2.7/bsddb/test/test_queue.py +++ b/lib-python/2.7/bsddb/test/test_queue.py @@ -10,6 +10,7 @@ #---------------------------------------------------------------------- + at unittest.skip("fails on Windows; see issue 22943") class SimpleQueueTestCase(unittest.TestCase): def setUp(self): self.filename = get_new_database_path() diff --git a/lib-python/2.7/cookielib.py b/lib-python/2.7/cookielib.py --- a/lib-python/2.7/cookielib.py +++ b/lib-python/2.7/cookielib.py @@ -1719,12 +1719,12 @@ def __repr__(self): r = [] for cookie in self: r.append(repr(cookie)) - return "<%s[%s]>" % (self.__class__, ", ".join(r)) + return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r)) def __str__(self): r = [] for cookie in self: r.append(str(cookie)) - return "<%s[%s]>" % (self.__class__, ", ".join(r)) + return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r)) # derives from IOError for backwards-compatibility with Python 2.4.0 diff --git a/lib-python/2.7/ctypes/test/test_pointers.py b/lib-python/2.7/ctypes/test/test_pointers.py --- a/lib-python/2.7/ctypes/test/test_pointers.py +++ b/lib-python/2.7/ctypes/test/test_pointers.py @@ -7,6 +7,8 @@ c_long, c_ulong, c_longlong, c_ulonglong, c_double, c_float] python_types = [int, int, int, int, int, long, int, long, long, long, float, float] +LargeNamedType = type('T' * 2 ** 25, (Structure,), {}) +large_string = 'T' * 2 ** 25 class PointersTestCase(unittest.TestCase): @@ -188,5 +190,11 @@ mth = WINFUNCTYPE(None)(42, "name", (), None) self.assertEqual(bool(mth), True) + def test_pointer_type_name(self): + self.assertTrue(POINTER(LargeNamedType)) + + def test_pointer_type_str_name(self): + self.assertTrue(POINTER(large_string)) + if __name__ == '__main__': unittest.main() diff --git a/lib-python/2.7/ctypes/test/test_python_api.py b/lib-python/2.7/ctypes/test/test_python_api.py --- a/lib-python/2.7/ctypes/test/test_python_api.py +++ b/lib-python/2.7/ctypes/test/test_python_api.py @@ -46,8 +46,8 @@ # This test is unreliable, because it is possible that code in # unittest changes the refcount of the '42' integer. So, it # is disabled by default. - @requires("refcount") def test_PyInt_Long(self): + requires("refcount") ref42 = grc(42) pythonapi.PyInt_FromLong.restype = py_object self.assertEqual(pythonapi.PyInt_FromLong(42), 42) diff --git a/lib-python/2.7/ctypes/test/test_win32.py b/lib-python/2.7/ctypes/test/test_win32.py --- a/lib-python/2.7/ctypes/test/test_win32.py +++ b/lib-python/2.7/ctypes/test/test_win32.py @@ -38,8 +38,11 @@ @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') class FunctionCallTestCase(unittest.TestCase): - @requires("SEH") + @unittest.skipUnless('MSC' in sys.version, "SEH only supported by MSC") + @unittest.skipIf(sys.executable.endswith('_d.exe'), + "SEH not enabled in debug builds") def test_SEH(self): + requires("SEH") # Call functions with invalid arguments, and make sure # that access violations are trapped and raise an # exception. @@ -87,9 +90,29 @@ dll = CDLL(_ctypes_test.__file__) - pt = POINT(10, 10) - rect = RECT(0, 0, 20, 20) - self.assertEqual(1, dll.PointInRect(byref(rect), pt)) + pt = POINT(15, 25) + left = c_long.in_dll(dll, 'left') + top = c_long.in_dll(dll, 'top') + right = c_long.in_dll(dll, 'right') + bottom = c_long.in_dll(dll, 'bottom') + rect = RECT(left, top, right, bottom) + PointInRect = dll.PointInRect + PointInRect.argtypes = [POINTER(RECT), POINT] + self.assertEqual(1, PointInRect(byref(rect), pt)) + + ReturnRect = dll.ReturnRect + ReturnRect.argtypes = [c_int, RECT, POINTER(RECT), POINT, RECT, + POINTER(RECT), POINT, RECT] + ReturnRect.restype = RECT + for i in range(4): + ret = ReturnRect(i, rect, pointer(rect), pt, rect, + byref(rect), pt, rect) + # the c function will check and modify ret if something is + # passed in improperly + self.assertEqual(ret.left, left.value) + self.assertEqual(ret.right, right.value) + self.assertEqual(ret.top, top.value) + self.assertEqual(ret.bottom, bottom.value) if __name__ == '__main__': unittest.main() diff --git a/lib-python/2.7/decimal.py b/lib-python/2.7/decimal.py --- a/lib-python/2.7/decimal.py +++ b/lib-python/2.7/decimal.py @@ -136,7 +136,6 @@ __version__ = '1.70' # Highest version of the spec this complies with -import copy as _copy import math as _math import numbers as _numbers @@ -3665,6 +3664,8 @@ if self._is_special: sign = _format_sign(self._sign, spec) body = str(self.copy_abs()) + if spec['type'] == '%': + body += '%' return _format_align(sign, body, spec) # a type of None defaults to 'g' or 'G', depending on context @@ -6033,7 +6034,10 @@ format_dict['decimal_point'] = '.' # record whether return type should be str or unicode - format_dict['unicode'] = isinstance(format_spec, unicode) + try: + format_dict['unicode'] = isinstance(format_spec, unicode) + except NameError: + format_dict['unicode'] = False return format_dict diff --git a/lib-python/2.7/distutils/__init__.py b/lib-python/2.7/distutils/__init__.py --- a/lib-python/2.7/distutils/__init__.py +++ b/lib-python/2.7/distutils/__init__.py @@ -15,5 +15,5 @@ # Updated automatically by the Python release process. # #--start constants-- -__version__ = "2.7.8" +__version__ = "2.7.9" #--end constants-- diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py --- a/lib-python/2.7/distutils/command/build_ext.py +++ b/lib-python/2.7/distutils/command/build_ext.py @@ -245,7 +245,7 @@ # Python's library directory must be appended to library_dirs # See Issues: #1600860, #4366 if (sysconfig.get_config_var('Py_ENABLE_SHARED')): - if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")): + if not sysconfig.python_build: # building third party extensions self.library_dirs.append(sysconfig.get_config_var('LIBDIR')) else: diff --git a/lib-python/2.7/distutils/command/upload.py b/lib-python/2.7/distutils/command/upload.py --- a/lib-python/2.7/distutils/command/upload.py +++ b/lib-python/2.7/distutils/command/upload.py @@ -136,8 +136,8 @@ # Build up the MIME payload for the POST data boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254' - sep_boundary = '\n--' + boundary - end_boundary = sep_boundary + '--' + sep_boundary = '\r\n--' + boundary + end_boundary = sep_boundary + '--\r\n' body = StringIO.StringIO() for key, value in data.items(): # handle multiple entries for the same name @@ -151,14 +151,13 @@ fn = "" body.write(sep_boundary) - body.write('\nContent-Disposition: form-data; name="%s"'%key) + body.write('\r\nContent-Disposition: form-data; name="%s"' % key) body.write(fn) - body.write("\n\n") + body.write("\r\n\r\n") body.write(value) if value and value[-1] == '\r': body.write('\n') # write an extra newline (lurve Macs) body.write(end_boundary) - body.write("\n") body = body.getvalue() self.announce("Submitting %s to %s" % (filename, self.repository), log.INFO) diff --git a/lib-python/2.7/distutils/file_util.py b/lib-python/2.7/distutils/file_util.py --- a/lib-python/2.7/distutils/file_util.py +++ b/lib-python/2.7/distutils/file_util.py @@ -85,7 +85,8 @@ (os.symlink) instead of copying: set it to "hard" or "sym"; if it is None (the default), files are copied. Don't set 'link' on systems that don't support it: 'copy_file()' doesn't check if hard or symbolic - linking is available. + linking is available. If hardlink fails, falls back to + _copy_file_contents(). Under Mac OS, uses the native file copy function in macostools; on other systems, uses '_copy_file_contents()' to copy file contents. @@ -137,24 +138,31 @@ # (Unix only, of course, but that's the caller's responsibility) if link == 'hard': if not (os.path.exists(dst) and os.path.samefile(src, dst)): - os.link(src, dst) + try: + os.link(src, dst) + return (dst, 1) + except OSError: + # If hard linking fails, fall back on copying file + # (some special filesystems don't support hard linking + # even under Unix, see issue #8876). + pass elif link == 'sym': if not (os.path.exists(dst) and os.path.samefile(src, dst)): os.symlink(src, dst) + return (dst, 1) # Otherwise (non-Mac, not linking), copy the file contents and # (optionally) copy the times and mode. - else: - _copy_file_contents(src, dst) - if preserve_mode or preserve_times: - st = os.stat(src) + _copy_file_contents(src, dst) + if preserve_mode or preserve_times: + st = os.stat(src) - # According to David Ascher , utime() should be done - # before chmod() (at least under NT). - if preserve_times: - os.utime(dst, (st[ST_ATIME], st[ST_MTIME])) - if preserve_mode: - os.chmod(dst, S_IMODE(st[ST_MODE])) + # According to David Ascher , utime() should be done + # before chmod() (at least under NT). + if preserve_times: + os.utime(dst, (st[ST_ATIME], st[ST_MTIME])) + if preserve_mode: + os.chmod(dst, S_IMODE(st[ST_MODE])) return (dst, 1) diff --git a/lib-python/2.7/distutils/sysconfig_cpython.py b/lib-python/2.7/distutils/sysconfig_cpython.py --- a/lib-python/2.7/distutils/sysconfig_cpython.py +++ b/lib-python/2.7/distutils/sysconfig_cpython.py @@ -165,7 +165,8 @@ # version and build tools may not support the same set # of CPU architectures for universal builds. global _config_vars - if not _config_vars.get('CUSTOMIZED_OSX_COMPILER', ''): + # Use get_config_var() to ensure _config_vars is initialized. + if not get_config_var('CUSTOMIZED_OSX_COMPILER'): import _osx_support _osx_support.customize_compiler(_config_vars) _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True' diff --git a/lib-python/2.7/distutils/tests/test_bdist_rpm.py b/lib-python/2.7/distutils/tests/test_bdist_rpm.py --- a/lib-python/2.7/distutils/tests/test_bdist_rpm.py +++ b/lib-python/2.7/distutils/tests/test_bdist_rpm.py @@ -25,6 +25,7 @@ """ class BuildRpmTestCase(support.TempdirManager, + support.EnvironGuard, support.LoggingSilencer, unittest.TestCase): @@ -50,6 +51,7 @@ def test_quiet(self): # let's create a package tmp_dir = self.mkdtemp() + os.environ['HOME'] = tmp_dir # to confine dir '.rpmdb' creation pkg_dir = os.path.join(tmp_dir, 'foo') os.mkdir(pkg_dir) self.write_file((pkg_dir, 'setup.py'), SETUP_PY) @@ -92,6 +94,7 @@ def test_no_optimize_flag(self): # let's create a package that brakes bdist_rpm tmp_dir = self.mkdtemp() + os.environ['HOME'] = tmp_dir # to confine dir '.rpmdb' creation pkg_dir = os.path.join(tmp_dir, 'foo') os.mkdir(pkg_dir) self.write_file((pkg_dir, 'setup.py'), SETUP_PY) diff --git a/lib-python/2.7/distutils/tests/test_dist.py b/lib-python/2.7/distutils/tests/test_dist.py --- a/lib-python/2.7/distutils/tests/test_dist.py +++ b/lib-python/2.7/distutils/tests/test_dist.py @@ -11,7 +11,7 @@ from distutils.dist import Distribution, fix_help_options from distutils.cmd import Command import distutils.dist -from test.test_support import TESTFN, captured_stdout, run_unittest +from test.test_support import TESTFN, captured_stdout, run_unittest, unlink from distutils.tests import support @@ -64,6 +64,7 @@ with open(TESTFN, "w") as f: f.write("[global]\n") f.write("command_packages = foo.bar, splat") + self.addCleanup(unlink, TESTFN) files = [TESTFN] sys.argv.append("build") diff --git a/lib-python/2.7/distutils/tests/test_file_util.py b/lib-python/2.7/distutils/tests/test_file_util.py --- a/lib-python/2.7/distutils/tests/test_file_util.py +++ b/lib-python/2.7/distutils/tests/test_file_util.py @@ -8,6 +8,11 @@ from distutils.tests import support from test.test_support import run_unittest + +requires_os_link = unittest.skipUnless(hasattr(os, "link"), + "test requires os.link()") + + class FileUtilTestCase(support.TempdirManager, unittest.TestCase): def _log(self, msg, *args): @@ -74,6 +79,44 @@ copy_file(foo, dst_dir) self.assertTrue(os.path.exists(os.path.join(dst_dir, 'foo'))) + @requires_os_link + def test_copy_file_hard_link(self): + with open(self.source, 'w') as f: + f.write('some content') + st = os.stat(self.source) + copy_file(self.source, self.target, link='hard') + st2 = os.stat(self.source) + st3 = os.stat(self.target) + self.assertTrue(os.path.samestat(st, st2), (st, st2)) + self.assertTrue(os.path.samestat(st2, st3), (st2, st3)) + with open(self.source, 'r') as f: + self.assertEqual(f.read(), 'some content') + + @requires_os_link + def test_copy_file_hard_link_failure(self): + # If hard linking fails, copy_file() falls back on copying file + # (some special filesystems don't support hard linking even under + # Unix, see issue #8876). + with open(self.source, 'w') as f: + f.write('some content') + st = os.stat(self.source) + def _os_link(*args): + raise OSError(0, "linking unsupported") + old_link = os.link + os.link = _os_link + try: + copy_file(self.source, self.target, link='hard') + finally: + os.link = old_link + st2 = os.stat(self.source) + st3 = os.stat(self.target) + self.assertTrue(os.path.samestat(st, st2), (st, st2)) + self.assertFalse(os.path.samestat(st2, st3), (st2, st3)) + for fn in (self.source, self.target): + with open(fn, 'r') as f: + self.assertEqual(f.read(), 'some content') + + def test_suite(): return unittest.makeSuite(FileUtilTestCase) diff --git a/lib-python/2.7/distutils/tests/test_sysconfig.py b/lib-python/2.7/distutils/tests/test_sysconfig.py --- a/lib-python/2.7/distutils/tests/test_sysconfig.py +++ b/lib-python/2.7/distutils/tests/test_sysconfig.py @@ -3,6 +3,9 @@ import test import unittest import shutil +import subprocess +import sys +import textwrap from distutils import sysconfig from distutils.tests import support @@ -99,6 +102,24 @@ self.assertEqual(global_sysconfig.get_config_var('LDSHARED'), sysconfig.get_config_var('LDSHARED')) self.assertEqual(global_sysconfig.get_config_var('CC'), sysconfig.get_config_var('CC')) + def test_customize_compiler_before_get_config_vars(self): + # Issue #21923: test that a Distribution compiler + # instance can be called without an explicit call to + # get_config_vars(). + with open(TESTFN, 'w') as f: + f.writelines(textwrap.dedent('''\ + from distutils.core import Distribution + config = Distribution().get_command_obj('config') + # try_compile may pass or it may fail if no compiler + # is found but it should not raise an exception. + rc = config.try_compile('int x;') + ''')) + p = subprocess.Popen([str(sys.executable), TESTFN], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True) + outs, errs = p.communicate() + self.assertEqual(0, p.returncode, "Subprocess failed: " + outs) def test_suite(): diff --git a/lib-python/2.7/distutils/tests/test_upload.py b/lib-python/2.7/distutils/tests/test_upload.py --- a/lib-python/2.7/distutils/tests/test_upload.py +++ b/lib-python/2.7/distutils/tests/test_upload.py @@ -119,7 +119,7 @@ # what did we send ? self.assertIn('dédé', self.last_open.req.data) headers = dict(self.last_open.req.headers) - self.assertEqual(headers['Content-length'], '2085') + self.assertEqual(headers['Content-length'], '2159') self.assertTrue(headers['Content-type'].startswith('multipart/form-data')) self.assertEqual(self.last_open.req.get_method(), 'POST') self.assertEqual(self.last_open.req.get_full_url(), diff --git a/lib-python/2.7/doctest.py b/lib-python/2.7/doctest.py --- a/lib-python/2.7/doctest.py +++ b/lib-python/2.7/doctest.py @@ -216,7 +216,7 @@ # get_data() opens files as 'rb', so one must do the equivalent # conversion as universal newlines would do. return file_contents.replace(os.linesep, '\n'), filename - with open(filename) as f: + with open(filename, 'U') as f: return f.read(), filename # Use sys.stdout encoding for ouput. diff --git a/lib-python/2.7/email/feedparser.py b/lib-python/2.7/email/feedparser.py --- a/lib-python/2.7/email/feedparser.py +++ b/lib-python/2.7/email/feedparser.py @@ -49,8 +49,8 @@ simple abstraction -- it parses until EOF closes the current message. """ def __init__(self): - # The last partial line pushed into this object. - self._partial = '' + # Chunks of the last partial line pushed into this object. + self._partial = [] # The list of full, pushed lines, in reverse order self._lines = [] # The stack of false-EOF checking predicates. @@ -66,8 +66,8 @@ def close(self): # Don't forget any trailing partial line. - self._lines.append(self._partial) - self._partial = '' + self.pushlines(''.join(self._partial).splitlines(True)) + self._partial = [] self._closed = True def readline(self): @@ -95,8 +95,29 @@ def push(self, data): """Push some new data into this object.""" - # Handle any previous leftovers - data, self._partial = self._partial + data, '' + # Crack into lines, but preserve the linesep characters on the end of each + parts = data.splitlines(True) + + if not parts or not parts[0].endswith(('\n', '\r')): + # No new complete lines, so just accumulate partials + self._partial += parts + return + + if self._partial: + # If there are previous leftovers, complete them now + self._partial.append(parts[0]) + parts[0:1] = ''.join(self._partial).splitlines(True) + del self._partial[:] + + # If the last element of the list does not end in a newline, then treat + # it as a partial line. We only check for '\n' here because a line + # ending with '\r' might be a line that was split in the middle of a + # '\r\n' sequence (see bugs 1555570 and 1721862). + if not parts[-1].endswith('\n'): + self._partial = [parts.pop()] + self.pushlines(parts) + + def pushlines(self, lines): # Crack into lines, but preserve the newlines on the end of each parts = NLCRE_crack.split(data) # The *ahem* interesting behaviour of re.split when supplied grouping diff --git a/lib-python/2.7/email/mime/nonmultipart.py b/lib-python/2.7/email/mime/nonmultipart.py --- a/lib-python/2.7/email/mime/nonmultipart.py +++ b/lib-python/2.7/email/mime/nonmultipart.py @@ -12,7 +12,7 @@ class MIMENonMultipart(MIMEBase): - """Base class for MIME multipart/* type messages.""" + """Base class for MIME non-multipart type messages.""" def attach(self, payload): # The public API prohibits attaching multiple subparts to MIMEBase diff --git a/lib-python/2.7/email/test/test_email.py b/lib-python/2.7/email/test/test_email.py --- a/lib-python/2.7/email/test/test_email.py +++ b/lib-python/2.7/email/test/test_email.py @@ -11,6 +11,7 @@ import warnings import textwrap from cStringIO import StringIO +from random import choice import email @@ -2578,16 +2579,64 @@ bsf.push(il) nt += n n1 = 0 - while True: - ol = bsf.readline() - if ol == NeedMoreData: - break + for ol in iter(bsf.readline, NeedMoreData): om.append(ol) n1 += 1 self.assertEqual(n, n1) self.assertEqual(len(om), nt) self.assertEqual(''.join([il for il, n in imt]), ''.join(om)) + def test_push_random(self): + from email.feedparser import BufferedSubFile, NeedMoreData + + n = 10000 + chunksize = 5 + chars = 'abcd \t\r\n' + + s = ''.join(choice(chars) for i in range(n)) + '\n' + target = s.splitlines(True) + + bsf = BufferedSubFile() + lines = [] + for i in range(0, len(s), chunksize): + chunk = s[i:i+chunksize] + bsf.push(chunk) + lines.extend(iter(bsf.readline, NeedMoreData)) + self.assertEqual(lines, target) + + +class TestFeedParsers(TestEmailBase): + + def parse(self, chunks): + from email.feedparser import FeedParser + feedparser = FeedParser() + for chunk in chunks: + feedparser.feed(chunk) + return feedparser.close() + + def test_newlines(self): + m = self.parse(['a:\nb:\rc:\r\nd:\n']) + self.assertEqual(m.keys(), ['a', 'b', 'c', 'd']) + m = self.parse(['a:\nb:\rc:\r\nd:']) + self.assertEqual(m.keys(), ['a', 'b', 'c', 'd']) + m = self.parse(['a:\rb', 'c:\n']) + self.assertEqual(m.keys(), ['a', 'bc']) + m = self.parse(['a:\r', 'b:\n']) + self.assertEqual(m.keys(), ['a', 'b']) + m = self.parse(['a:\r', '\nb:\n']) + self.assertEqual(m.keys(), ['a', 'b']) + + def test_long_lines(self): + # Expected peak memory use on 32-bit platform: 4*N*M bytes. + M, N = 1000, 20000 + m = self.parse(['a:b\n\n'] + ['x'*M] * N) + self.assertEqual(m.items(), [('a', 'b')]) + self.assertEqual(m.get_payload(), 'x'*M*N) + m = self.parse(['a:b\r\r'] + ['x'*M] * N) + self.assertEqual(m.items(), [('a', 'b')]) + self.assertEqual(m.get_payload(), 'x'*M*N) + m = self.parse(['a:\r', 'b: '] + ['x'*M] * N) + self.assertEqual(m.items(), [('a', ''), ('b', 'x'*M*N)]) class TestParsers(TestEmailBase): @@ -3180,7 +3229,6 @@ self.assertEqual(res, '=?iso-8859-2?q?abc?=') self.assertIsInstance(res, str) - # Test RFC 2231 header parameters (en/de)coding class TestRFC2231(TestEmailBase): def test_get_param(self): diff --git a/lib-python/2.7/ensurepip/__init__.py b/lib-python/2.7/ensurepip/__init__.py new file mode 100644 --- /dev/null +++ b/lib-python/2.7/ensurepip/__init__.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python2 +from __future__ import print_function + +import os +import os.path +import pkgutil +import shutil +import sys +import tempfile + + +__all__ = ["version", "bootstrap"] + + +_SETUPTOOLS_VERSION = "7.0" + +_PIP_VERSION = "1.5.6" + +# pip currently requires ssl support, so we try to provide a nicer +# error message when that is missing (http://bugs.python.org/issue19744) +_MISSING_SSL_MESSAGE = ("pip {} requires SSL/TLS".format(_PIP_VERSION)) +try: + import ssl +except ImportError: + ssl = None + + def _require_ssl_for_pip(): + raise RuntimeError(_MISSING_SSL_MESSAGE) +else: + def _require_ssl_for_pip(): + pass + +_PROJECTS = [ + ("setuptools", _SETUPTOOLS_VERSION), + ("pip", _PIP_VERSION), +] + + +def _run_pip(args, additional_paths=None): + # Add our bundled software to the sys.path so we can import it + if additional_paths is not None: + sys.path = additional_paths + sys.path + + # Install the bundled software + import pip + pip.main(args) + + +def version(): + """ + Returns a string specifying the bundled version of pip. + """ + return _PIP_VERSION + + +def _disable_pip_configuration_settings(): + # We deliberately ignore all pip environment variables + # when invoking pip + # See http://bugs.python.org/issue19734 for details + keys_to_remove = [k for k in os.environ if k.startswith("PIP_")] + for k in keys_to_remove: + del os.environ[k] + # We also ignore the settings in the default pip configuration file + # See http://bugs.python.org/issue20053 for details + os.environ['PIP_CONFIG_FILE'] = os.devnull + + +def bootstrap(root=None, upgrade=False, user=False, + altinstall=False, default_pip=True, + verbosity=0): + """ + Bootstrap pip into the current Python installation (or the given root + directory). + + Note that calling this function will alter both sys.path and os.environ. + """ + if altinstall and default_pip: + raise ValueError("Cannot use altinstall and default_pip together") + + _require_ssl_for_pip() + _disable_pip_configuration_settings() + + # By default, installing pip and setuptools installs all of the + # following scripts (X.Y == running Python version): + # + # pip, pipX, pipX.Y, easy_install, easy_install-X.Y + # + # pip 1.5+ allows ensurepip to request that some of those be left out + if altinstall: + # omit pip, pipX and easy_install + os.environ["ENSUREPIP_OPTIONS"] = "altinstall" + elif not default_pip: + # omit pip and easy_install + os.environ["ENSUREPIP_OPTIONS"] = "install" + + tmpdir = tempfile.mkdtemp() + try: + # Put our bundled wheels into a temporary directory and construct the + # additional paths that need added to sys.path + additional_paths = [] + for project, version in _PROJECTS: + wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version) + whl = pkgutil.get_data( + "ensurepip", + "_bundled/{}".format(wheel_name), + ) + with open(os.path.join(tmpdir, wheel_name), "wb") as fp: + fp.write(whl) + + additional_paths.append(os.path.join(tmpdir, wheel_name)) + + # Construct the arguments to be passed to the pip command + args = ["install", "--no-index", "--find-links", tmpdir] + if root: + args += ["--root", root] + if upgrade: + args += ["--upgrade"] + if user: + args += ["--user"] + if verbosity: + args += ["-" + "v" * verbosity] + + _run_pip(args + [p[0] for p in _PROJECTS], additional_paths) + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + +def _uninstall_helper(verbosity=0): + """Helper to support a clean default uninstall process on Windows + + Note that calling this function may alter os.environ. + """ + # Nothing to do if pip was never installed, or has been removed + try: + import pip + except ImportError: + return + + # If the pip version doesn't match the bundled one, leave it alone + if pip.__version__ != _PIP_VERSION: + msg = ("ensurepip will only uninstall a matching version " + "({!r} installed, {!r} bundled)") + print(msg.format(pip.__version__, _PIP_VERSION), file=sys.stderr) + return + + _require_ssl_for_pip() + _disable_pip_configuration_settings() + + # Construct the arguments to be passed to the pip command + args = ["uninstall", "-y"] + if verbosity: + args += ["-" + "v" * verbosity] + + _run_pip(args + [p[0] for p in reversed(_PROJECTS)]) + + +def _main(argv=None): + if ssl is None: + print("Ignoring ensurepip failure: {}".format(_MISSING_SSL_MESSAGE), + file=sys.stderr) + return + + import argparse + parser = argparse.ArgumentParser(prog="python -m ensurepip") + parser.add_argument( + "--version", + action="version", + version="pip {}".format(version()), + help="Show the version of pip that is bundled with this Python.", + ) + parser.add_argument( + "-v", "--verbose", + action="count", + default=0, + dest="verbosity", + help=("Give more output. Option is additive, and can be used up to 3 " + "times."), + ) + parser.add_argument( + "-U", "--upgrade", + action="store_true", + default=False, + help="Upgrade pip and dependencies, even if already installed.", + ) + parser.add_argument( + "--user", + action="store_true", + default=False, + help="Install using the user scheme.", + ) + parser.add_argument( + "--root", + default=None, + help="Install everything relative to this alternate root directory.", + ) + parser.add_argument( + "--altinstall", + action="store_true", + default=False, + help=("Make an alternate install, installing only the X.Y versioned" + "scripts (Default: pipX, pipX.Y, easy_install-X.Y)"), + ) + parser.add_argument( + "--default-pip", + action="store_true", + default=True, + dest="default_pip", + help=argparse.SUPPRESS, + ) + parser.add_argument( + "--no-default-pip", + action="store_false", + dest="default_pip", + help=("Make a non default install, installing only the X and X.Y " + "versioned scripts."), + ) + + args = parser.parse_args(argv) + + bootstrap( + root=args.root, + upgrade=args.upgrade, + user=args.user, + verbosity=args.verbosity, + altinstall=args.altinstall, + default_pip=args.default_pip, + ) diff --git a/lib-python/2.7/ensurepip/__main__.py b/lib-python/2.7/ensurepip/__main__.py new file mode 100644 --- /dev/null +++ b/lib-python/2.7/ensurepip/__main__.py @@ -0,0 +1,4 @@ +import ensurepip + +if __name__ == "__main__": + ensurepip._main() diff --git a/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..097ab43430d4c1302b0be353a8c16407c370693b GIT binary patch [cut] diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..fa1d1054da1dab98f8906555d31a9fda271b3a85 GIT binary patch [cut] diff --git a/lib-python/2.7/ensurepip/_uninstall.py b/lib-python/2.7/ensurepip/_uninstall.py new file mode 100644 --- /dev/null +++ b/lib-python/2.7/ensurepip/_uninstall.py @@ -0,0 +1,30 @@ +"""Basic pip uninstallation support, helper for the Windows uninstaller""" + +import argparse +import ensurepip + + +def _main(argv=None): + parser = argparse.ArgumentParser(prog="python -m ensurepip._uninstall") + parser.add_argument( + "--version", + action="version", + version="pip {}".format(ensurepip.version()), + help="Show the version of pip this will attempt to uninstall.", + ) + parser.add_argument( + "-v", "--verbose", + action="count", + default=0, + dest="verbosity", + help=("Give more output. Option is additive, and can be used up to 3 " + "times."), + ) + + args = parser.parse_args(argv) + + ensurepip._uninstall_helper(verbosity=args.verbosity) + + +if __name__ == "__main__": + _main() diff --git a/lib-python/2.7/glob.py b/lib-python/2.7/glob.py --- a/lib-python/2.7/glob.py +++ b/lib-python/2.7/glob.py @@ -35,11 +35,16 @@ patterns. """ + dirname, basename = os.path.split(pathname) if not has_magic(pathname): - if os.path.lexists(pathname): - yield pathname + if basename: + if os.path.lexists(pathname): + yield pathname + else: + # Patterns ending with a slash should match only directories + if os.path.isdir(dirname): + yield pathname return - dirname, basename = os.path.split(pathname) if not dirname: for name in glob1(os.curdir, basename): yield name diff --git a/lib-python/2.7/gzip.py b/lib-python/2.7/gzip.py --- a/lib-python/2.7/gzip.py +++ b/lib-python/2.7/gzip.py @@ -164,9 +164,16 @@ def _write_gzip_header(self): self.fileobj.write('\037\213') # magic header self.fileobj.write('\010') # compression method - fname = os.path.basename(self.name) - if fname.endswith(".gz"): - fname = fname[:-3] + try: + # RFC 1952 requires the FNAME field to be Latin-1. Do not + # include filenames that cannot be represented that way. + fname = os.path.basename(self.name) + if not isinstance(fname, str): + fname = fname.encode('latin-1') + if fname.endswith('.gz'): + fname = fname[:-3] + except UnicodeEncodeError: + fname = '' flags = 0 if fname: flags = FNAME diff --git a/lib-python/2.7/hashlib.py b/lib-python/2.7/hashlib.py --- a/lib-python/2.7/hashlib.py +++ b/lib-python/2.7/hashlib.py @@ -15,8 +15,9 @@ md5(), sha1(), sha224(), sha256(), sha384(), and sha512() -More algorithms may be available on your platform but the above are -guaranteed to exist. +More algorithms may be available on your platform but the above are guaranteed +to exist. See the algorithms_guaranteed and algorithms_available attributes +to find out what algorithm names can be passed to new(). NOTE: If you want the adler32 or crc32 hash functions they are available in the zlib module. @@ -58,9 +59,14 @@ # always available algorithm is added. __always_supported = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512') +algorithms_guaranteed = set(__always_supported) +algorithms_available = set(__always_supported) + algorithms = __always_supported -__all__ = __always_supported + ('new', 'algorithms', 'pbkdf2_hmac') +__all__ = __always_supported + ('new', 'algorithms_guaranteed', + 'algorithms_available', 'algorithms', + 'pbkdf2_hmac') def __get_builtin_constructor(name): @@ -128,6 +134,8 @@ import _hashlib new = __hash_new __get_hash = __get_openssl_constructor + algorithms_available = algorithms_available.union( + _hashlib.openssl_md_meth_names) except ImportError: new = __py_new __get_hash = __get_builtin_constructor diff --git a/lib-python/2.7/httplib.py b/lib-python/2.7/httplib.py --- a/lib-python/2.7/httplib.py +++ b/lib-python/2.7/httplib.py @@ -215,6 +215,10 @@ # maximal line length when calling readline(). _MAXLINE = 65536 +# maximum amount of headers accepted +_MAXHEADERS = 100 + + class HTTPMessage(mimetools.Message): def addheader(self, key, value): @@ -271,6 +275,8 @@ elif self.seekable: tell = self.fp.tell while True: + if len(hlist) > _MAXHEADERS: + raise HTTPException("got more than %d headers" % _MAXHEADERS) if tell: try: startofline = tell() @@ -1185,21 +1191,29 @@ def __init__(self, host, port=None, key_file=None, cert_file=None, strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - source_address=None): + source_address=None, context=None): HTTPConnection.__init__(self, host, port, strict, timeout, source_address) self.key_file = key_file self.cert_file = cert_file + if context is None: + context = ssl._create_default_https_context() + if key_file or cert_file: + context.load_cert_chain(cert_file, key_file) + self._context = context def connect(self): "Connect to a host on a given (SSL) port." - sock = self._create_connection((self.host, self.port), - self.timeout, self.source_address) + HTTPConnection.connect(self) + if self._tunnel_host: - self.sock = sock - self._tunnel() - self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file) + server_hostname = self._tunnel_host + else: + server_hostname = self.host + + self.sock = self._context.wrap_socket(self.sock, + server_hostname=server_hostname) __all__.append("HTTPSConnection") @@ -1214,14 +1228,15 @@ _connection_class = HTTPSConnection def __init__(self, host='', port=None, key_file=None, cert_file=None, - strict=None): + strict=None, context=None): # provide a default host, pass the X509 cert info # urf. compensate for bad input. if port == 0: port = None self._setup(self._connection_class(host, port, key_file, - cert_file, strict)) + cert_file, strict, + context=context)) # we never actually use these for anything, but we keep them # here for compatibility with post-1.5.2 CVS. diff --git a/lib-python/2.7/idlelib/Bindings.py b/lib-python/2.7/idlelib/Bindings.py --- a/lib-python/2.7/idlelib/Bindings.py +++ b/lib-python/2.7/idlelib/Bindings.py @@ -75,7 +75,8 @@ ('!_Auto-open Stack Viewer', '<>'), ]), ('options', [ - ('_Configure IDLE...', '<>'), + ('Configure _IDLE', '<>'), + ('Configure _Extensions', '<>'), None, ]), ('help', [ diff --git a/lib-python/2.7/idlelib/CallTipWindow.py b/lib-python/2.7/idlelib/CallTipWindow.py --- a/lib-python/2.7/idlelib/CallTipWindow.py +++ b/lib-python/2.7/idlelib/CallTipWindow.py @@ -2,9 +2,8 @@ After ToolTip.py, which uses ideas gleaned from PySol Used by the CallTips IDLE extension. - """ -from Tkinter import * +from Tkinter import Toplevel, Label, LEFT, SOLID, TclError HIDE_VIRTUAL_EVENT_NAME = "<>" HIDE_SEQUENCES = ("", "") @@ -133,35 +132,28 @@ return bool(self.tipwindow) -def _calltip_window(parent): - root = Tk() - root.title("Test calltips") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) +def _calltip_window(parent): # htest # + from Tkinter import Toplevel, Text, LEFT, BOTH - class MyEditWin: # comparenceptually an editor_window - def __init__(self): - text = self.text = Text(root) - text.pack(side=LEFT, fill=BOTH, expand=1) - text.insert("insert", "string.split") - root.update() - self.calltip = CallTip(text) + top = Toplevel(parent) + top.title("Test calltips") + top.geometry("200x100+%d+%d" % (parent.winfo_rootx() + 200, + parent.winfo_rooty() + 150)) + text = Text(top) + text.pack(side=LEFT, fill=BOTH, expand=1) + text.insert("insert", "string.split") + top.update() + calltip = CallTip(text) - text.event_add("<>", "(") - text.event_add("<>", ")") - text.bind("<>", self.calltip_show) - text.bind("<>", self.calltip_hide) - - text.focus_set() - root.mainloop() - - def calltip_show(self, event): - self.calltip.showtip("Hello world", "insert", "end") - - def calltip_hide(self, event): - self.calltip.hidetip() - - editwin = MyEditWin() + def calltip_show(event): + calltip.showtip("(s=Hello world)", "insert", "end") + def calltip_hide(event): + calltip.hidetip() + text.event_add("<>", "(") + text.event_add("<>", ")") + text.bind("<>", calltip_show) + text.bind("<>", calltip_hide) + text.focus_set() if __name__=='__main__': from idlelib.idle_test.htest import run diff --git a/lib-python/2.7/idlelib/ClassBrowser.py b/lib-python/2.7/idlelib/ClassBrowser.py --- a/lib-python/2.7/idlelib/ClassBrowser.py +++ b/lib-python/2.7/idlelib/ClassBrowser.py @@ -19,6 +19,9 @@ from idlelib.TreeWidget import TreeNode, TreeItem, ScrolledCanvas from idlelib.configHandler import idleConf +file_open = None # Method...Item and Class...Item use this. +# Normally PyShell.flist.open, but there is no PyShell.flist for htest. + class ClassBrowser: def __init__(self, flist, name, path, _htest=False): @@ -27,6 +30,9 @@ """ _htest - bool, change box when location running htest. """ + global file_open + if not _htest: + file_open = PyShell.flist.open self.name = name self.file = os.path.join(path[0], self.name + ".py") self._htest = _htest @@ -101,7 +107,7 @@ return [] try: dict = pyclbr.readmodule_ex(name, [dir] + sys.path) - except ImportError, msg: + except ImportError: return [] items = [] self.classes = {} @@ -170,7 +176,7 @@ def OnDoubleClick(self): if not os.path.exists(self.file): return - edit = PyShell.flist.open(self.file) + edit = file_open(self.file) if hasattr(self.cl, 'lineno'): lineno = self.cl.lineno edit.gotoline(lineno) @@ -206,7 +212,7 @@ def OnDoubleClick(self): if not os.path.exists(self.file): return - edit = PyShell.flist.open(self.file) + edit = file_open(self.file) edit.gotoline(self.cl.methods[self.name]) def _class_browser(parent): #Wrapper for htest @@ -221,8 +227,9 @@ dir, file = os.path.split(file) name = os.path.splitext(file)[0] flist = PyShell.PyShellFileList(parent) + global file_open + file_open = flist.open ClassBrowser(flist, name, [dir], _htest=True) - parent.mainloop() if __name__ == "__main__": from idlelib.idle_test.htest import run diff --git a/lib-python/2.7/idlelib/ColorDelegator.py b/lib-python/2.7/idlelib/ColorDelegator.py --- a/lib-python/2.7/idlelib/ColorDelegator.py +++ b/lib-python/2.7/idlelib/ColorDelegator.py @@ -2,7 +2,6 @@ import re import keyword import __builtin__ -from Tkinter import * from idlelib.Delegator import Delegator from idlelib.configHandler import idleConf @@ -34,7 +33,6 @@ prog = re.compile(make_pat(), re.S) idprog = re.compile(r"\s+(\w+)", re.S) -asprog = re.compile(r".*?\b(as)\b") class ColorDelegator(Delegator): @@ -42,7 +40,6 @@ Delegator.__init__(self) self.prog = prog self.idprog = idprog - self.asprog = asprog self.LoadTagDefs() def setdelegate(self, delegate): @@ -74,7 +71,6 @@ "DEFINITION": idleConf.GetHighlight(theme, "definition"), "SYNC": {'background':None,'foreground':None}, "TODO": {'background':None,'foreground':None}, - "BREAK": idleConf.GetHighlight(theme, "break"), "ERROR": idleConf.GetHighlight(theme, "error"), # The following is used by ReplaceDialog: "hit": idleConf.GetHighlight(theme, "hit"), @@ -216,22 +212,6 @@ self.tag_add("DEFINITION", head + "+%dc" % a, head + "+%dc" % b) - elif value == "import": - # color all the "as" words on same line, except - # if in a comment; cheap approximation to the - # truth - if '#' in chars: - endpos = chars.index('#') - else: - endpos = len(chars) - while True: - m1 = self.asprog.match(chars, b, endpos) - if not m1: - break - a, b = m1.span(1) - self.tag_add("KEYWORD", - head + "+%dc" % a, - head + "+%dc" % b) m = self.prog.search(chars, m.end()) if "SYNC" in self.tag_names(next + "-1c"): head = next @@ -255,20 +235,23 @@ for tag in self.tagdefs.keys(): self.tag_remove(tag, "1.0", "end") -def _color_delegator(parent): +def _color_delegator(parent): # htest # + from Tkinter import Toplevel, Text from idlelib.Percolator import Percolator - root = Tk() - root.title("Test ColorDelegator") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) - source = "if somename: x = 'abc' # comment\nprint" - text = Text(root, background="white") + + top = Toplevel(parent) + top.title("Test ColorDelegator") + top.geometry("200x100+%d+%d" % (parent.winfo_rootx() + 200, + parent.winfo_rooty() + 150)) + source = "if somename: x = 'abc' # comment\nprint\n" + text = Text(top, background="white") + text.pack(expand=1, fill="both") text.insert("insert", source) - text.pack(expand=1, fill="both") + text.focus_set() + p = Percolator(text) d = ColorDelegator() p.insertfilter(d) - root.mainloop() if __name__ == "__main__": from idlelib.idle_test.htest import run diff --git a/lib-python/2.7/idlelib/Debugger.py b/lib-python/2.7/idlelib/Debugger.py --- a/lib-python/2.7/idlelib/Debugger.py +++ b/lib-python/2.7/idlelib/Debugger.py @@ -1,6 +1,5 @@ import os import bdb -import types from Tkinter import * from idlelib.WindowList import ListedToplevel from idlelib.ScrolledList import ScrolledList diff --git a/lib-python/2.7/idlelib/EditorWindow.py b/lib-python/2.7/idlelib/EditorWindow.py --- a/lib-python/2.7/idlelib/EditorWindow.py +++ b/lib-python/2.7/idlelib/EditorWindow.py @@ -1,6 +1,6 @@ import sys import os -from platform import python_version +import platform import re import imp from Tkinter import * @@ -22,6 +22,8 @@ # The default tab setting for a Text widget, in average-width characters. TK_TABWIDTH_DEFAULT = 8 +_py_version = ' (%s)' % platform.python_version() + def _sphinx_version(): "Format sys.version_info to produce the Sphinx version string used to install the chm docs" major, minor, micro, level, serial = sys.version_info @@ -151,7 +153,7 @@ # Safari requires real file:-URLs EditorWindow.help_url = 'file://' + EditorWindow.help_url else: - EditorWindow.help_url = "http://docs.python.org/%d.%d" % sys.version_info[:2] + EditorWindow.help_url = "https://docs.python.org/%d.%d/" % sys.version_info[:2] currentTheme=idleConf.CurrentTheme() self.flist = flist root = root or flist.root @@ -214,6 +216,8 @@ text.bind("<>", self.python_docs) text.bind("<>", self.about_dialog) text.bind("<>", self.config_dialog) + text.bind("<>", + self.config_extensions_dialog) text.bind("<>", self.open_module) text.bind("<>", lambda event: "break") text.bind("<>", self.select_all) @@ -568,6 +572,8 @@ def config_dialog(self, event=None): configDialog.ConfigDialog(self.top,'Settings') + def config_extensions_dialog(self, event=None): + configDialog.ConfigExtensionsDialog(self.top) def help_dialog(self, event=None): if self.root: @@ -691,30 +697,29 @@ return # XXX Ought to insert current file's directory in front of path try: - (f, file, (suffix, mode, type)) = _find_module(name) + (f, file_path, (suffix, mode, mtype)) = _find_module(name) except (NameError, ImportError) as msg: tkMessageBox.showerror("Import error", str(msg), parent=self.text) return - if type != imp.PY_SOURCE: + if mtype != imp.PY_SOURCE: tkMessageBox.showerror("Unsupported type", "%s is not a source module" % name, parent=self.text) return if f: f.close() if self.flist: - self.flist.open(file) + self.flist.open(file_path) else: - self.io.loadfile(file) + self.io.loadfile(file_path) + return file_path def open_class_browser(self, event=None): filename = self.io.filename - if not filename: - tkMessageBox.showerror( - "No filename", - "This buffer has no associated filename", - master=self.text) - self.text.focus_set() - return None + if not (self.__class__.__name__ == 'PyShellEditorWindow' + and filename): + filename = self.open_module() + if filename is None: + return head, tail = os.path.split(filename) base, ext = os.path.splitext(tail) from idlelib import ClassBrowser @@ -779,7 +784,7 @@ self.color = None def ResetColorizer(self): - "Update the colour theme" + "Update the color theme" # Called from self.filename_change_hook and from configDialog.py self._rmcolorizer() self._addcolorizer() @@ -944,7 +949,7 @@ short = self.short_title() long = self.long_title() if short and long: - title = short + " - " + long + title = short + " - " + long + _py_version elif short: title = short elif long: @@ -968,14 +973,13 @@ self.undo.reset_undo() def short_title(self): - pyversion = "Python " + python_version() + ": " filename = self.io.filename if filename: filename = os.path.basename(filename) else: filename = "Untitled" # return unicode string to display non-ASCII chars correctly - return pyversion + self._filename_to_unicode(filename) + return self._filename_to_unicode(filename) def long_title(self): # return unicode string to display non-ASCII chars correctly @@ -1711,7 +1715,8 @@ tk.call('set', 'tcl_nonwordchars', '[^a-zA-Z0-9_]') -def _editor_window(parent): +def _editor_window(parent): # htest # + # error if close master window first - timer event, after script root = parent fixwordbreaks(root) if sys.argv[1:]: @@ -1721,7 +1726,8 @@ macosxSupport.setupApp(root, None) edit = EditorWindow(root=root, filename=filename) edit.text.bind("<>", edit.close_event) - parent.mainloop() + # Does not stop error, neither does following + # edit.text.bind("<>", edit.close_event) if __name__ == '__main__': diff --git a/lib-python/2.7/idlelib/GrepDialog.py b/lib-python/2.7/idlelib/GrepDialog.py --- a/lib-python/2.7/idlelib/GrepDialog.py +++ b/lib-python/2.7/idlelib/GrepDialog.py @@ -45,10 +45,10 @@ def create_entries(self): SearchDialogBase.create_entries(self) - self.globent = self.make_entry("In files:", self.globvar) + self.globent = self.make_entry("In files:", self.globvar)[0] def create_other_buttons(self): - f = self.make_frame() + f = self.make_frame()[0] btn = Checkbutton(f, anchor="w", variable=self.recvar, @@ -131,7 +131,7 @@ self.top.withdraw() -def _grep_dialog(parent): # for htest +def _grep_dialog(parent): # htest # from idlelib.PyShell import PyShellFileList root = Tk() root.title("Test GrepDialog") diff --git a/lib-python/2.7/idlelib/IOBinding.py b/lib-python/2.7/idlelib/IOBinding.py --- a/lib-python/2.7/idlelib/IOBinding.py +++ b/lib-python/2.7/idlelib/IOBinding.py @@ -19,11 +19,7 @@ from idlelib.configHandler import idleConf -try: - from codecs import BOM_UTF8 -except ImportError: - # only available since Python 2.3 - BOM_UTF8 = '\xef\xbb\xbf' +from codecs import BOM_UTF8 # Try setting the locale, so that we can find out # what encoding to use @@ -72,6 +68,7 @@ encoding = encoding.lower() coding_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)') +blank_re = re.compile(r'^[ \t\f]*(?:[#\r\n]|$)') class EncodingMessage(SimpleDialog): "Inform user that an encoding declaration is needed." @@ -130,6 +127,8 @@ match = coding_re.match(line) if match is not None: break + if not blank_re.match(line): + return None else: return None name = match.group(1) @@ -529,6 +528,8 @@ ("All files", "*"), ] + defaultextension = '.py' if sys.platform == 'darwin' else '' + def askopenfile(self): dir, base = self.defaultfilename("open") if not self.opendialog: @@ -554,8 +555,10 @@ def asksavefile(self): dir, base = self.defaultfilename("save") if not self.savedialog: - self.savedialog = tkFileDialog.SaveAs(master=self.text, - filetypes=self.filetypes) + self.savedialog = tkFileDialog.SaveAs( + master=self.text, + filetypes=self.filetypes, + defaultextension=self.defaultextension) filename = self.savedialog.show(initialdir=dir, initialfile=base) if isinstance(filename, unicode): filename = filename.encode(filesystemencoding) diff --git a/lib-python/2.7/idlelib/NEWS.txt b/lib-python/2.7/idlelib/NEWS.txt --- a/lib-python/2.7/idlelib/NEWS.txt +++ b/lib-python/2.7/idlelib/NEWS.txt @@ -1,6 +1,183 @@ +What's New in IDLE 2.7.9? +========================= + +*Release data: 2014-12-07* (projected) + +- Issue #16893: Update Idle doc chapter to match current Idle and add new + information. + +- Issue #3068: Add Idle extension configuration dialog to Options menu. + Changes are written to HOME/.idlerc/config-extensions.cfg. + Original patch by Tal Einat. + +- Issue #16233: A module browser (File : Class Browser, Alt+C) requires a + editor window with a filename. When Class Browser is requested otherwise, + from a shell, output window, or 'Untitled' editor, Idle no longer displays + an error box. It now pops up an Open Module box (Alt+M). If a valid name + is entered and a module is opened, a corresponding browser is also opened. + +- Issue #4832: Save As to type Python files automatically adds .py to the + name you enter (even if your system does not display it). Some systems + automatically add .txt when type is Text files. + +- Issue #21986: Code objects are not normally pickled by the pickle module. + To match this, they are no longer pickled when running under Idle. + +- Issue #22221: IDLE now ignores the source encoding declaration on the second + line if the first line contains anything except a comment. + +- Issue #17390: Adjust Editor window title; remove 'Python', + move version to end. + +- Issue #14105: Idle debugger breakpoints no longer disappear + when inseting or deleting lines. + + +What's New in IDLE 2.7.8? +========================= + +*Release date: 2014-06-29* + +- Issue #21940: Add unittest for WidgetRedirector. Initial patch by Saimadhav + Heblikar. + +- Issue #18592: Add unittest for SearchDialogBase. Patch by Phil Webster. + +- Issue #21694: Add unittest for ParenMatch. Patch by Saimadhav Heblikar. + +- Issue #21686: add unittest for HyperParser. Original patch by Saimadhav + Heblikar. + +- Issue #12387: Add missing upper(lower)case versions of default Windows key + bindings for Idle so Caps Lock does not disable them. Patch by Roger Serwy. + +- Issue #21695: Closing a Find-in-files output window while the search is + still in progress no longer closes Idle. + +- Issue #18910: Add unittest for textView. Patch by Phil Webster. + +- Issue #18292: Add unittest for AutoExpand. Patch by Saihadhav Heblikar. + +- Issue #18409: Add unittest for AutoComplete. Patch by Phil Webster. + + +What's New in IDLE 2.7.7? +========================= + +*Release date: 2014-05-31* + +- Issue #18104: Add idlelib/idle_test/htest.py with a few sample tests to begin + consolidating and improving human-validated tests of Idle. Change other files + as needed to work with htest. Running the module as __main__ runs all tests. + +- Issue #21139: Change default paragraph width to 72, the PEP 8 recommendation. + +- Issue #21284: Paragraph reformat test passes after user changes reformat width. + +- Issue #20406: Use Python application icons for Idle window title bars. + Patch mostly by Serhiy Storchaka. + +- Issue #21029: Occurrences of "print" are now consistently colored as + being a keyword (the colorizer doesn't know if print functions are + enabled in the source). + +- Issue #17721: Remove non-functional configuration dialog help button until we + make it actually gives some help when clicked. Patch by Guilherme Sim�es. + +- Issue #17390: Add Python version to Idle editor window title bar. + Original patches by Edmond Burnett and Kent Johnson. + +- Issue #20058: sys.stdin.readline() in IDLE now always returns only one line. + +- Issue #19481: print() of unicode, str or bytearray subclass instance in IDLE + no more hangs. + +- Issue #18270: Prevent possible IDLE AttributeError on OS X when no initial + shell window is present. + +- Issue #17654: Ensure IDLE menus are customized properly on OS X for + non-framework builds and for all variants of Tk. + + +What's New in IDLE 2.7.6? +========================= + +*Release date: 2013-11-10* + +- Issue #19426: Fixed the opening of Python source file with specified encoding. + +- Issue #18873: IDLE now detects Python source code encoding only in comment + lines. + +- Issue #18988: The "Tab" key now works when a word is already autocompleted. + +- Issue #18489: Add tests for SearchEngine. Original patch by Phil Webster. + +- Issue #18429: Format / Format Paragraph, now works when comment blocks + are selected. As with text blocks, this works best when the selection + only includes complete lines. + +- Issue #18226: Add docstrings and unittests for FormatParagraph.py. + Original patches by Todd Rovito and Phil Webster. + +- Issue #18279: Format - Strip trailing whitespace no longer marks a file as + changed when it has not been changed. This fix followed the addition of a + test file originally written by Phil Webster (the issue's main goal). + +- Issue #18539: Calltips now work for float default arguments. + +- Issue #7136: In the Idle File menu, "New Window" is renamed "New File". + Patch by Tal Einat, Roget Serwy, and Todd Rovito. + +- Issue #8515: Set __file__ when run file in IDLE. + Initial patch by Bruce Frederiksen. + +- Issue #5492: Avoid traceback when exiting IDLE caused by a race condition. + +- Issue #17511: Keep IDLE find dialog open after clicking "Find Next". + Original patch by Sarah K. + +- Issue #15392: Create a unittest framework for IDLE. + Preliminary patch by Rajagopalasarma Jayakrishnan + See Lib/idlelib/idle_test/README.txt for how to run Idle tests. + +- Issue #14146: Highlight source line while debugging on Windows. + +- Issue #17532: Always include Options menu for IDLE on OS X. + Patch by Guilherme Sim�es. + + What's New in IDLE 2.7.5? ========================= +*Release date: 2013-05-12* + +- Issue #17838: Allow sys.stdin to be reassigned. + +- Issue #14735: Update IDLE docs to omit "Control-z on Windows". + +- Issue #17585: Fixed IDLE regression. Now closes when using exit() or quit(). + +- Issue #17657: Show full Tk version in IDLE's about dialog. + Patch by Todd Rovito. + +- Issue #17613: Prevent traceback when removing syntax colorizer in IDLE. + +- Issue #1207589: Backwards-compatibility patch for right-click menu in IDLE. + +- Issue #16887: IDLE now accepts Cancel in tabify/untabify dialog box. + +- Issue #14254: IDLE now handles readline correctly across shell restarts. + +- Issue #17614: IDLE no longer raises exception when quickly closing a file. + +- Issue #6698: IDLE now opens just an editor window when configured to do so. + +- Issue #8900: Using keyboard shortcuts in IDLE to open a file no longer + raises an exception. + +- Issue #6649: Fixed missing exit status in IDLE. Patch by Guilherme Polo. + - Issue #17390: Display Python version on Idle title bar. Initial patch by Edmond Burnett. @@ -8,17 +185,67 @@ What's New in IDLE 2.7.4? ========================= +*Release date: 2013-04-06* + +- Issue #17625: In IDLE, close the replace dialog after it is used. + +- IDLE was displaying spurious SystemExit tracebacks when running scripts + that terminated by raising SystemExit (i.e. unittest and turtledemo). + +- Issue #9290: In IDLE the sys.std* streams now implement io.TextIOBase + interface and support all mandatory methods and properties. + +- Issue #16829: IDLE printing no longer fails if there are spaces or other + special characters in the file path. + +- Issue #16819: IDLE method completion now correctly works for unicode literals. + +- Issue #16504: IDLE now catches SyntaxErrors raised by tokenizer. Patch by + Roger Serwy. + +- Issue #1207589: Add Cut/Copy/Paste items to IDLE right click Context Menu + Patch by Todd Rovito. + +- Issue #13052: Fix IDLE crashing when replace string in Search/Replace dialog + ended with '\'. Patch by Roger Serwy. + +- Issue #9803: Don't close IDLE on saving if breakpoint is open. + Patch by Roger Serwy. + +- Issue #14958: Change IDLE systax highlighting to recognize all string and byte + literals currently supported in Python 2.7. + +- Issue #14962: Update text coloring in IDLE shell window after changing + options. Patch by Roger Serwy. + +- Issue #10997: Prevent a duplicate entry in IDLE's "Recent Files" menu. + +- Issue #12510: Attempting to get invalid tooltip no longer closes IDLE. + Original patch by Roger Serwy. + +- Issue #10365: File open dialog now works instead of crashing + even when parent window is closed. Patch by Roger Serwy. + +- Issue #14876: Use user-selected font for highlight configuration. + Patch by Roger Serwy. + +- Issue #14409: IDLE now properly executes commands in the Shell window + when it cannot read the normal config files on startup and + has to use the built-in default key bindings. + There was previously a bug in one of the defaults. + +- Issue #3573: IDLE hangs when passing invalid command line args + (directory(ies) instead of file(s)) (Patch by Guilherme Polo) + +- Issue #5219: Prevent event handler cascade in IDLE. + - Issue #15318: Prevent writing to sys.stdin. - Issue #13532, #15319: Check that arguments to sys.stdout.write are strings. -- Issue # 12510: Attempt to get certain tool tips no longer crashes IDLE. - -- Issue10365: File open dialog now works instead of crashing even when +- Issue #10365: File open dialog now works instead of crashing even when parent window is closed while dialog is open. -- Issue 14876: use user-selected font for highlight configuration. - - Issue #14018: Update checks for unstable system Tcl/Tk versions on OS X to include versions shipped with OS X 10.7 and 10.8 in addition to 10.6. From noreply at buildbot.pypy.org Sun Mar 29 16:26:28 2015 From: noreply at buildbot.pypy.org (mjacob) Date: Sun, 29 Mar 2015 16:26:28 +0200 (CEST) Subject: [pypy-commit] pypy default: Back out 994b78f6c6eb: it breaks translation. Message-ID: <20150329142628.36D591C01D0@cobra.cs.uni-duesseldorf.de> Author: Manuel Jacob Branch: Changeset: r76619:31f5030575aa Date: 2015-03-29 16:00 +0200 http://bitbucket.org/pypy/pypy/changeset/31f5030575aa/ Log: Back out 994b78f6c6eb: it breaks translation. diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -521,7 +521,6 @@ custom_trace_funcs_unrolled = unrolling_iterable( [(self.get_type_id(TP), func) for TP, func in custom_trace_funcs]) - @rgc.no_collect @specialize.arg(2) def custom_trace_dispatcher(obj, typeid, callback, arg): for type_id_exp, func in custom_trace_funcs_unrolled: From noreply at buildbot.pypy.org Sun Mar 29 16:26:29 2015 From: noreply at buildbot.pypy.org (mjacob) Date: Sun, 29 Mar 2015 16:26:29 +0200 (CEST) Subject: [pypy-commit] pypy llvm-translation-backend: hg merge default Message-ID: <20150329142629.58DD21C01D0@cobra.cs.uni-duesseldorf.de> Author: Manuel Jacob Branch: llvm-translation-backend Changeset: r76620:9f1f8c424c2b Date: 2015-03-29 16:01 +0200 http://bitbucket.org/pypy/pypy/changeset/9f1f8c424c2b/ Log: hg merge default diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -521,7 +521,6 @@ custom_trace_funcs_unrolled = unrolling_iterable( [(self.get_type_id(TP), func) for TP, func in custom_trace_funcs]) - @rgc.no_collect @specialize.arg(2) def custom_trace_dispatcher(obj, typeid, callback, arg): for type_id_exp, func in custom_trace_funcs_unrolled: From noreply at buildbot.pypy.org Sun Mar 29 17:07:34 2015 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 29 Mar 2015 17:07:34 +0200 (CEST) Subject: [pypy-commit] pypy default: Merged in gc-incminimark-pinning-improve (pull request #313) Message-ID: <20150329150734.4AE1C1C11E0@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r76621:9f53b151c5d6 Date: 2015-03-29 17:07 +0200 http://bitbucket.org/pypy/pypy/changeset/9f53b151c5d6/ Log: Merged in gc-incminimark-pinning-improve (pull request #313) Use pinning and additional GC debugging env var diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -5,3 +5,9 @@ .. this is a revision shortly after release-2.5.1 .. startrev: 397b96217b85 +.. branch: gc-incminimark-pinning-improve +Object Pinning is now used in `bz2` and `rzlib` (therefore also affects +Python's `zlib`). In case the data to compress/decompress is inside the nursery +(incminimark) it no longer needs to create a non-moving copy of it. This saves +one `malloc` and copying the data. Additionally a new GC environment variable +is introduced (`PYPY_GC_MAX_PINNED`) primarily for debugging purposes. diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py --- a/pypy/module/bz2/interp_bz2.py +++ b/pypy/module/bz2/interp_bz2.py @@ -562,10 +562,7 @@ in_bufsize = datasize with OutBuffer(self.bzs) as out: - with lltype.scoped_alloc(rffi.CCHARP.TO, in_bufsize) as in_buf: - - for i in range(datasize): - in_buf[i] = data[i] + with rffi.scoped_nonmovingbuffer(data) as in_buf: self.bzs.c_next_in = in_buf rffi.setintfield(self.bzs, 'c_avail_in', in_bufsize) @@ -663,9 +660,7 @@ in_bufsize = len(data) - with lltype.scoped_alloc(rffi.CCHARP.TO, in_bufsize) as in_buf: - for i in range(in_bufsize): - in_buf[i] = data[i] + with rffi.scoped_nonmovingbuffer(data) as in_buf: self.bzs.c_next_in = in_buf rffi.setintfield(self.bzs, 'c_avail_in', in_bufsize) @@ -716,9 +711,7 @@ with lltype.scoped_alloc(bz_stream.TO, zero=True) as bzs: in_bufsize = len(data) - with lltype.scoped_alloc(rffi.CCHARP.TO, in_bufsize) as in_buf: - for i in range(in_bufsize): - in_buf[i] = data[i] + with rffi.scoped_nonmovingbuffer(data) as in_buf: bzs.c_next_in = in_buf rffi.setintfield(bzs, 'c_avail_in', in_bufsize) @@ -758,9 +751,7 @@ return space.wrap("") with lltype.scoped_alloc(bz_stream.TO, zero=True) as bzs: - with lltype.scoped_alloc(rffi.CCHARP.TO, in_bufsize) as in_buf: - for i in range(in_bufsize): - in_buf[i] = data[i] + with rffi.scoped_nonmovingbuffer(data) as in_buf: bzs.c_next_in = in_buf rffi.setintfield(bzs, 'c_avail_in', in_bufsize) diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -47,6 +47,11 @@ too slow for normal use. Values are 0 (off), 1 (on major collections) or 2 (also on minor collections). + + PYPY_GC_MAX_PINNED The maximal number of pinned objects at any point + in time. Defaults to a conservative value depending + on nursery size and maximum object size inside the + nursery. Useful for debugging by setting it to 0. """ # XXX Should find a way to bound the major collection threshold by the # XXX total addressable size. Maybe by keeping some minimarkpage arenas @@ -56,6 +61,7 @@ # XXX try merging old_objects_pointing_to_pinned into # XXX old_objects_pointing_to_young (IRC 2014-10-22, fijal and gregor_w) import sys +import os from rpython.rtyper.lltypesystem import lltype, llmemory, llarena, llgroup from rpython.rtyper.lltypesystem.lloperation import llop from rpython.rtyper.lltypesystem.llmemory import raw_malloc_usage @@ -463,9 +469,19 @@ self.nursery_size = newsize self.allocate_nursery() # - # Estimate this number conservatively - bigobj = self.nonlarge_max + 1 - self.max_number_of_pinned_objects = self.nursery_size / (bigobj * 2) + env_max_number_of_pinned_objects = os.environ.get('PYPY_GC_MAX_PINNED') + if env_max_number_of_pinned_objects: + try: + env_max_number_of_pinned_objects = int(env_max_number_of_pinned_objects) + except ValueError: + env_max_number_of_pinned_objects = 0 + # + if env_max_number_of_pinned_objects >= 0: # 0 allows to disable pinning completely + self.max_number_of_pinned_objects = env_max_number_of_pinned_objects + else: + # Estimate this number conservatively + bigobj = self.nonlarge_max + 1 + self.max_number_of_pinned_objects = self.nursery_size / (bigobj * 2) def _nursery_memory_size(self): extra = self.nonlarge_max + 1 diff --git a/rpython/rlib/rzlib.py b/rpython/rlib/rzlib.py --- a/rpython/rlib/rzlib.py +++ b/rpython/rlib/rzlib.py @@ -365,10 +365,8 @@ """Common code for compress() and decompress(). """ # Prepare the input buffer for the stream - with lltype.scoped_alloc(rffi.CCHARP.TO, len(data)) as inbuf: - # XXX (groggi) should be possible to improve this with pinning by - # not performing the 'copy_string_to_raw' if non-movable/pinned - copy_string_to_raw(llstr(data), inbuf, 0, len(data)) + assert data is not None # XXX seems to be sane assumption, however not for sure + with rffi.scoped_nonmovingbuffer(data) as inbuf: stream.c_next_in = rffi.cast(Bytefp, inbuf) rffi.setintfield(stream, 'c_avail_in', len(data)) From noreply at buildbot.pypy.org Sun Mar 29 17:15:33 2015 From: noreply at buildbot.pypy.org (groggi) Date: Sun, 29 Mar 2015 17:15:33 +0200 (CEST) Subject: [pypy-commit] pypy gc-incminimark-pinning-improve: Close branch gc-incminimark-pinning-improve. Message-ID: <20150329151533.86AEC1C01D0@cobra.cs.uni-duesseldorf.de> Author: Gregor Wegberg Branch: gc-incminimark-pinning-improve Changeset: r76622:08b71058baa1 Date: 2015-03-29 15:16 +0000 http://bitbucket.org/pypy/pypy/changeset/08b71058baa1/ Log: Close branch gc-incminimark-pinning-improve. From noreply at buildbot.pypy.org Sun Mar 29 17:20:41 2015 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Sun, 29 Mar 2015 17:20:41 +0200 (CEST) Subject: [pypy-commit] pypy default: Do this whole operation without a copy Message-ID: <20150329152041.B02391C0498@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r76623:bea034347eb0 Date: 2015-03-29 11:20 -0400 http://bitbucket.org/pypy/pypy/changeset/bea034347eb0/ Log: Do this whole operation without a copy diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py --- a/pypy/module/_hashlib/interp_hashlib.py +++ b/pypy/module/_hashlib/interp_hashlib.py @@ -144,10 +144,10 @@ with self.lock: ropenssl.EVP_MD_CTX_copy(ctx, self.ctx) digest_size = self.digest_size - with lltype.scoped_alloc(rffi.CCHARP.TO, digest_size) as digest: - ropenssl.EVP_DigestFinal(ctx, digest, None) + with rffi.scoped_alloc_buffer(digest_size) as buf: + ropenssl.EVP_DigestFinal(ctx, buf.raw, None) ropenssl.EVP_MD_CTX_cleanup(ctx) - return rffi.charpsize2str(digest, digest_size) + return buf.str(digest_size) W_Hash.typedef = TypeDef( From noreply at buildbot.pypy.org Sun Mar 29 19:34:23 2015 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 29 Mar 2015 19:34:23 +0200 (CEST) Subject: [pypy-commit] pypy vmprof: dont inline those functions for markers Message-ID: <20150329173423.1FF9D1C02AD@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76624:ed79ad29842d Date: 2015-03-29 19:33 +0200 http://bitbucket.org/pypy/pypy/changeset/ed79ad29842d/ Log: dont inline those functions for markers diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py --- a/rpython/jit/metainterp/blackhole.py +++ b/rpython/jit/metainterp/blackhole.py @@ -1653,6 +1653,7 @@ resumedescr.guard_opnum, deadframe) _run_forever(blackholeinterp, current_exc) +resume_in_blackhole._dont_inline_ = True def convert_and_run_from_pyjitpl(metainterp, raising_exception=False): # Get a chain of blackhole interpreters and fill them by copying @@ -1676,3 +1677,4 @@ current_exc = lltype.nullptr(rclass.OBJECTPTR.TO) # _run_forever(firstbh, current_exc) +convert_and_run_from_pyjitpl._dont_inline_ = True From noreply at buildbot.pypy.org Sun Mar 29 19:34:24 2015 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 29 Mar 2015 19:34:24 +0200 (CEST) Subject: [pypy-commit] pypy vmprof: merge Message-ID: <20150329173424.5E0A01C02AD@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76625:5110ae276a39 Date: 2015-03-29 19:34 +0200 http://bitbucket.org/pypy/pypy/changeset/5110ae276a39/ Log: merge diff --git a/rpython/bin/rpython-vmprof b/rpython/bin/rpython-vmprof --- a/rpython/bin/rpython-vmprof +++ b/rpython/bin/rpython-vmprof @@ -21,7 +21,7 @@ x = open("vmprof.log", "w") _vmprof.enable(x.fileno(), 1000) try: - main() + main() finally: - _vmprof.disable() + _vmprof.disable() x.close() From noreply at buildbot.pypy.org Sun Mar 29 21:04:41 2015 From: noreply at buildbot.pypy.org (mattip) Date: Sun, 29 Mar 2015 21:04:41 +0200 (CEST) Subject: [pypy-commit] pypy object-dtype2: remove print, for loop but still segfaults since 'length' is not a valid value Message-ID: <20150329190441.C0E3A1C02AD@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: object-dtype2 Changeset: r76626:b15a337adf20 Date: 2015-03-29 22:05 +0300 http://bitbucket.org/pypy/pypy/changeset/b15a337adf20/ Log: remove print, for loop but still segfaults since 'length' is not a valid value diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py --- a/pypy/module/micronumpy/concrete.py +++ b/pypy/module/micronumpy/concrete.py @@ -1,7 +1,7 @@ from pypy.interpreter.error import OperationError, oefmt from rpython.rlib import jit, rgc from rpython.rlib.buffer import Buffer -from rpython.rlib.debug import make_sure_not_resized +from rpython.rlib.debug import make_sure_not_resized, debug_print from rpython.rlib.rawstorage import alloc_raw_storage, free_raw_storage, \ raw_storage_getitem, raw_storage_setitem, RAW_STORAGE from rpython.rtyper.lltypesystem import rffi, lltype, llmemory @@ -345,17 +345,19 @@ offset_of_step = llmemory.offsetof(OBJECTSTORE, 'step') def customtrace(gc, obj, callback, arg): - print 'in customtrace w/obj',obj + #debug_print('in customtrace w/obj', obj) length = rffi.cast(lltype.Signed, obj + offset_of_length) step = rffi.cast(lltype.Signed, obj + offset_of_step) storage = obj + offset_of_storage - print 'tracing', length, 'objects in ndarray.storage' - for i in range(length): - gcref = rffi.cast(llmemory.GCREF, storage) - w_obj = cast_gcref_to_instance(W_Root, gcref) - print 'tracing', w_obj + debug_print('tracing', length, 'objects in ndarray.storage') + i = 0 + while i < length: + #gcref = rffi.cast(llmemory.GCREF, storage) + #w_obj = cast_gcref_to_instance(W_Root, gcref) + #debug_print('tracing', w_obj) gc._trace_callback(callback, arg, storage) storage += step + i += 1 lambda_customtrace = lambda: customtrace From noreply at buildbot.pypy.org Sun Mar 29 21:30:00 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 29 Mar 2015 21:30:00 +0200 (CEST) Subject: [pypy-commit] pypy default: Improve the CollectAnalyzer to not crash when reporting a wrong '@rgc.no_collect' Message-ID: <20150329193000.94BF61C1F07@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76627:8c146378bebe Date: 2015-03-29 19:40 +0200 http://bitbucket.org/pypy/pypy/changeset/8c146378bebe/ Log: Improve the CollectAnalyzer to not crash when reporting a wrong '@rgc.no_collect' diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -36,7 +36,10 @@ return graphanalyze.BoolGraphAnalyzer.analyze_direct_call(self, graph, seen) def analyze_external_call(self, op, seen=None): - funcobj = op.args[0].value._obj + try: + funcobj = op.args[0].value._obj + except lltype.DelayedPointer: + return True if getattr(funcobj, 'random_effects_on_gcobjs', False): return True return graphanalyze.BoolGraphAnalyzer.analyze_external_call(self, op, From noreply at buildbot.pypy.org Sun Mar 29 21:30:01 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 29 Mar 2015 21:30:01 +0200 (CEST) Subject: [pypy-commit] pypy default: redo the equivalent of 994b78f6c6eb in a way that works (can't use rgc.no_collect for various reasons) Message-ID: <20150329193001.C2AE61C1F07@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76628:881354657952 Date: 2015-03-29 21:30 +0200 http://bitbucket.org/pypy/pypy/changeset/881354657952/ Log: redo the equivalent of 994b78f6c6eb in a way that works (can't use rgc.no_collect for various reasons) diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -251,6 +251,8 @@ annhelper.finish() # at this point, annotate all mix-level helpers annhelper.backend_optimize() + self.check_custom_trace_funcs(gcdata.gc, translator.rtyper) + self.collect_analyzer = CollectAnalyzer(self.translator) self.collect_analyzer.analyze_all() @@ -540,6 +542,24 @@ self.gcdata._has_got_custom_trace(self.get_type_id(TP)) specialize.arg(2)(func) + def check_custom_trace_funcs(self, gc, rtyper): + # detect if one of the custom trace functions uses the GC + # (it must not!) + for TP, func in rtyper.custom_trace_funcs: + def no_op_callback(obj, arg): + pass + def ll_check_no_collect(obj): + func(gc, obj, no_op_callback, None) + annhelper = annlowlevel.MixLevelHelperAnnotator(rtyper) + graph1 = annhelper.getgraph(ll_check_no_collect, [SomeAddress()], + annmodel.s_None) + annhelper.finish() + collect_analyzer = CollectAnalyzer(self.translator) + if collect_analyzer.analyze_direct_call(graph1): + raise Exception( + "the custom trace hook %r for %r can cause " + "the GC to be called!" % (func, TP)) + def consider_constant(self, TYPE, value): self.layoutbuilder.consider_constant(TYPE, value, self.gcdata.gc) diff --git a/rpython/memory/gctransform/test/test_framework.py b/rpython/memory/gctransform/test/test_framework.py --- a/rpython/memory/gctransform/test/test_framework.py +++ b/rpython/memory/gctransform/test/test_framework.py @@ -143,6 +143,31 @@ expected = "'no_collect' function can trigger collection: Author: Remi Meier Branch: Changeset: r1743:f6788cf5fb73 Date: 2015-03-30 10:07 +0200 http://bitbucket.org/pypy/stmgc/changeset/f6788cf5fb73/ Log: port the recent changes to c8 diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -2,6 +2,11 @@ # error "must be compiled via stmgc.c" #endif +char *stm_object_pages; +long _stm_segment_nb_pages = NB_PAGES; +int _stm_nb_segments = NB_SEGMENTS; +int _stm_psegment_ofs = (int)(uintptr_t)STM_PSEGMENT; + /* *** MISC *** */ static void free_bk(struct stm_undo_s *undo) { @@ -1127,6 +1132,14 @@ check_nursery_at_transaction_start(); + /* Change read-version here, because if we do stm_validate in the + safe-point below, we should not see our old reads from the last + transaction. */ + uint8_t rv = STM_SEGMENT->transaction_read_version; + if (rv < 0xff) /* else, rare (maybe impossible?) case: we did already */ + rv++; /* incr it but enter_safe_point_if_requested() aborted */ + STM_SEGMENT->transaction_read_version = rv; + /* Warning: this safe-point may run light finalizers and register commit/abort callbacks if a major GC is triggered here */ enter_safe_point_if_requested(); @@ -1134,9 +1147,7 @@ s_mutex_unlock(); // XXX it's probably possible to not acquire this here - uint8_t old_rv = STM_SEGMENT->transaction_read_version; - STM_SEGMENT->transaction_read_version = old_rv + 1; - if (UNLIKELY(old_rv == 0xff)) { + if (UNLIKELY(rv == 0xff)) { reset_transaction_read_version(); } diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -222,11 +222,10 @@ static void free_cle(struct stm_commit_log_entry_s *e); -#ifndef STM_TESTS -static char *stm_object_pages; -#else -char *stm_object_pages; -#endif +extern char *stm_object_pages; +extern long _stm_segment_nb_pages; +extern int _stm_nb_segments; +extern int _stm_psegment_ofs; static stm_thread_local_t *stm_all_thread_locals = NULL; diff --git a/c8/stm/sync.c b/c8/stm/sync.c --- a/c8/stm/sync.c +++ b/c8/stm/sync.c @@ -103,6 +103,24 @@ /************************************************************/ +void stm_wait_for_current_inevitable_transaction(void) +{ + struct stm_commit_log_entry_s *current = STM_PSEGMENT->last_commit_log_entry; + + /* XXX: don't do busy-waiting */ + while (1) { + if (current->next == NULL) { + break; + } else if (current->next == INEV_RUNNING) { + usleep(10); + continue; + } + current = current->next; + } +} + + + static bool acquire_thread_segment(stm_thread_local_t *tl) { /* This function acquires a segment for the currently running thread, diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -304,6 +304,11 @@ void stm_start_inevitable_transaction(stm_thread_local_t *tl); void stm_commit_transaction(void); + +/* Temporary fix? Call this outside a transaction. If there is an + inevitable transaction running somewhere else, wait until it finishes. */ +void stm_wait_for_current_inevitable_transaction(void); + void stm_abort_transaction(void) __attribute__((noreturn)); void stm_collect(long level); From noreply at buildbot.pypy.org Mon Mar 30 10:07:50 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 30 Mar 2015 10:07:50 +0200 (CEST) Subject: [pypy-commit] extradoc extradoc: Prepare the blog post Message-ID: <20150330080750.34D5D1C0207@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r5515:4f27759c4bac Date: 2015-03-30 10:08 +0200 http://bitbucket.org/pypy/extradoc/changeset/4f27759c4bac/ Log: Prepare the blog post diff --git a/blog/draft/stm-mar2015.txt b/blog/draft/stm-mar2015.txt --- a/blog/draft/stm-mar2015.txt +++ b/blog/draft/stm-mar2015.txt @@ -1,10 +1,66 @@ -============= -Status of STM -============= +============================= +PyPy-STM 2.5.1 - Mawhrin-Skel +============================= -- stmgc-c7 reasonably good results, now porting to stmgc-c8 +We're pleased to announce PyPy-STM 2.5.1, Mawhrin-Skel. This is the +first full release of PyPy-STM. You can download this release here: -- stmgc-c7 results (with download link), explain the new - features + http://pypy.org/download.html -- stmgc-c8 in two words +Documentation: + + http://pypy.readthedocs.org/en/latest/stm.html + +This is a special version of PyPy that contains the "Software +Transactional Memory" (STM) core called stmgc-c7. It gives a +replacement for Python's classical Global Interpreter Lock (GIL). The +current version scales only up to around 4 cores; the next version of +the core, stmgc-c8, is in development and should address that +limitation. Both versions only supports 64-bit Linux for now +(contributions welcome). + +This is the first full release: it runs all regular PyPy tests. It +passes most of them... only, but that's still much better than the +previous versions. In other words, you should be able to drop in +PyPy-STM instead of the regular PyPy and your program should still +work. See `current status`_ for more information. + +.. _`current status`: http://pypy.readthedocs.org/en/latest/stm.html#current-status-stmgc-c7 + +It seems the performance is now more stable than it used to be. More +precisely, the best case is still "25%-40% single-core slow-down with +very good scaling up to 4 threads", but the average performance seems +not too far from that. There are still dark spots --- notably, the +JIT is still slower to warm up, though it was improved. Apart from +that, we should not get worse than 2x single-core slow-down in the +worst case. Please report such cases as bugs! + +This work was done by Remi Meier and Armin Rigo. Thanks to all donors +for `crowd-funding the STM work`_ so far! (As usual, it took longer +than we would have thought. I really want to thank the people that +kept making donations anyway. Your trust is great!) + +.. _`crowd-funding the STM work`: http://pypy.org/tmdonate2.html + + +TransactionQueue +---------------- + +PyPy-STM is more than "just" a Python without GIL. It is a Python in +which you can do minor tweaks to your existing, *non-multithreaded* +program, and get them to use multiple cores. The basic idea is to +identify medium- or large-sized likely-independent parts of the code, +like every iteration of some outermost loop over all items of a +dictionary; then ask PyPy-STM to try to run these parts in parallel. +This is done with a ``transaction.TransactionQueue`` instance. See +``help(TransactionQueue)`` or read more about it in `the STM user +guide`_. + +This is not a 100% mechanical change: very likely, you need to hunt +for and fix "STM conflicts" that prevent parallel execution (see +docs_). However, at all points your program runs correctly, and you +can stop the hunt when you get acceptable performance. You don't get +deadlocks or corrupted state. + +.. _`the STM user guide`: http://pypy.readthedocs.org/en/latest/stm.html#user-guide +.. _docs: http://pypy.readthedocs.org/en/latest/stm.html#transaction-transactionqueue From noreply at buildbot.pypy.org Mon Mar 30 10:32:29 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Mon, 30 Mar 2015 10:32:29 +0200 (CEST) Subject: [pypy-commit] extradoc extradoc: some rewording Message-ID: <20150330083229.499901C02AD@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: extradoc Changeset: r5516:dcffa5ce26c7 Date: 2015-03-30 10:34 +0200 http://bitbucket.org/pypy/extradoc/changeset/dcffa5ce26c7/ Log: some rewording diff --git a/blog/draft/stm-mar2015.txt b/blog/draft/stm-mar2015.txt --- a/blog/draft/stm-mar2015.txt +++ b/blog/draft/stm-mar2015.txt @@ -16,29 +16,28 @@ replacement for Python's classical Global Interpreter Lock (GIL). The current version scales only up to around 4 cores; the next version of the core, stmgc-c8, is in development and should address that -limitation. Both versions only supports 64-bit Linux for now +limitation. Both versions only support 64-bit Linux for now (contributions welcome). -This is the first full release: it runs all regular PyPy tests. It -passes most of them... only, but that's still much better than the -previous versions. In other words, you should be able to drop in +This is the first full release: it passes all regular PyPy tests, +except for a few special cases. In other words, you should be able to drop in PyPy-STM instead of the regular PyPy and your program should still work. See `current status`_ for more information. .. _`current status`: http://pypy.readthedocs.org/en/latest/stm.html#current-status-stmgc-c7 -It seems the performance is now more stable than it used to be. More -precisely, the best case is still "25%-40% single-core slow-down with -very good scaling up to 4 threads", but the average performance seems +The performance is now more stable than it used to be. More +precisely, the best case is "25%-40% single-core slow-down with +very good scaling up to 4 threads", and the average performance seems not too far from that. There are still dark spots --- notably, the -JIT is still slower to warm up, though it was improved. Apart from -that, we should not get worse than 2x single-core slow-down in the +JIT is still slower to warm up, though it was improved a lot. Apart from +that, we should not get more than 2x single-core slow-down in the worst case. Please report such cases as bugs! This work was done by Remi Meier and Armin Rigo. Thanks to all donors for `crowd-funding the STM work`_ so far! (As usual, it took longer than we would have thought. I really want to thank the people that -kept making donations anyway. Your trust is great!) +kept making donations anyway. Your trust is greatly appreciated!) .. _`crowd-funding the STM work`: http://pypy.org/tmdonate2.html @@ -48,10 +47,10 @@ PyPy-STM is more than "just" a Python without GIL. It is a Python in which you can do minor tweaks to your existing, *non-multithreaded* -program, and get them to use multiple cores. The basic idea is to -identify medium- or large-sized likely-independent parts of the code, -like every iteration of some outermost loop over all items of a -dictionary; then ask PyPy-STM to try to run these parts in parallel. +programs and get them to use multiple cores. The basic idea is to +identify medium- or large-sized, likely-independent parts of the code +and to ask PyPy-STM to run these parts in parallel. An example would be +every iteration of some outermost loop over all items of a dictionary. This is done with a ``transaction.TransactionQueue`` instance. See ``help(TransactionQueue)`` or read more about it in `the STM user guide`_. From noreply at buildbot.pypy.org Mon Mar 30 10:47:00 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 30 Mar 2015 10:47:00 +0200 (CEST) Subject: [pypy-commit] pypy default: Check and complain if there is a "libpypy-c.so" that appears not to be needed Message-ID: <20150330084700.C2A561C02AD@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76629:98e7823b191a Date: 2015-03-30 10:47 +0200 http://bitbucket.org/pypy/pypy/changeset/98e7823b191a/ Log: Check and complain if there is a "libpypy-c.so" that appears not to be needed diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -65,6 +65,9 @@ add --without-{0} option to skip packaging binary CFFI extension.""".format(module) raise MissingDependenciesError(module) +def pypy_runs(pypy_c): + return subprocess.call([str(pypy_c), '-c', 'pass']) == 0 + def create_package(basedir, options): retval = 0 name = options.name @@ -87,6 +90,8 @@ ' Please compile pypy first, using translate.py,' ' or check that you gave the correct path' ' with --override_pypy_c' % pypy_c) + if not pypy_runs(pypy_c): + raise OSError("Running %r failed!" % (str(pypy_c),)) if not options.no_cffi: try: create_cffi_import_libraries(pypy_c, options) @@ -100,6 +105,15 @@ libpypy_name = 'libpypy-c.so' if not sys.platform.startswith('darwin') else 'libpypy-c.dylib' libpypy_c = pypy_c.new(basename=libpypy_name) if libpypy_c.check(): + # check that this libpypy_c is really needed + os.rename(str(libpypy_c), str(libpypy_c) + '~') + try: + if pypy_runs(pypy_c): + raise Exception("It seems that %r runs without needing %r. " + "Please check and remove the latter" % + (str(pypy_c), str(libpypy_c))) + finally: + os.rename(str(libpypy_c) + '~', str(libpypy_c)) binaries.append((libpypy_c, libpypy_name)) # builddir = options.builddir From noreply at buildbot.pypy.org Mon Mar 30 10:49:23 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 30 Mar 2015 10:49:23 +0200 (CEST) Subject: [pypy-commit] pypy stmgc-c7: Check and complain if there is a "libpypy-c.so" that appears not to be needed Message-ID: <20150330084923.C8E7A1C01D0@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76630:394a15666e6a Date: 2015-03-30 10:49 +0200 http://bitbucket.org/pypy/pypy/changeset/394a15666e6a/ Log: Check and complain if there is a "libpypy-c.so" that appears not to be needed diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -65,6 +65,9 @@ add --without-{0} option to skip packaging binary CFFI extension.""".format(module) raise MissingDependenciesError(module) +def pypy_runs(pypy_c): + return subprocess.call([str(pypy_c), '-c', 'pass']) == 0 + def create_package(basedir, options): retval = 0 name = options.name @@ -87,6 +90,8 @@ ' Please compile pypy first, using translate.py,' ' or check that you gave the correct path' ' with --override_pypy_c' % pypy_c) + if not pypy_runs(pypy_c): + raise OSError("Running %r failed!" % (str(pypy_c),)) if not options.no_cffi: try: create_cffi_import_libraries(pypy_c, options) @@ -100,6 +105,15 @@ libpypy_name = 'libpypy-c.so' if not sys.platform.startswith('darwin') else 'libpypy-c.dylib' libpypy_c = pypy_c.new(basename=libpypy_name) if libpypy_c.check(): + # check that this libpypy_c is really needed + os.rename(str(libpypy_c), str(libpypy_c) + '~') + try: + if pypy_runs(pypy_c): + raise Exception("It seems that %r runs without needing %r. " + "Please check and remove the latter" % + (str(pypy_c), str(libpypy_c))) + finally: + os.rename(str(libpypy_c) + '~', str(libpypy_c)) binaries.append((libpypy_c, libpypy_name)) # builddir = options.builddir From noreply at buildbot.pypy.org Mon Mar 30 10:56:27 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 30 Mar 2015 10:56:27 +0200 (CEST) Subject: [pypy-commit] extradoc extradoc: It's actually the second official release. Message-ID: <20150330085627.1A4C81C1279@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r5517:957fa318a94f Date: 2015-03-30 10:56 +0200 http://bitbucket.org/pypy/extradoc/changeset/957fa318a94f/ Log: It's actually the second official release. diff --git a/blog/draft/stm-mar2015.txt b/blog/draft/stm-mar2015.txt --- a/blog/draft/stm-mar2015.txt +++ b/blog/draft/stm-mar2015.txt @@ -3,7 +3,8 @@ ============================= We're pleased to announce PyPy-STM 2.5.1, Mawhrin-Skel. This is the -first full release of PyPy-STM. You can download this release here: +second official release of PyPy-STM. You can download this release +here: http://pypy.org/download.html @@ -19,8 +20,8 @@ limitation. Both versions only support 64-bit Linux for now (contributions welcome). -This is the first full release: it passes all regular PyPy tests, -except for a few special cases. In other words, you should be able to drop in +This release passes all regular PyPy tests, except for a few +special cases. In other words, you should be able to drop in PyPy-STM instead of the regular PyPy and your program should still work. See `current status`_ for more information. From noreply at buildbot.pypy.org Mon Mar 30 11:10:15 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 30 Mar 2015 11:10:15 +0200 (CEST) Subject: [pypy-commit] pypy stmgc-c7: pypy-stm only: add 'print_stm_log.py' Message-ID: <20150330091015.55D111C01D0@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76631:5a2a649dc56b Date: 2015-03-30 11:10 +0200 http://bitbucket.org/pypy/pypy/changeset/5a2a649dc56b/ Log: pypy-stm only: add 'print_stm_log.py' diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -116,6 +116,11 @@ os.rename(str(libpypy_c) + '~', str(libpypy_c)) binaries.append((libpypy_c, libpypy_name)) # + # PyPy-STM only + p = basedir.join('pypy', 'stm', 'print_stm_log.py') + assert p.check(), "this version should be run in the stm branch" + binaries.append((p, p.basename)) + # builddir = options.builddir pypydir = builddir.ensure(name, dir=True) includedir = basedir.join('include') From noreply at buildbot.pypy.org Mon Mar 30 11:31:43 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 30 Mar 2015 11:31:43 +0200 (CEST) Subject: [pypy-commit] extradoc extradoc: Mention the PYPYSTM reports. This is actually the most important new feature... Message-ID: <20150330093143.5B5F91C01D0@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r5518:a0b6c0fb4b6c Date: 2015-03-30 11:32 +0200 http://bitbucket.org/pypy/extradoc/changeset/a0b6c0fb4b6c/ Log: Mention the PYPYSTM reports. This is actually the most important new feature... diff --git a/blog/draft/stm-mar2015.txt b/blog/draft/stm-mar2015.txt --- a/blog/draft/stm-mar2015.txt +++ b/blog/draft/stm-mar2015.txt @@ -25,7 +25,39 @@ PyPy-STM instead of the regular PyPy and your program should still work. See `current status`_ for more information. +This work was done by Remi Meier and Armin Rigo. Thanks to all donors +for `crowd-funding the STM work`_ so far! As usual, it took longer +than we would have thought. I really want to thank the people that +kept making donations anyway. Your trust is greatly appreciated! + .. _`current status`: http://pypy.readthedocs.org/en/latest/stm.html#current-status-stmgc-c7 +.. _`crowd-funding the STM work`: http://pypy.org/tmdonate2.html +.. _`July 2014 release`: http://morepypy.blogspot.ch/2014/07/pypy-stm-first-interesting-release.html + + +What's new? +----------- + +Compared to the `July 2014 release`_, the main addition is a way to +get reports about STM conflicts. This is an essential new feature. + +To understand why this is so important, consider that if you already +played around with the previous release, chances are that you didn't +get very far. It probably felt like a toy: on very small examples it +would nicely scale, but on any larger example it would not scale at +all. You didn't get any feedback about why, but the underlying reason +for that is that, in any large example, there are a few unexpected STM +conflicts that occur all the time. This prevents any parallelization. + +Now PyPy-STM is no longer a black box: you have a way to learn about +these conflicts and fix them. The tl;dr version is to run:: + + PYPYSTM=stmlog pypy yourprogr.py + print_stm_log.py stmlog + + +Performance +----------- The performance is now more stable than it used to be. More precisely, the best case is "25%-40% single-core slow-down with @@ -35,13 +67,6 @@ that, we should not get more than 2x single-core slow-down in the worst case. Please report such cases as bugs! -This work was done by Remi Meier and Armin Rigo. Thanks to all donors -for `crowd-funding the STM work`_ so far! (As usual, it took longer -than we would have thought. I really want to thank the people that -kept making donations anyway. Your trust is greatly appreciated!) - -.. _`crowd-funding the STM work`: http://pypy.org/tmdonate2.html - TransactionQueue ---------------- From noreply at buildbot.pypy.org Mon Mar 30 11:33:54 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 30 Mar 2015 11:33:54 +0200 (CEST) Subject: [pypy-commit] extradoc extradoc: Renaming the binary "pypy-stm" Message-ID: <20150330093354.56A171C01D0@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r5519:d500687241e2 Date: 2015-03-30 11:34 +0200 http://bitbucket.org/pypy/extradoc/changeset/d500687241e2/ Log: Renaming the binary "pypy-stm" diff --git a/blog/draft/stm-mar2015.txt b/blog/draft/stm-mar2015.txt --- a/blog/draft/stm-mar2015.txt +++ b/blog/draft/stm-mar2015.txt @@ -52,7 +52,7 @@ Now PyPy-STM is no longer a black box: you have a way to learn about these conflicts and fix them. The tl;dr version is to run:: - PYPYSTM=stmlog pypy yourprogr.py + PYPYSTM=stmlog pypy-stm yourprogr.py print_stm_log.py stmlog From noreply at buildbot.pypy.org Mon Mar 30 12:54:10 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 30 Mar 2015 12:54:10 +0200 (CEST) Subject: [pypy-commit] pypy default: Attempt a better explanation Message-ID: <20150330105410.9FD741C02AD@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76632:116b2b19f065 Date: 2015-03-30 12:54 +0200 http://bitbucket.org/pypy/pypy/changeset/116b2b19f065/ Log: Attempt a better explanation diff --git a/rpython/doc/rpython.rst b/rpython/doc/rpython.rst --- a/rpython/doc/rpython.rst +++ b/rpython/doc/rpython.rst @@ -59,7 +59,7 @@ **exceptions** - fully supported + fully supported. see below `Exception rules`_ for restrictions on exceptions raised by built-in operations @@ -136,9 +136,16 @@ **functions** -+ statically called functions may use defaults and a variable number of - arguments (which may be passed as a list instead of a tuple, so write code - that does not depend on it being a tuple). ++ function declarations may use defaults and ``*args``, but not + ``**keywords``. + ++ function calls may be done to a known function or to a variable one, + or to a method. You can call with positional and keyword arguments, + and you can pass a ``*args`` argument (it must be a tuple). + ++ as explained above, tuples are not of a variable length. If you need + to call a function with a variable number of arguments, refactor the + function itself to accept a single argument which is a regular list. + dynamic dispatch enforces the use of signatures that are equal for all possible called function, or at least "compatible enough". This From noreply at buildbot.pypy.org Mon Mar 30 12:58:57 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 30 Mar 2015 12:58:57 +0200 (CEST) Subject: [pypy-commit] pypy default: no list->tuple conversion in rpython Message-ID: <20150330105857.BF1ED1C11DF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76633:dd7e1a7a1009 Date: 2015-03-30 12:59 +0200 http://bitbucket.org/pypy/pypy/changeset/dd7e1a7a1009/ Log: no list->tuple conversion in rpython diff --git a/rpython/doc/rpython.rst b/rpython/doc/rpython.rst --- a/rpython/doc/rpython.rst +++ b/rpython/doc/rpython.rst @@ -92,6 +92,11 @@ no variable-length tuples; use them to store or return pairs or n-tuples of values. Each combination of types for elements and length constitute a separate and not mixable type. + + There is no general way to convert a list into a tuple, because the + length of the result would not be known statically. (You can of course + do ``t = (lst[0], lst[1], lst[2])`` if you know that ``lst`` has got 3 + items.) **lists** From noreply at buildbot.pypy.org Mon Mar 30 13:32:10 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 30 Mar 2015 13:32:10 +0200 (CEST) Subject: [pypy-commit] pypy default: Fix: don't hard-code in the translated pypy the value for Message-ID: <20150330113210.33B961C11B5@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76634:180f86a2e1fb Date: 2015-03-30 13:32 +0200 http://bitbucket.org/pypy/pypy/changeset/180f86a2e1fb/ Log: Fix: don't hard-code in the translated pypy the value for 'openssl_md_meth_names'. diff --git a/pypy/module/_hashlib/__init__.py b/pypy/module/_hashlib/__init__.py --- a/pypy/module/_hashlib/__init__.py +++ b/pypy/module/_hashlib/__init__.py @@ -1,11 +1,10 @@ from pypy.interpreter.mixedmodule import MixedModule -from pypy.module._hashlib.interp_hashlib import algorithms +from pypy.module._hashlib.interp_hashlib import algorithms, fetch_names class Module(MixedModule): interpleveldefs = { 'new' : 'interp_hashlib.new', - 'openssl_md_meth_names': 'interp_hashlib.get(space).w_meth_names' } appleveldefs = { @@ -15,5 +14,5 @@ interpleveldefs['openssl_' + name] = 'interp_hashlib.new_' + name def startup(self, space): - from rpython.rlib.ropenssl import init_digests - init_digests() + w_meth_names = fetch_names(space) + space.setattr(self, space.wrap('openssl_md_meth_names'), w_meth_names) diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py --- a/pypy/module/_hashlib/interp_hashlib.py +++ b/pypy/module/_hashlib/interp_hashlib.py @@ -16,8 +16,6 @@ algorithms = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512') def hash_name_mapper_callback(obj_name, userdata): - state = global_state[0] - assert state is not None if not obj_name: return # Ignore aliased names, they pollute the list and OpenSSL appears @@ -27,36 +25,31 @@ if obj_name[0].c_alias: return try: - w_name = state.space.wrap(rffi.charp2str(obj_name[0].c_name)) - state.space.call_method(state.w_meth_names, "add", w_name) + space = global_name_fetcher.space + w_name = space.wrap(rffi.charp2str(obj_name[0].c_name)) + space.call_method(global_name_fetcher.w_meth_names, "add", w_name) except OperationError, e: - state.w_error = e + global_name_fetcher.w_error = e -# XXX make it threadlocal? -global_state = [None] +class NameFetcher: + def setup(self, space): + self.space = space + self.w_meth_names = space.call_function(space.w_set) + self.w_error = None + def _cleanup_(self): + del self.space + del self.w_meth_names + del self.w_error +global_name_fetcher = NameFetcher() -class State: - def __init__(self, space): - self.space = space - self.generate_method_names(space) - - def generate_method_names(self, space): - if not we_are_translated(): - ropenssl.init_digests() - self.w_error = None - try: - global_state[0] = self - self.w_meth_names = space.call_function(space.w_set) - ropenssl.OBJ_NAME_do_all( - ropenssl.OBJ_NAME_TYPE_MD_METH, - hash_name_mapper_callback, None) - finally: - global_state[0] = None - if self.w_error: - raise self.w_error - -def get(space): - return space.fromcache(State) +def fetch_names(space): + global_name_fetcher.setup(space) + ropenssl.init_digests() + ropenssl.OBJ_NAME_do_all(ropenssl.OBJ_NAME_TYPE_MD_METH, + hash_name_mapper_callback, None) + if global_name_fetcher.w_error: + raise global_name_fetcher.w_error + return global_name_fetcher.w_meth_names class W_Hash(W_Root): NULL_CTX = lltype.nullptr(ropenssl.EVP_MD_CTX.TO) From noreply at buildbot.pypy.org Mon Mar 30 13:44:27 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 30 Mar 2015 13:44:27 +0200 (CEST) Subject: [pypy-commit] pypy default: Fix: maybe these attributes are not defined at all so far Message-ID: <20150330114427.291C21C0207@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76635:5f90d0283413 Date: 2015-03-30 13:44 +0200 http://bitbucket.org/pypy/pypy/changeset/5f90d0283413/ Log: Fix: maybe these attributes are not defined at all so far diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py --- a/pypy/module/_hashlib/interp_hashlib.py +++ b/pypy/module/_hashlib/interp_hashlib.py @@ -37,9 +37,7 @@ self.w_meth_names = space.call_function(space.w_set) self.w_error = None def _cleanup_(self): - del self.space - del self.w_meth_names - del self.w_error + self.__dict__.clear() global_name_fetcher = NameFetcher() def fetch_names(space): From noreply at buildbot.pypy.org Mon Mar 30 13:52:27 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Mon, 30 Mar 2015 13:52:27 +0200 (CEST) Subject: [pypy-commit] pypy stmgc-c8: Merge branch stmgc-c7 Message-ID: <20150330115227.2A7F61C0207@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: stmgc-c8 Changeset: r76636:a1e882561425 Date: 2015-03-30 10:54 +0200 http://bitbucket.org/pypy/pypy/changeset/a1e882561425/ Log: Merge branch stmgc-c7 diff too long, truncating to 2000 out of 44186 lines diff --git a/.gitignore b/.gitignore --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,10 @@ bin/pypy-c include/*.h +include/numpy/ lib_pypy/ctypes_config_cache/_[^_]*_*.py +libpypy-c.* +pypy-c pypy/_cache pypy/doc/*.html pypy/doc/config/*.html @@ -18,4 +21,5 @@ pypy/translator/c/src/dtoa.o pypy/translator/goal/pypy-c pypy/translator/goal/target*-c -release/ \ No newline at end of file +release/ +rpython/_cache/ diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -7,10 +7,12 @@ 9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm ab0dd631c22015ed88e583d9fdd4c43eebf0be21 pypy-2.1-beta1-arm 20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0 -20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0 -0000000000000000000000000000000000000000 release-2.3.0 394146e9bb673514c61f0150ab2013ccf78e8de7 release-2.3 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1 -32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1 -0000000000000000000000000000000000000000 release-2.2=3.1 +10f1b29a2bd21f837090286174a9ca030b8680b2 release-2.5.0 +8e24dac0b8e2db30d46d59f2c4daa3d4aaab7861 release-2.5.1 +8e24dac0b8e2db30d46d59f2c4daa3d4aaab7861 release-2.5.1 +0000000000000000000000000000000000000000 release-2.5.1 +0000000000000000000000000000000000000000 release-2.5.1 +e3d046c43451403f5969580fc1c41d5df6c4082a release-2.5.1 diff --git a/lib-python/2.7/CGIHTTPServer.py b/lib-python/2.7/CGIHTTPServer.py --- a/lib-python/2.7/CGIHTTPServer.py +++ b/lib-python/2.7/CGIHTTPServer.py @@ -106,16 +106,16 @@ def run_cgi(self): """Execute a CGI script.""" dir, rest = self.cgi_info - - i = rest.find('/') + path = dir + '/' + rest + i = path.find('/', len(dir)+1) while i >= 0: - nextdir = rest[:i] - nextrest = rest[i+1:] + nextdir = path[:i] + nextrest = path[i+1:] scriptdir = self.translate_path(nextdir) if os.path.isdir(scriptdir): dir, rest = nextdir, nextrest - i = rest.find('/') + i = path.find('/', len(dir)+1) else: break diff --git a/lib-python/2.7/Cookie.py b/lib-python/2.7/Cookie.py --- a/lib-python/2.7/Cookie.py +++ b/lib-python/2.7/Cookie.py @@ -56,7 +56,7 @@ >>> C = Cookie.SmartCookie() [Note: Long-time users of Cookie.py will remember using -Cookie.Cookie() to create an Cookie object. Although deprecated, it +Cookie.Cookie() to create a Cookie object. Although deprecated, it is still supported by the code. See the Backward Compatibility notes for more information.] @@ -426,6 +426,8 @@ "version" : "Version", } + _flags = {'secure', 'httponly'} + def __init__(self): # Set defaults self.key = self.value = self.coded_value = None @@ -529,9 +531,11 @@ _LegalCharsPatt = r"[\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=]" _CookiePattern = re.compile( r"(?x)" # This is a Verbose pattern + r"\s*" # Optional whitespace at start of cookie r"(?P" # Start of group 'key' ""+ _LegalCharsPatt +"+?" # Any word of at least one letter, nongreedy r")" # End of group 'key' + r"(" # Optional group: there may not be a value. r"\s*=\s*" # Equal Sign r"(?P" # Start of group 'val' r'"(?:[^\\"]|\\.)*"' # Any doublequoted string @@ -540,7 +544,9 @@ r"|" # or ""+ _LegalCharsPatt +"*" # Any word or empty string r")" # End of group 'val' - r"\s*;?" # Probably ending in a semi-colon + r")?" # End of optional value group + r"\s*" # Any number of spaces. + r"(\s+|;|$)" # Ending either at space, semicolon, or EOS. ) @@ -585,8 +591,12 @@ def __setitem__(self, key, value): """Dictionary style assignment.""" - rval, cval = self.value_encode(value) - self.__set(key, rval, cval) + if isinstance(value, Morsel): + # allow assignment of constructed Morsels (e.g. for pickling) + dict.__setitem__(self, key, value) + else: + rval, cval = self.value_encode(value) + self.__set(key, rval, cval) # end __setitem__ def output(self, attrs=None, header="Set-Cookie:", sep="\015\012"): @@ -641,7 +651,7 @@ while 0 <= i < n: # Start looking for a cookie - match = patt.search(str, i) + match = patt.match(str, i) if not match: break # No more cookies K,V = match.group("key"), match.group("val") @@ -656,8 +666,12 @@ M[ K[1:] ] = V elif K.lower() in Morsel._reserved: if M: - M[ K ] = _unquote(V) - else: + if V is None: + if K.lower() in Morsel._flags: + M[K] = True + else: + M[K] = _unquote(V) + elif V is not None: rval, cval = self.value_decode(V) self.__set(K, rval, cval) M = self[K] diff --git a/lib-python/2.7/SocketServer.py b/lib-python/2.7/SocketServer.py --- a/lib-python/2.7/SocketServer.py +++ b/lib-python/2.7/SocketServer.py @@ -416,8 +416,12 @@ self.socket = socket.socket(self.address_family, self.socket_type) if bind_and_activate: - self.server_bind() - self.server_activate() + try: + self.server_bind() + self.server_activate() + except: + self.server_close() + raise def server_bind(self): """Called by constructor to bind the socket. diff --git a/lib-python/2.7/_abcoll.py b/lib-python/2.7/_abcoll.py --- a/lib-python/2.7/_abcoll.py +++ b/lib-python/2.7/_abcoll.py @@ -143,7 +143,7 @@ methods except for __contains__, __iter__ and __len__. To override the comparisons (presumably for speed, as the - semantics are fixed), all you have to do is redefine __le__ and + semantics are fixed), redefine __le__ and __ge__, then the other operations will automatically follow suit. """ diff --git a/lib-python/2.7/argparse.py b/lib-python/2.7/argparse.py --- a/lib-python/2.7/argparse.py +++ b/lib-python/2.7/argparse.py @@ -1089,7 +1089,14 @@ # parse all the remaining options into the namespace # store any unrecognized options on the object, so that the top # level parser can decide what to do with them - namespace, arg_strings = parser.parse_known_args(arg_strings, namespace) + + # In case this subparser defines new defaults, we parse them + # in a new namespace object and then update the original + # namespace for the relevant parts. + subnamespace, arg_strings = parser.parse_known_args(arg_strings, None) + for key, value in vars(subnamespace).items(): + setattr(namespace, key, value) + if arg_strings: vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, []) getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings) diff --git a/lib-python/2.7/asynchat.py b/lib-python/2.7/asynchat.py --- a/lib-python/2.7/asynchat.py +++ b/lib-python/2.7/asynchat.py @@ -46,12 +46,17 @@ you - by calling your self.found_terminator() method. """ +import asyncore +import errno import socket -import asyncore from collections import deque from sys import py3kwarning from warnings import filterwarnings, catch_warnings +_BLOCKING_IO_ERRORS = (errno.EAGAIN, errno.EALREADY, errno.EINPROGRESS, + errno.EWOULDBLOCK) + + class async_chat (asyncore.dispatcher): """This is an abstract class. You must derive from this class, and add the two methods collect_incoming_data() and found_terminator()""" @@ -109,6 +114,8 @@ try: data = self.recv (self.ac_in_buffer_size) except socket.error, why: + if why.args[0] in _BLOCKING_IO_ERRORS: + return self.handle_error() return diff --git a/lib-python/2.7/bsddb/test/test_queue.py b/lib-python/2.7/bsddb/test/test_queue.py --- a/lib-python/2.7/bsddb/test/test_queue.py +++ b/lib-python/2.7/bsddb/test/test_queue.py @@ -10,6 +10,7 @@ #---------------------------------------------------------------------- + at unittest.skip("fails on Windows; see issue 22943") class SimpleQueueTestCase(unittest.TestCase): def setUp(self): self.filename = get_new_database_path() diff --git a/lib-python/2.7/cookielib.py b/lib-python/2.7/cookielib.py --- a/lib-python/2.7/cookielib.py +++ b/lib-python/2.7/cookielib.py @@ -1719,12 +1719,12 @@ def __repr__(self): r = [] for cookie in self: r.append(repr(cookie)) - return "<%s[%s]>" % (self.__class__, ", ".join(r)) + return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r)) def __str__(self): r = [] for cookie in self: r.append(str(cookie)) - return "<%s[%s]>" % (self.__class__, ", ".join(r)) + return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r)) # derives from IOError for backwards-compatibility with Python 2.4.0 diff --git a/lib-python/2.7/ctypes/test/test_pointers.py b/lib-python/2.7/ctypes/test/test_pointers.py --- a/lib-python/2.7/ctypes/test/test_pointers.py +++ b/lib-python/2.7/ctypes/test/test_pointers.py @@ -7,6 +7,8 @@ c_long, c_ulong, c_longlong, c_ulonglong, c_double, c_float] python_types = [int, int, int, int, int, long, int, long, long, long, float, float] +LargeNamedType = type('T' * 2 ** 25, (Structure,), {}) +large_string = 'T' * 2 ** 25 class PointersTestCase(unittest.TestCase): @@ -188,5 +190,11 @@ mth = WINFUNCTYPE(None)(42, "name", (), None) self.assertEqual(bool(mth), True) + def test_pointer_type_name(self): + self.assertTrue(POINTER(LargeNamedType)) + + def test_pointer_type_str_name(self): + self.assertTrue(POINTER(large_string)) + if __name__ == '__main__': unittest.main() diff --git a/lib-python/2.7/ctypes/test/test_python_api.py b/lib-python/2.7/ctypes/test/test_python_api.py --- a/lib-python/2.7/ctypes/test/test_python_api.py +++ b/lib-python/2.7/ctypes/test/test_python_api.py @@ -46,8 +46,8 @@ # This test is unreliable, because it is possible that code in # unittest changes the refcount of the '42' integer. So, it # is disabled by default. - @requires("refcount") def test_PyInt_Long(self): + requires("refcount") ref42 = grc(42) pythonapi.PyInt_FromLong.restype = py_object self.assertEqual(pythonapi.PyInt_FromLong(42), 42) diff --git a/lib-python/2.7/ctypes/test/test_win32.py b/lib-python/2.7/ctypes/test/test_win32.py --- a/lib-python/2.7/ctypes/test/test_win32.py +++ b/lib-python/2.7/ctypes/test/test_win32.py @@ -38,8 +38,11 @@ @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') class FunctionCallTestCase(unittest.TestCase): - @requires("SEH") + @unittest.skipUnless('MSC' in sys.version, "SEH only supported by MSC") + @unittest.skipIf(sys.executable.endswith('_d.exe'), + "SEH not enabled in debug builds") def test_SEH(self): + requires("SEH") # Call functions with invalid arguments, and make sure # that access violations are trapped and raise an # exception. @@ -87,9 +90,29 @@ dll = CDLL(_ctypes_test.__file__) - pt = POINT(10, 10) - rect = RECT(0, 0, 20, 20) - self.assertEqual(1, dll.PointInRect(byref(rect), pt)) + pt = POINT(15, 25) + left = c_long.in_dll(dll, 'left') + top = c_long.in_dll(dll, 'top') + right = c_long.in_dll(dll, 'right') + bottom = c_long.in_dll(dll, 'bottom') + rect = RECT(left, top, right, bottom) + PointInRect = dll.PointInRect + PointInRect.argtypes = [POINTER(RECT), POINT] + self.assertEqual(1, PointInRect(byref(rect), pt)) + + ReturnRect = dll.ReturnRect + ReturnRect.argtypes = [c_int, RECT, POINTER(RECT), POINT, RECT, + POINTER(RECT), POINT, RECT] + ReturnRect.restype = RECT + for i in range(4): + ret = ReturnRect(i, rect, pointer(rect), pt, rect, + byref(rect), pt, rect) + # the c function will check and modify ret if something is + # passed in improperly + self.assertEqual(ret.left, left.value) + self.assertEqual(ret.right, right.value) + self.assertEqual(ret.top, top.value) + self.assertEqual(ret.bottom, bottom.value) if __name__ == '__main__': unittest.main() diff --git a/lib-python/2.7/decimal.py b/lib-python/2.7/decimal.py --- a/lib-python/2.7/decimal.py +++ b/lib-python/2.7/decimal.py @@ -136,7 +136,6 @@ __version__ = '1.70' # Highest version of the spec this complies with -import copy as _copy import math as _math import numbers as _numbers @@ -3665,6 +3664,8 @@ if self._is_special: sign = _format_sign(self._sign, spec) body = str(self.copy_abs()) + if spec['type'] == '%': + body += '%' return _format_align(sign, body, spec) # a type of None defaults to 'g' or 'G', depending on context @@ -6033,7 +6034,10 @@ format_dict['decimal_point'] = '.' # record whether return type should be str or unicode - format_dict['unicode'] = isinstance(format_spec, unicode) + try: + format_dict['unicode'] = isinstance(format_spec, unicode) + except NameError: + format_dict['unicode'] = False return format_dict diff --git a/lib-python/2.7/distutils/__init__.py b/lib-python/2.7/distutils/__init__.py --- a/lib-python/2.7/distutils/__init__.py +++ b/lib-python/2.7/distutils/__init__.py @@ -15,5 +15,5 @@ # Updated automatically by the Python release process. # #--start constants-- -__version__ = "2.7.8" +__version__ = "2.7.9" #--end constants-- diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py --- a/lib-python/2.7/distutils/command/build_ext.py +++ b/lib-python/2.7/distutils/command/build_ext.py @@ -245,7 +245,7 @@ # Python's library directory must be appended to library_dirs # See Issues: #1600860, #4366 if (sysconfig.get_config_var('Py_ENABLE_SHARED')): - if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")): + if not sysconfig.python_build: # building third party extensions self.library_dirs.append(sysconfig.get_config_var('LIBDIR')) else: diff --git a/lib-python/2.7/distutils/command/upload.py b/lib-python/2.7/distutils/command/upload.py --- a/lib-python/2.7/distutils/command/upload.py +++ b/lib-python/2.7/distutils/command/upload.py @@ -136,8 +136,8 @@ # Build up the MIME payload for the POST data boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254' - sep_boundary = '\n--' + boundary - end_boundary = sep_boundary + '--' + sep_boundary = '\r\n--' + boundary + end_boundary = sep_boundary + '--\r\n' body = StringIO.StringIO() for key, value in data.items(): # handle multiple entries for the same name @@ -151,14 +151,13 @@ fn = "" body.write(sep_boundary) - body.write('\nContent-Disposition: form-data; name="%s"'%key) + body.write('\r\nContent-Disposition: form-data; name="%s"' % key) body.write(fn) - body.write("\n\n") + body.write("\r\n\r\n") body.write(value) if value and value[-1] == '\r': body.write('\n') # write an extra newline (lurve Macs) body.write(end_boundary) - body.write("\n") body = body.getvalue() self.announce("Submitting %s to %s" % (filename, self.repository), log.INFO) diff --git a/lib-python/2.7/distutils/file_util.py b/lib-python/2.7/distutils/file_util.py --- a/lib-python/2.7/distutils/file_util.py +++ b/lib-python/2.7/distutils/file_util.py @@ -85,7 +85,8 @@ (os.symlink) instead of copying: set it to "hard" or "sym"; if it is None (the default), files are copied. Don't set 'link' on systems that don't support it: 'copy_file()' doesn't check if hard or symbolic - linking is available. + linking is available. If hardlink fails, falls back to + _copy_file_contents(). Under Mac OS, uses the native file copy function in macostools; on other systems, uses '_copy_file_contents()' to copy file contents. @@ -137,24 +138,31 @@ # (Unix only, of course, but that's the caller's responsibility) if link == 'hard': if not (os.path.exists(dst) and os.path.samefile(src, dst)): - os.link(src, dst) + try: + os.link(src, dst) + return (dst, 1) + except OSError: + # If hard linking fails, fall back on copying file + # (some special filesystems don't support hard linking + # even under Unix, see issue #8876). + pass elif link == 'sym': if not (os.path.exists(dst) and os.path.samefile(src, dst)): os.symlink(src, dst) + return (dst, 1) # Otherwise (non-Mac, not linking), copy the file contents and # (optionally) copy the times and mode. - else: - _copy_file_contents(src, dst) - if preserve_mode or preserve_times: - st = os.stat(src) + _copy_file_contents(src, dst) + if preserve_mode or preserve_times: + st = os.stat(src) - # According to David Ascher , utime() should be done - # before chmod() (at least under NT). - if preserve_times: - os.utime(dst, (st[ST_ATIME], st[ST_MTIME])) - if preserve_mode: - os.chmod(dst, S_IMODE(st[ST_MODE])) + # According to David Ascher , utime() should be done + # before chmod() (at least under NT). + if preserve_times: + os.utime(dst, (st[ST_ATIME], st[ST_MTIME])) + if preserve_mode: + os.chmod(dst, S_IMODE(st[ST_MODE])) return (dst, 1) diff --git a/lib-python/2.7/distutils/sysconfig_cpython.py b/lib-python/2.7/distutils/sysconfig_cpython.py --- a/lib-python/2.7/distutils/sysconfig_cpython.py +++ b/lib-python/2.7/distutils/sysconfig_cpython.py @@ -165,7 +165,8 @@ # version and build tools may not support the same set # of CPU architectures for universal builds. global _config_vars - if not _config_vars.get('CUSTOMIZED_OSX_COMPILER', ''): + # Use get_config_var() to ensure _config_vars is initialized. + if not get_config_var('CUSTOMIZED_OSX_COMPILER'): import _osx_support _osx_support.customize_compiler(_config_vars) _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True' diff --git a/lib-python/2.7/distutils/tests/test_bdist_rpm.py b/lib-python/2.7/distutils/tests/test_bdist_rpm.py --- a/lib-python/2.7/distutils/tests/test_bdist_rpm.py +++ b/lib-python/2.7/distutils/tests/test_bdist_rpm.py @@ -25,6 +25,7 @@ """ class BuildRpmTestCase(support.TempdirManager, + support.EnvironGuard, support.LoggingSilencer, unittest.TestCase): @@ -50,6 +51,7 @@ def test_quiet(self): # let's create a package tmp_dir = self.mkdtemp() + os.environ['HOME'] = tmp_dir # to confine dir '.rpmdb' creation pkg_dir = os.path.join(tmp_dir, 'foo') os.mkdir(pkg_dir) self.write_file((pkg_dir, 'setup.py'), SETUP_PY) @@ -92,6 +94,7 @@ def test_no_optimize_flag(self): # let's create a package that brakes bdist_rpm tmp_dir = self.mkdtemp() + os.environ['HOME'] = tmp_dir # to confine dir '.rpmdb' creation pkg_dir = os.path.join(tmp_dir, 'foo') os.mkdir(pkg_dir) self.write_file((pkg_dir, 'setup.py'), SETUP_PY) diff --git a/lib-python/2.7/distutils/tests/test_dist.py b/lib-python/2.7/distutils/tests/test_dist.py --- a/lib-python/2.7/distutils/tests/test_dist.py +++ b/lib-python/2.7/distutils/tests/test_dist.py @@ -11,7 +11,7 @@ from distutils.dist import Distribution, fix_help_options from distutils.cmd import Command import distutils.dist -from test.test_support import TESTFN, captured_stdout, run_unittest +from test.test_support import TESTFN, captured_stdout, run_unittest, unlink from distutils.tests import support @@ -64,6 +64,7 @@ with open(TESTFN, "w") as f: f.write("[global]\n") f.write("command_packages = foo.bar, splat") + self.addCleanup(unlink, TESTFN) files = [TESTFN] sys.argv.append("build") diff --git a/lib-python/2.7/distutils/tests/test_file_util.py b/lib-python/2.7/distutils/tests/test_file_util.py --- a/lib-python/2.7/distutils/tests/test_file_util.py +++ b/lib-python/2.7/distutils/tests/test_file_util.py @@ -8,6 +8,11 @@ from distutils.tests import support from test.test_support import run_unittest + +requires_os_link = unittest.skipUnless(hasattr(os, "link"), + "test requires os.link()") + + class FileUtilTestCase(support.TempdirManager, unittest.TestCase): def _log(self, msg, *args): @@ -74,6 +79,44 @@ copy_file(foo, dst_dir) self.assertTrue(os.path.exists(os.path.join(dst_dir, 'foo'))) + @requires_os_link + def test_copy_file_hard_link(self): + with open(self.source, 'w') as f: + f.write('some content') + st = os.stat(self.source) + copy_file(self.source, self.target, link='hard') + st2 = os.stat(self.source) + st3 = os.stat(self.target) + self.assertTrue(os.path.samestat(st, st2), (st, st2)) + self.assertTrue(os.path.samestat(st2, st3), (st2, st3)) + with open(self.source, 'r') as f: + self.assertEqual(f.read(), 'some content') + + @requires_os_link + def test_copy_file_hard_link_failure(self): + # If hard linking fails, copy_file() falls back on copying file + # (some special filesystems don't support hard linking even under + # Unix, see issue #8876). + with open(self.source, 'w') as f: + f.write('some content') + st = os.stat(self.source) + def _os_link(*args): + raise OSError(0, "linking unsupported") + old_link = os.link + os.link = _os_link + try: + copy_file(self.source, self.target, link='hard') + finally: + os.link = old_link + st2 = os.stat(self.source) + st3 = os.stat(self.target) + self.assertTrue(os.path.samestat(st, st2), (st, st2)) + self.assertFalse(os.path.samestat(st2, st3), (st2, st3)) + for fn in (self.source, self.target): + with open(fn, 'r') as f: + self.assertEqual(f.read(), 'some content') + + def test_suite(): return unittest.makeSuite(FileUtilTestCase) diff --git a/lib-python/2.7/distutils/tests/test_sysconfig.py b/lib-python/2.7/distutils/tests/test_sysconfig.py --- a/lib-python/2.7/distutils/tests/test_sysconfig.py +++ b/lib-python/2.7/distutils/tests/test_sysconfig.py @@ -3,6 +3,9 @@ import test import unittest import shutil +import subprocess +import sys +import textwrap from distutils import sysconfig from distutils.tests import support @@ -99,6 +102,24 @@ self.assertEqual(global_sysconfig.get_config_var('LDSHARED'), sysconfig.get_config_var('LDSHARED')) self.assertEqual(global_sysconfig.get_config_var('CC'), sysconfig.get_config_var('CC')) + def test_customize_compiler_before_get_config_vars(self): + # Issue #21923: test that a Distribution compiler + # instance can be called without an explicit call to + # get_config_vars(). + with open(TESTFN, 'w') as f: + f.writelines(textwrap.dedent('''\ + from distutils.core import Distribution + config = Distribution().get_command_obj('config') + # try_compile may pass or it may fail if no compiler + # is found but it should not raise an exception. + rc = config.try_compile('int x;') + ''')) + p = subprocess.Popen([str(sys.executable), TESTFN], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True) + outs, errs = p.communicate() + self.assertEqual(0, p.returncode, "Subprocess failed: " + outs) def test_suite(): diff --git a/lib-python/2.7/distutils/tests/test_upload.py b/lib-python/2.7/distutils/tests/test_upload.py --- a/lib-python/2.7/distutils/tests/test_upload.py +++ b/lib-python/2.7/distutils/tests/test_upload.py @@ -119,7 +119,7 @@ # what did we send ? self.assertIn('dédé', self.last_open.req.data) headers = dict(self.last_open.req.headers) - self.assertEqual(headers['Content-length'], '2085') + self.assertEqual(headers['Content-length'], '2159') self.assertTrue(headers['Content-type'].startswith('multipart/form-data')) self.assertEqual(self.last_open.req.get_method(), 'POST') self.assertEqual(self.last_open.req.get_full_url(), diff --git a/lib-python/2.7/doctest.py b/lib-python/2.7/doctest.py --- a/lib-python/2.7/doctest.py +++ b/lib-python/2.7/doctest.py @@ -216,7 +216,7 @@ # get_data() opens files as 'rb', so one must do the equivalent # conversion as universal newlines would do. return file_contents.replace(os.linesep, '\n'), filename - with open(filename) as f: + with open(filename, 'U') as f: return f.read(), filename # Use sys.stdout encoding for ouput. diff --git a/lib-python/2.7/email/feedparser.py b/lib-python/2.7/email/feedparser.py --- a/lib-python/2.7/email/feedparser.py +++ b/lib-python/2.7/email/feedparser.py @@ -49,8 +49,8 @@ simple abstraction -- it parses until EOF closes the current message. """ def __init__(self): - # The last partial line pushed into this object. - self._partial = '' + # Chunks of the last partial line pushed into this object. + self._partial = [] # The list of full, pushed lines, in reverse order self._lines = [] # The stack of false-EOF checking predicates. @@ -66,8 +66,8 @@ def close(self): # Don't forget any trailing partial line. - self._lines.append(self._partial) - self._partial = '' + self.pushlines(''.join(self._partial).splitlines(True)) + self._partial = [] self._closed = True def readline(self): @@ -95,8 +95,29 @@ def push(self, data): """Push some new data into this object.""" - # Handle any previous leftovers - data, self._partial = self._partial + data, '' + # Crack into lines, but preserve the linesep characters on the end of each + parts = data.splitlines(True) + + if not parts or not parts[0].endswith(('\n', '\r')): + # No new complete lines, so just accumulate partials + self._partial += parts + return + + if self._partial: + # If there are previous leftovers, complete them now + self._partial.append(parts[0]) + parts[0:1] = ''.join(self._partial).splitlines(True) + del self._partial[:] + + # If the last element of the list does not end in a newline, then treat + # it as a partial line. We only check for '\n' here because a line + # ending with '\r' might be a line that was split in the middle of a + # '\r\n' sequence (see bugs 1555570 and 1721862). + if not parts[-1].endswith('\n'): + self._partial = [parts.pop()] + self.pushlines(parts) + + def pushlines(self, lines): # Crack into lines, but preserve the newlines on the end of each parts = NLCRE_crack.split(data) # The *ahem* interesting behaviour of re.split when supplied grouping diff --git a/lib-python/2.7/email/mime/nonmultipart.py b/lib-python/2.7/email/mime/nonmultipart.py --- a/lib-python/2.7/email/mime/nonmultipart.py +++ b/lib-python/2.7/email/mime/nonmultipart.py @@ -12,7 +12,7 @@ class MIMENonMultipart(MIMEBase): - """Base class for MIME multipart/* type messages.""" + """Base class for MIME non-multipart type messages.""" def attach(self, payload): # The public API prohibits attaching multiple subparts to MIMEBase diff --git a/lib-python/2.7/email/test/test_email.py b/lib-python/2.7/email/test/test_email.py --- a/lib-python/2.7/email/test/test_email.py +++ b/lib-python/2.7/email/test/test_email.py @@ -11,6 +11,7 @@ import warnings import textwrap from cStringIO import StringIO +from random import choice import email @@ -2578,16 +2579,64 @@ bsf.push(il) nt += n n1 = 0 - while True: - ol = bsf.readline() - if ol == NeedMoreData: - break + for ol in iter(bsf.readline, NeedMoreData): om.append(ol) n1 += 1 self.assertEqual(n, n1) self.assertEqual(len(om), nt) self.assertEqual(''.join([il for il, n in imt]), ''.join(om)) + def test_push_random(self): + from email.feedparser import BufferedSubFile, NeedMoreData + + n = 10000 + chunksize = 5 + chars = 'abcd \t\r\n' + + s = ''.join(choice(chars) for i in range(n)) + '\n' + target = s.splitlines(True) + + bsf = BufferedSubFile() + lines = [] + for i in range(0, len(s), chunksize): + chunk = s[i:i+chunksize] + bsf.push(chunk) + lines.extend(iter(bsf.readline, NeedMoreData)) + self.assertEqual(lines, target) + + +class TestFeedParsers(TestEmailBase): + + def parse(self, chunks): + from email.feedparser import FeedParser + feedparser = FeedParser() + for chunk in chunks: + feedparser.feed(chunk) + return feedparser.close() + + def test_newlines(self): + m = self.parse(['a:\nb:\rc:\r\nd:\n']) + self.assertEqual(m.keys(), ['a', 'b', 'c', 'd']) + m = self.parse(['a:\nb:\rc:\r\nd:']) + self.assertEqual(m.keys(), ['a', 'b', 'c', 'd']) + m = self.parse(['a:\rb', 'c:\n']) + self.assertEqual(m.keys(), ['a', 'bc']) + m = self.parse(['a:\r', 'b:\n']) + self.assertEqual(m.keys(), ['a', 'b']) + m = self.parse(['a:\r', '\nb:\n']) + self.assertEqual(m.keys(), ['a', 'b']) + + def test_long_lines(self): + # Expected peak memory use on 32-bit platform: 4*N*M bytes. + M, N = 1000, 20000 + m = self.parse(['a:b\n\n'] + ['x'*M] * N) + self.assertEqual(m.items(), [('a', 'b')]) + self.assertEqual(m.get_payload(), 'x'*M*N) + m = self.parse(['a:b\r\r'] + ['x'*M] * N) + self.assertEqual(m.items(), [('a', 'b')]) + self.assertEqual(m.get_payload(), 'x'*M*N) + m = self.parse(['a:\r', 'b: '] + ['x'*M] * N) + self.assertEqual(m.items(), [('a', ''), ('b', 'x'*M*N)]) class TestParsers(TestEmailBase): @@ -3180,7 +3229,6 @@ self.assertEqual(res, '=?iso-8859-2?q?abc?=') self.assertIsInstance(res, str) - # Test RFC 2231 header parameters (en/de)coding class TestRFC2231(TestEmailBase): def test_get_param(self): diff --git a/lib-python/2.7/ensurepip/__init__.py b/lib-python/2.7/ensurepip/__init__.py new file mode 100644 --- /dev/null +++ b/lib-python/2.7/ensurepip/__init__.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python2 +from __future__ import print_function + +import os +import os.path +import pkgutil +import shutil +import sys +import tempfile + + +__all__ = ["version", "bootstrap"] + + +_SETUPTOOLS_VERSION = "7.0" + +_PIP_VERSION = "1.5.6" + +# pip currently requires ssl support, so we try to provide a nicer +# error message when that is missing (http://bugs.python.org/issue19744) +_MISSING_SSL_MESSAGE = ("pip {} requires SSL/TLS".format(_PIP_VERSION)) +try: + import ssl +except ImportError: + ssl = None + + def _require_ssl_for_pip(): + raise RuntimeError(_MISSING_SSL_MESSAGE) +else: + def _require_ssl_for_pip(): + pass + +_PROJECTS = [ + ("setuptools", _SETUPTOOLS_VERSION), + ("pip", _PIP_VERSION), +] + + +def _run_pip(args, additional_paths=None): + # Add our bundled software to the sys.path so we can import it + if additional_paths is not None: + sys.path = additional_paths + sys.path + + # Install the bundled software + import pip + pip.main(args) + + +def version(): + """ + Returns a string specifying the bundled version of pip. + """ + return _PIP_VERSION + + +def _disable_pip_configuration_settings(): + # We deliberately ignore all pip environment variables + # when invoking pip + # See http://bugs.python.org/issue19734 for details + keys_to_remove = [k for k in os.environ if k.startswith("PIP_")] + for k in keys_to_remove: + del os.environ[k] + # We also ignore the settings in the default pip configuration file + # See http://bugs.python.org/issue20053 for details + os.environ['PIP_CONFIG_FILE'] = os.devnull + + +def bootstrap(root=None, upgrade=False, user=False, + altinstall=False, default_pip=True, + verbosity=0): + """ + Bootstrap pip into the current Python installation (or the given root + directory). + + Note that calling this function will alter both sys.path and os.environ. + """ + if altinstall and default_pip: + raise ValueError("Cannot use altinstall and default_pip together") + + _require_ssl_for_pip() + _disable_pip_configuration_settings() + + # By default, installing pip and setuptools installs all of the + # following scripts (X.Y == running Python version): + # + # pip, pipX, pipX.Y, easy_install, easy_install-X.Y + # + # pip 1.5+ allows ensurepip to request that some of those be left out + if altinstall: + # omit pip, pipX and easy_install + os.environ["ENSUREPIP_OPTIONS"] = "altinstall" + elif not default_pip: + # omit pip and easy_install + os.environ["ENSUREPIP_OPTIONS"] = "install" + + tmpdir = tempfile.mkdtemp() + try: + # Put our bundled wheels into a temporary directory and construct the + # additional paths that need added to sys.path + additional_paths = [] + for project, version in _PROJECTS: + wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version) + whl = pkgutil.get_data( + "ensurepip", + "_bundled/{}".format(wheel_name), + ) + with open(os.path.join(tmpdir, wheel_name), "wb") as fp: + fp.write(whl) + + additional_paths.append(os.path.join(tmpdir, wheel_name)) + + # Construct the arguments to be passed to the pip command + args = ["install", "--no-index", "--find-links", tmpdir] + if root: + args += ["--root", root] + if upgrade: + args += ["--upgrade"] + if user: + args += ["--user"] + if verbosity: + args += ["-" + "v" * verbosity] + + _run_pip(args + [p[0] for p in _PROJECTS], additional_paths) + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + +def _uninstall_helper(verbosity=0): + """Helper to support a clean default uninstall process on Windows + + Note that calling this function may alter os.environ. + """ + # Nothing to do if pip was never installed, or has been removed + try: + import pip + except ImportError: + return + + # If the pip version doesn't match the bundled one, leave it alone + if pip.__version__ != _PIP_VERSION: + msg = ("ensurepip will only uninstall a matching version " + "({!r} installed, {!r} bundled)") + print(msg.format(pip.__version__, _PIP_VERSION), file=sys.stderr) + return + + _require_ssl_for_pip() + _disable_pip_configuration_settings() + + # Construct the arguments to be passed to the pip command + args = ["uninstall", "-y"] + if verbosity: + args += ["-" + "v" * verbosity] + + _run_pip(args + [p[0] for p in reversed(_PROJECTS)]) + + +def _main(argv=None): + if ssl is None: + print("Ignoring ensurepip failure: {}".format(_MISSING_SSL_MESSAGE), + file=sys.stderr) + return + + import argparse + parser = argparse.ArgumentParser(prog="python -m ensurepip") + parser.add_argument( + "--version", + action="version", + version="pip {}".format(version()), + help="Show the version of pip that is bundled with this Python.", + ) + parser.add_argument( + "-v", "--verbose", + action="count", + default=0, + dest="verbosity", + help=("Give more output. Option is additive, and can be used up to 3 " + "times."), + ) + parser.add_argument( + "-U", "--upgrade", + action="store_true", + default=False, + help="Upgrade pip and dependencies, even if already installed.", + ) + parser.add_argument( + "--user", + action="store_true", + default=False, + help="Install using the user scheme.", + ) + parser.add_argument( + "--root", + default=None, + help="Install everything relative to this alternate root directory.", + ) + parser.add_argument( + "--altinstall", + action="store_true", + default=False, + help=("Make an alternate install, installing only the X.Y versioned" + "scripts (Default: pipX, pipX.Y, easy_install-X.Y)"), + ) + parser.add_argument( + "--default-pip", + action="store_true", + default=True, + dest="default_pip", + help=argparse.SUPPRESS, + ) + parser.add_argument( + "--no-default-pip", + action="store_false", + dest="default_pip", + help=("Make a non default install, installing only the X and X.Y " + "versioned scripts."), + ) + + args = parser.parse_args(argv) + + bootstrap( + root=args.root, + upgrade=args.upgrade, + user=args.user, + verbosity=args.verbosity, + altinstall=args.altinstall, + default_pip=args.default_pip, + ) diff --git a/lib-python/2.7/ensurepip/__main__.py b/lib-python/2.7/ensurepip/__main__.py new file mode 100644 --- /dev/null +++ b/lib-python/2.7/ensurepip/__main__.py @@ -0,0 +1,4 @@ +import ensurepip + +if __name__ == "__main__": + ensurepip._main() diff --git a/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..097ab43430d4c1302b0be353a8c16407c370693b GIT binary patch [cut] diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..fa1d1054da1dab98f8906555d31a9fda271b3a85 GIT binary patch [cut] diff --git a/lib-python/2.7/ensurepip/_uninstall.py b/lib-python/2.7/ensurepip/_uninstall.py new file mode 100644 --- /dev/null +++ b/lib-python/2.7/ensurepip/_uninstall.py @@ -0,0 +1,30 @@ +"""Basic pip uninstallation support, helper for the Windows uninstaller""" + +import argparse +import ensurepip + + +def _main(argv=None): + parser = argparse.ArgumentParser(prog="python -m ensurepip._uninstall") + parser.add_argument( + "--version", + action="version", + version="pip {}".format(ensurepip.version()), + help="Show the version of pip this will attempt to uninstall.", + ) + parser.add_argument( + "-v", "--verbose", + action="count", + default=0, + dest="verbosity", + help=("Give more output. Option is additive, and can be used up to 3 " + "times."), + ) + + args = parser.parse_args(argv) + + ensurepip._uninstall_helper(verbosity=args.verbosity) + + +if __name__ == "__main__": + _main() diff --git a/lib-python/2.7/glob.py b/lib-python/2.7/glob.py --- a/lib-python/2.7/glob.py +++ b/lib-python/2.7/glob.py @@ -35,11 +35,16 @@ patterns. """ + dirname, basename = os.path.split(pathname) if not has_magic(pathname): - if os.path.lexists(pathname): - yield pathname + if basename: + if os.path.lexists(pathname): + yield pathname + else: + # Patterns ending with a slash should match only directories + if os.path.isdir(dirname): + yield pathname return - dirname, basename = os.path.split(pathname) if not dirname: for name in glob1(os.curdir, basename): yield name diff --git a/lib-python/2.7/gzip.py b/lib-python/2.7/gzip.py --- a/lib-python/2.7/gzip.py +++ b/lib-python/2.7/gzip.py @@ -164,9 +164,16 @@ def _write_gzip_header(self): self.fileobj.write('\037\213') # magic header self.fileobj.write('\010') # compression method - fname = os.path.basename(self.name) - if fname.endswith(".gz"): - fname = fname[:-3] + try: + # RFC 1952 requires the FNAME field to be Latin-1. Do not + # include filenames that cannot be represented that way. + fname = os.path.basename(self.name) + if not isinstance(fname, str): + fname = fname.encode('latin-1') + if fname.endswith('.gz'): + fname = fname[:-3] + except UnicodeEncodeError: + fname = '' flags = 0 if fname: flags = FNAME diff --git a/lib-python/2.7/hashlib.py b/lib-python/2.7/hashlib.py --- a/lib-python/2.7/hashlib.py +++ b/lib-python/2.7/hashlib.py @@ -15,8 +15,9 @@ md5(), sha1(), sha224(), sha256(), sha384(), and sha512() -More algorithms may be available on your platform but the above are -guaranteed to exist. +More algorithms may be available on your platform but the above are guaranteed +to exist. See the algorithms_guaranteed and algorithms_available attributes +to find out what algorithm names can be passed to new(). NOTE: If you want the adler32 or crc32 hash functions they are available in the zlib module. @@ -58,9 +59,14 @@ # always available algorithm is added. __always_supported = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512') +algorithms_guaranteed = set(__always_supported) +algorithms_available = set(__always_supported) + algorithms = __always_supported -__all__ = __always_supported + ('new', 'algorithms', 'pbkdf2_hmac') +__all__ = __always_supported + ('new', 'algorithms_guaranteed', + 'algorithms_available', 'algorithms', + 'pbkdf2_hmac') def __get_builtin_constructor(name): @@ -128,6 +134,8 @@ import _hashlib new = __hash_new __get_hash = __get_openssl_constructor + algorithms_available = algorithms_available.union( + _hashlib.openssl_md_meth_names) except ImportError: new = __py_new __get_hash = __get_builtin_constructor diff --git a/lib-python/2.7/httplib.py b/lib-python/2.7/httplib.py --- a/lib-python/2.7/httplib.py +++ b/lib-python/2.7/httplib.py @@ -215,6 +215,10 @@ # maximal line length when calling readline(). _MAXLINE = 65536 +# maximum amount of headers accepted +_MAXHEADERS = 100 + + class HTTPMessage(mimetools.Message): def addheader(self, key, value): @@ -271,6 +275,8 @@ elif self.seekable: tell = self.fp.tell while True: + if len(hlist) > _MAXHEADERS: + raise HTTPException("got more than %d headers" % _MAXHEADERS) if tell: try: startofline = tell() @@ -1185,21 +1191,29 @@ def __init__(self, host, port=None, key_file=None, cert_file=None, strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - source_address=None): + source_address=None, context=None): HTTPConnection.__init__(self, host, port, strict, timeout, source_address) self.key_file = key_file self.cert_file = cert_file + if context is None: + context = ssl._create_default_https_context() + if key_file or cert_file: + context.load_cert_chain(cert_file, key_file) + self._context = context def connect(self): "Connect to a host on a given (SSL) port." - sock = self._create_connection((self.host, self.port), - self.timeout, self.source_address) + HTTPConnection.connect(self) + if self._tunnel_host: - self.sock = sock - self._tunnel() - self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file) + server_hostname = self._tunnel_host + else: + server_hostname = self.host + + self.sock = self._context.wrap_socket(self.sock, + server_hostname=server_hostname) __all__.append("HTTPSConnection") @@ -1214,14 +1228,15 @@ _connection_class = HTTPSConnection def __init__(self, host='', port=None, key_file=None, cert_file=None, - strict=None): + strict=None, context=None): # provide a default host, pass the X509 cert info # urf. compensate for bad input. if port == 0: port = None self._setup(self._connection_class(host, port, key_file, - cert_file, strict)) + cert_file, strict, + context=context)) # we never actually use these for anything, but we keep them # here for compatibility with post-1.5.2 CVS. diff --git a/lib-python/2.7/idlelib/Bindings.py b/lib-python/2.7/idlelib/Bindings.py --- a/lib-python/2.7/idlelib/Bindings.py +++ b/lib-python/2.7/idlelib/Bindings.py @@ -75,7 +75,8 @@ ('!_Auto-open Stack Viewer', '<>'), ]), ('options', [ - ('_Configure IDLE...', '<>'), + ('Configure _IDLE', '<>'), + ('Configure _Extensions', '<>'), None, ]), ('help', [ diff --git a/lib-python/2.7/idlelib/CallTipWindow.py b/lib-python/2.7/idlelib/CallTipWindow.py --- a/lib-python/2.7/idlelib/CallTipWindow.py +++ b/lib-python/2.7/idlelib/CallTipWindow.py @@ -2,9 +2,8 @@ After ToolTip.py, which uses ideas gleaned from PySol Used by the CallTips IDLE extension. - """ -from Tkinter import * +from Tkinter import Toplevel, Label, LEFT, SOLID, TclError HIDE_VIRTUAL_EVENT_NAME = "<>" HIDE_SEQUENCES = ("", "") @@ -133,35 +132,28 @@ return bool(self.tipwindow) -def _calltip_window(parent): - root = Tk() - root.title("Test calltips") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) +def _calltip_window(parent): # htest # + from Tkinter import Toplevel, Text, LEFT, BOTH - class MyEditWin: # comparenceptually an editor_window - def __init__(self): - text = self.text = Text(root) - text.pack(side=LEFT, fill=BOTH, expand=1) - text.insert("insert", "string.split") - root.update() - self.calltip = CallTip(text) + top = Toplevel(parent) + top.title("Test calltips") + top.geometry("200x100+%d+%d" % (parent.winfo_rootx() + 200, + parent.winfo_rooty() + 150)) + text = Text(top) + text.pack(side=LEFT, fill=BOTH, expand=1) + text.insert("insert", "string.split") + top.update() + calltip = CallTip(text) - text.event_add("<>", "(") - text.event_add("<>", ")") - text.bind("<>", self.calltip_show) - text.bind("<>", self.calltip_hide) - - text.focus_set() - root.mainloop() - - def calltip_show(self, event): - self.calltip.showtip("Hello world", "insert", "end") - - def calltip_hide(self, event): - self.calltip.hidetip() - - editwin = MyEditWin() + def calltip_show(event): + calltip.showtip("(s=Hello world)", "insert", "end") + def calltip_hide(event): + calltip.hidetip() + text.event_add("<>", "(") + text.event_add("<>", ")") + text.bind("<>", calltip_show) + text.bind("<>", calltip_hide) + text.focus_set() if __name__=='__main__': from idlelib.idle_test.htest import run diff --git a/lib-python/2.7/idlelib/ClassBrowser.py b/lib-python/2.7/idlelib/ClassBrowser.py --- a/lib-python/2.7/idlelib/ClassBrowser.py +++ b/lib-python/2.7/idlelib/ClassBrowser.py @@ -19,6 +19,9 @@ from idlelib.TreeWidget import TreeNode, TreeItem, ScrolledCanvas from idlelib.configHandler import idleConf +file_open = None # Method...Item and Class...Item use this. +# Normally PyShell.flist.open, but there is no PyShell.flist for htest. + class ClassBrowser: def __init__(self, flist, name, path, _htest=False): @@ -27,6 +30,9 @@ """ _htest - bool, change box when location running htest. """ + global file_open + if not _htest: + file_open = PyShell.flist.open self.name = name self.file = os.path.join(path[0], self.name + ".py") self._htest = _htest @@ -101,7 +107,7 @@ return [] try: dict = pyclbr.readmodule_ex(name, [dir] + sys.path) - except ImportError, msg: + except ImportError: return [] items = [] self.classes = {} @@ -170,7 +176,7 @@ def OnDoubleClick(self): if not os.path.exists(self.file): return - edit = PyShell.flist.open(self.file) + edit = file_open(self.file) if hasattr(self.cl, 'lineno'): lineno = self.cl.lineno edit.gotoline(lineno) @@ -206,7 +212,7 @@ def OnDoubleClick(self): if not os.path.exists(self.file): return - edit = PyShell.flist.open(self.file) + edit = file_open(self.file) edit.gotoline(self.cl.methods[self.name]) def _class_browser(parent): #Wrapper for htest @@ -221,8 +227,9 @@ dir, file = os.path.split(file) name = os.path.splitext(file)[0] flist = PyShell.PyShellFileList(parent) + global file_open + file_open = flist.open ClassBrowser(flist, name, [dir], _htest=True) - parent.mainloop() if __name__ == "__main__": from idlelib.idle_test.htest import run diff --git a/lib-python/2.7/idlelib/ColorDelegator.py b/lib-python/2.7/idlelib/ColorDelegator.py --- a/lib-python/2.7/idlelib/ColorDelegator.py +++ b/lib-python/2.7/idlelib/ColorDelegator.py @@ -2,7 +2,6 @@ import re import keyword import __builtin__ -from Tkinter import * from idlelib.Delegator import Delegator from idlelib.configHandler import idleConf @@ -34,7 +33,6 @@ prog = re.compile(make_pat(), re.S) idprog = re.compile(r"\s+(\w+)", re.S) -asprog = re.compile(r".*?\b(as)\b") class ColorDelegator(Delegator): @@ -42,7 +40,6 @@ Delegator.__init__(self) self.prog = prog self.idprog = idprog - self.asprog = asprog self.LoadTagDefs() def setdelegate(self, delegate): @@ -74,7 +71,6 @@ "DEFINITION": idleConf.GetHighlight(theme, "definition"), "SYNC": {'background':None,'foreground':None}, "TODO": {'background':None,'foreground':None}, - "BREAK": idleConf.GetHighlight(theme, "break"), "ERROR": idleConf.GetHighlight(theme, "error"), # The following is used by ReplaceDialog: "hit": idleConf.GetHighlight(theme, "hit"), @@ -216,22 +212,6 @@ self.tag_add("DEFINITION", head + "+%dc" % a, head + "+%dc" % b) - elif value == "import": - # color all the "as" words on same line, except - # if in a comment; cheap approximation to the - # truth - if '#' in chars: - endpos = chars.index('#') - else: - endpos = len(chars) - while True: - m1 = self.asprog.match(chars, b, endpos) - if not m1: - break - a, b = m1.span(1) - self.tag_add("KEYWORD", - head + "+%dc" % a, - head + "+%dc" % b) m = self.prog.search(chars, m.end()) if "SYNC" in self.tag_names(next + "-1c"): head = next @@ -255,20 +235,23 @@ for tag in self.tagdefs.keys(): self.tag_remove(tag, "1.0", "end") -def _color_delegator(parent): +def _color_delegator(parent): # htest # + from Tkinter import Toplevel, Text from idlelib.Percolator import Percolator - root = Tk() - root.title("Test ColorDelegator") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) - source = "if somename: x = 'abc' # comment\nprint" - text = Text(root, background="white") + + top = Toplevel(parent) + top.title("Test ColorDelegator") + top.geometry("200x100+%d+%d" % (parent.winfo_rootx() + 200, + parent.winfo_rooty() + 150)) + source = "if somename: x = 'abc' # comment\nprint\n" + text = Text(top, background="white") + text.pack(expand=1, fill="both") text.insert("insert", source) - text.pack(expand=1, fill="both") + text.focus_set() + p = Percolator(text) d = ColorDelegator() p.insertfilter(d) - root.mainloop() if __name__ == "__main__": from idlelib.idle_test.htest import run diff --git a/lib-python/2.7/idlelib/Debugger.py b/lib-python/2.7/idlelib/Debugger.py --- a/lib-python/2.7/idlelib/Debugger.py +++ b/lib-python/2.7/idlelib/Debugger.py @@ -1,6 +1,5 @@ import os import bdb -import types from Tkinter import * from idlelib.WindowList import ListedToplevel from idlelib.ScrolledList import ScrolledList diff --git a/lib-python/2.7/idlelib/EditorWindow.py b/lib-python/2.7/idlelib/EditorWindow.py --- a/lib-python/2.7/idlelib/EditorWindow.py +++ b/lib-python/2.7/idlelib/EditorWindow.py @@ -1,6 +1,6 @@ import sys import os -from platform import python_version +import platform import re import imp from Tkinter import * @@ -22,6 +22,8 @@ # The default tab setting for a Text widget, in average-width characters. TK_TABWIDTH_DEFAULT = 8 +_py_version = ' (%s)' % platform.python_version() + def _sphinx_version(): "Format sys.version_info to produce the Sphinx version string used to install the chm docs" major, minor, micro, level, serial = sys.version_info @@ -151,7 +153,7 @@ # Safari requires real file:-URLs EditorWindow.help_url = 'file://' + EditorWindow.help_url else: - EditorWindow.help_url = "http://docs.python.org/%d.%d" % sys.version_info[:2] + EditorWindow.help_url = "https://docs.python.org/%d.%d/" % sys.version_info[:2] currentTheme=idleConf.CurrentTheme() self.flist = flist root = root or flist.root @@ -214,6 +216,8 @@ text.bind("<>", self.python_docs) text.bind("<>", self.about_dialog) text.bind("<>", self.config_dialog) + text.bind("<>", + self.config_extensions_dialog) text.bind("<>", self.open_module) text.bind("<>", lambda event: "break") text.bind("<>", self.select_all) @@ -568,6 +572,8 @@ def config_dialog(self, event=None): configDialog.ConfigDialog(self.top,'Settings') + def config_extensions_dialog(self, event=None): + configDialog.ConfigExtensionsDialog(self.top) def help_dialog(self, event=None): if self.root: @@ -691,30 +697,29 @@ return # XXX Ought to insert current file's directory in front of path try: - (f, file, (suffix, mode, type)) = _find_module(name) + (f, file_path, (suffix, mode, mtype)) = _find_module(name) except (NameError, ImportError) as msg: tkMessageBox.showerror("Import error", str(msg), parent=self.text) return - if type != imp.PY_SOURCE: + if mtype != imp.PY_SOURCE: tkMessageBox.showerror("Unsupported type", "%s is not a source module" % name, parent=self.text) return if f: f.close() if self.flist: - self.flist.open(file) + self.flist.open(file_path) else: - self.io.loadfile(file) + self.io.loadfile(file_path) + return file_path def open_class_browser(self, event=None): filename = self.io.filename - if not filename: - tkMessageBox.showerror( - "No filename", - "This buffer has no associated filename", - master=self.text) - self.text.focus_set() - return None + if not (self.__class__.__name__ == 'PyShellEditorWindow' + and filename): + filename = self.open_module() + if filename is None: + return head, tail = os.path.split(filename) base, ext = os.path.splitext(tail) from idlelib import ClassBrowser @@ -779,7 +784,7 @@ self.color = None def ResetColorizer(self): - "Update the colour theme" + "Update the color theme" # Called from self.filename_change_hook and from configDialog.py self._rmcolorizer() self._addcolorizer() @@ -944,7 +949,7 @@ short = self.short_title() long = self.long_title() if short and long: - title = short + " - " + long + title = short + " - " + long + _py_version elif short: title = short elif long: @@ -968,14 +973,13 @@ self.undo.reset_undo() def short_title(self): - pyversion = "Python " + python_version() + ": " filename = self.io.filename if filename: filename = os.path.basename(filename) else: filename = "Untitled" # return unicode string to display non-ASCII chars correctly - return pyversion + self._filename_to_unicode(filename) + return self._filename_to_unicode(filename) def long_title(self): # return unicode string to display non-ASCII chars correctly @@ -1711,7 +1715,8 @@ tk.call('set', 'tcl_nonwordchars', '[^a-zA-Z0-9_]') -def _editor_window(parent): +def _editor_window(parent): # htest # + # error if close master window first - timer event, after script root = parent fixwordbreaks(root) if sys.argv[1:]: @@ -1721,7 +1726,8 @@ macosxSupport.setupApp(root, None) edit = EditorWindow(root=root, filename=filename) edit.text.bind("<>", edit.close_event) - parent.mainloop() + # Does not stop error, neither does following + # edit.text.bind("<>", edit.close_event) if __name__ == '__main__': diff --git a/lib-python/2.7/idlelib/GrepDialog.py b/lib-python/2.7/idlelib/GrepDialog.py --- a/lib-python/2.7/idlelib/GrepDialog.py +++ b/lib-python/2.7/idlelib/GrepDialog.py @@ -45,10 +45,10 @@ def create_entries(self): SearchDialogBase.create_entries(self) - self.globent = self.make_entry("In files:", self.globvar) + self.globent = self.make_entry("In files:", self.globvar)[0] def create_other_buttons(self): - f = self.make_frame() + f = self.make_frame()[0] btn = Checkbutton(f, anchor="w", variable=self.recvar, @@ -131,7 +131,7 @@ self.top.withdraw() -def _grep_dialog(parent): # for htest +def _grep_dialog(parent): # htest # from idlelib.PyShell import PyShellFileList root = Tk() root.title("Test GrepDialog") diff --git a/lib-python/2.7/idlelib/IOBinding.py b/lib-python/2.7/idlelib/IOBinding.py --- a/lib-python/2.7/idlelib/IOBinding.py +++ b/lib-python/2.7/idlelib/IOBinding.py @@ -19,11 +19,7 @@ from idlelib.configHandler import idleConf -try: - from codecs import BOM_UTF8 -except ImportError: - # only available since Python 2.3 - BOM_UTF8 = '\xef\xbb\xbf' +from codecs import BOM_UTF8 # Try setting the locale, so that we can find out # what encoding to use @@ -72,6 +68,7 @@ encoding = encoding.lower() coding_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)') +blank_re = re.compile(r'^[ \t\f]*(?:[#\r\n]|$)') class EncodingMessage(SimpleDialog): "Inform user that an encoding declaration is needed." @@ -130,6 +127,8 @@ match = coding_re.match(line) if match is not None: break + if not blank_re.match(line): + return None else: return None name = match.group(1) @@ -529,6 +528,8 @@ ("All files", "*"), ] + defaultextension = '.py' if sys.platform == 'darwin' else '' + def askopenfile(self): dir, base = self.defaultfilename("open") if not self.opendialog: @@ -554,8 +555,10 @@ def asksavefile(self): dir, base = self.defaultfilename("save") if not self.savedialog: - self.savedialog = tkFileDialog.SaveAs(master=self.text, - filetypes=self.filetypes) + self.savedialog = tkFileDialog.SaveAs( + master=self.text, + filetypes=self.filetypes, + defaultextension=self.defaultextension) filename = self.savedialog.show(initialdir=dir, initialfile=base) if isinstance(filename, unicode): filename = filename.encode(filesystemencoding) diff --git a/lib-python/2.7/idlelib/NEWS.txt b/lib-python/2.7/idlelib/NEWS.txt --- a/lib-python/2.7/idlelib/NEWS.txt +++ b/lib-python/2.7/idlelib/NEWS.txt @@ -1,6 +1,183 @@ +What's New in IDLE 2.7.9? +========================= + +*Release data: 2014-12-07* (projected) + +- Issue #16893: Update Idle doc chapter to match current Idle and add new + information. + +- Issue #3068: Add Idle extension configuration dialog to Options menu. + Changes are written to HOME/.idlerc/config-extensions.cfg. + Original patch by Tal Einat. + +- Issue #16233: A module browser (File : Class Browser, Alt+C) requires a + editor window with a filename. When Class Browser is requested otherwise, + from a shell, output window, or 'Untitled' editor, Idle no longer displays + an error box. It now pops up an Open Module box (Alt+M). If a valid name + is entered and a module is opened, a corresponding browser is also opened. + +- Issue #4832: Save As to type Python files automatically adds .py to the + name you enter (even if your system does not display it). Some systems + automatically add .txt when type is Text files. + +- Issue #21986: Code objects are not normally pickled by the pickle module. + To match this, they are no longer pickled when running under Idle. + +- Issue #22221: IDLE now ignores the source encoding declaration on the second + line if the first line contains anything except a comment. + +- Issue #17390: Adjust Editor window title; remove 'Python', + move version to end. + +- Issue #14105: Idle debugger breakpoints no longer disappear + when inseting or deleting lines. + + +What's New in IDLE 2.7.8? +========================= + +*Release date: 2014-06-29* + +- Issue #21940: Add unittest for WidgetRedirector. Initial patch by Saimadhav + Heblikar. + +- Issue #18592: Add unittest for SearchDialogBase. Patch by Phil Webster. + +- Issue #21694: Add unittest for ParenMatch. Patch by Saimadhav Heblikar. + +- Issue #21686: add unittest for HyperParser. Original patch by Saimadhav + Heblikar. + +- Issue #12387: Add missing upper(lower)case versions of default Windows key + bindings for Idle so Caps Lock does not disable them. Patch by Roger Serwy. + +- Issue #21695: Closing a Find-in-files output window while the search is + still in progress no longer closes Idle. + +- Issue #18910: Add unittest for textView. Patch by Phil Webster. + +- Issue #18292: Add unittest for AutoExpand. Patch by Saihadhav Heblikar. + +- Issue #18409: Add unittest for AutoComplete. Patch by Phil Webster. + + +What's New in IDLE 2.7.7? +========================= + +*Release date: 2014-05-31* + +- Issue #18104: Add idlelib/idle_test/htest.py with a few sample tests to begin + consolidating and improving human-validated tests of Idle. Change other files + as needed to work with htest. Running the module as __main__ runs all tests. + +- Issue #21139: Change default paragraph width to 72, the PEP 8 recommendation. + +- Issue #21284: Paragraph reformat test passes after user changes reformat width. + +- Issue #20406: Use Python application icons for Idle window title bars. + Patch mostly by Serhiy Storchaka. + +- Issue #21029: Occurrences of "print" are now consistently colored as + being a keyword (the colorizer doesn't know if print functions are + enabled in the source). + +- Issue #17721: Remove non-functional configuration dialog help button until we + make it actually gives some help when clicked. Patch by Guilherme Sim�es. + +- Issue #17390: Add Python version to Idle editor window title bar. + Original patches by Edmond Burnett and Kent Johnson. + +- Issue #20058: sys.stdin.readline() in IDLE now always returns only one line. + +- Issue #19481: print() of unicode, str or bytearray subclass instance in IDLE + no more hangs. + +- Issue #18270: Prevent possible IDLE AttributeError on OS X when no initial + shell window is present. + +- Issue #17654: Ensure IDLE menus are customized properly on OS X for + non-framework builds and for all variants of Tk. + + +What's New in IDLE 2.7.6? +========================= + +*Release date: 2013-11-10* + +- Issue #19426: Fixed the opening of Python source file with specified encoding. + +- Issue #18873: IDLE now detects Python source code encoding only in comment + lines. + +- Issue #18988: The "Tab" key now works when a word is already autocompleted. + +- Issue #18489: Add tests for SearchEngine. Original patch by Phil Webster. + +- Issue #18429: Format / Format Paragraph, now works when comment blocks + are selected. As with text blocks, this works best when the selection + only includes complete lines. + +- Issue #18226: Add docstrings and unittests for FormatParagraph.py. + Original patches by Todd Rovito and Phil Webster. + +- Issue #18279: Format - Strip trailing whitespace no longer marks a file as + changed when it has not been changed. This fix followed the addition of a + test file originally written by Phil Webster (the issue's main goal). + +- Issue #18539: Calltips now work for float default arguments. + +- Issue #7136: In the Idle File menu, "New Window" is renamed "New File". + Patch by Tal Einat, Roget Serwy, and Todd Rovito. + +- Issue #8515: Set __file__ when run file in IDLE. + Initial patch by Bruce Frederiksen. + +- Issue #5492: Avoid traceback when exiting IDLE caused by a race condition. + +- Issue #17511: Keep IDLE find dialog open after clicking "Find Next". + Original patch by Sarah K. + +- Issue #15392: Create a unittest framework for IDLE. + Preliminary patch by Rajagopalasarma Jayakrishnan + See Lib/idlelib/idle_test/README.txt for how to run Idle tests. + +- Issue #14146: Highlight source line while debugging on Windows. + +- Issue #17532: Always include Options menu for IDLE on OS X. + Patch by Guilherme Sim�es. + + What's New in IDLE 2.7.5? ========================= +*Release date: 2013-05-12* + +- Issue #17838: Allow sys.stdin to be reassigned. + +- Issue #14735: Update IDLE docs to omit "Control-z on Windows". + +- Issue #17585: Fixed IDLE regression. Now closes when using exit() or quit(). + +- Issue #17657: Show full Tk version in IDLE's about dialog. + Patch by Todd Rovito. + +- Issue #17613: Prevent traceback when removing syntax colorizer in IDLE. + +- Issue #1207589: Backwards-compatibility patch for right-click menu in IDLE. + +- Issue #16887: IDLE now accepts Cancel in tabify/untabify dialog box. + +- Issue #14254: IDLE now handles readline correctly across shell restarts. + +- Issue #17614: IDLE no longer raises exception when quickly closing a file. + +- Issue #6698: IDLE now opens just an editor window when configured to do so. + +- Issue #8900: Using keyboard shortcuts in IDLE to open a file no longer + raises an exception. + +- Issue #6649: Fixed missing exit status in IDLE. Patch by Guilherme Polo. + - Issue #17390: Display Python version on Idle title bar. Initial patch by Edmond Burnett. @@ -8,17 +185,67 @@ What's New in IDLE 2.7.4? ========================= +*Release date: 2013-04-06* + +- Issue #17625: In IDLE, close the replace dialog after it is used. + +- IDLE was displaying spurious SystemExit tracebacks when running scripts + that terminated by raising SystemExit (i.e. unittest and turtledemo). + +- Issue #9290: In IDLE the sys.std* streams now implement io.TextIOBase + interface and support all mandatory methods and properties. + +- Issue #16829: IDLE printing no longer fails if there are spaces or other + special characters in the file path. + +- Issue #16819: IDLE method completion now correctly works for unicode literals. + +- Issue #16504: IDLE now catches SyntaxErrors raised by tokenizer. Patch by + Roger Serwy. + +- Issue #1207589: Add Cut/Copy/Paste items to IDLE right click Context Menu + Patch by Todd Rovito. + +- Issue #13052: Fix IDLE crashing when replace string in Search/Replace dialog + ended with '\'. Patch by Roger Serwy. + +- Issue #9803: Don't close IDLE on saving if breakpoint is open. + Patch by Roger Serwy. + +- Issue #14958: Change IDLE systax highlighting to recognize all string and byte + literals currently supported in Python 2.7. + +- Issue #14962: Update text coloring in IDLE shell window after changing + options. Patch by Roger Serwy. + +- Issue #10997: Prevent a duplicate entry in IDLE's "Recent Files" menu. + +- Issue #12510: Attempting to get invalid tooltip no longer closes IDLE. + Original patch by Roger Serwy. + +- Issue #10365: File open dialog now works instead of crashing + even when parent window is closed. Patch by Roger Serwy. + +- Issue #14876: Use user-selected font for highlight configuration. + Patch by Roger Serwy. + +- Issue #14409: IDLE now properly executes commands in the Shell window + when it cannot read the normal config files on startup and + has to use the built-in default key bindings. + There was previously a bug in one of the defaults. + +- Issue #3573: IDLE hangs when passing invalid command line args + (directory(ies) instead of file(s)) (Patch by Guilherme Polo) + +- Issue #5219: Prevent event handler cascade in IDLE. + - Issue #15318: Prevent writing to sys.stdin. - Issue #13532, #15319: Check that arguments to sys.stdout.write are strings. -- Issue # 12510: Attempt to get certain tool tips no longer crashes IDLE. - -- Issue10365: File open dialog now works instead of crashing even when +- Issue #10365: File open dialog now works instead of crashing even when parent window is closed while dialog is open. -- Issue 14876: use user-selected font for highlight configuration. From noreply at buildbot.pypy.org Mon Mar 30 13:52:28 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Mon, 30 Mar 2015 13:52:28 +0200 (CEST) Subject: [pypy-commit] pypy stmgc-c8: import stmgc-c8 f6788cf5fb73 Message-ID: <20150330115228.5EB771C0207@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: stmgc-c8 Changeset: r76637:ddeaa80c4bf7 Date: 2015-03-30 10:55 +0200 http://bitbucket.org/pypy/pypy/changeset/ddeaa80c4bf7/ Log: import stmgc-c8 f6788cf5fb73 diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision --- a/rpython/translator/stm/src_stm/revision +++ b/rpython/translator/stm/src_stm/revision @@ -1,1 +1,1 @@ -b548b42c978e+ +f6788cf5fb73 diff --git a/rpython/translator/stm/src_stm/stm/atomic.h b/rpython/translator/stm/src_stm/stm/atomic.h --- a/rpython/translator/stm/src_stm/stm/atomic.h +++ b/rpython/translator/stm/src_stm/stm/atomic.h @@ -27,13 +27,11 @@ # define HAVE_FULL_EXCHANGE_INSN static inline void spin_loop(void) { asm("pause" : : : "memory"); } static inline void write_fence(void) { asm("" : : : "memory"); } - static inline void read_fence(void) { asm("" : : : "memory"); } #else static inline void spin_loop(void) { asm("" : : : "memory"); } static inline void write_fence(void) { __sync_synchronize(); } - static inline void read_fence(void) { asm("" : : : "memory"); } #endif diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c --- a/rpython/translator/stm/src_stm/stm/core.c +++ b/rpython/translator/stm/src_stm/stm/core.c @@ -2,6 +2,11 @@ #ifndef _STM_CORE_H_ # error "must be compiled via stmgc.c" #endif +char *stm_object_pages; +long _stm_segment_nb_pages = NB_PAGES; +int _stm_nb_segments = NB_SEGMENTS; +int _stm_psegment_ofs = (int)(uintptr_t)STM_PSEGMENT; + /* *** MISC *** */ static void free_bk(struct stm_undo_s *undo) { @@ -48,6 +53,7 @@ assert(IMPLY(from_segnum >= 0, get_priv_segment(from_segnum)->modification_lock)); assert(STM_PSEGMENT->modification_lock); + long my_segnum = STM_SEGMENT->segment_num; DEBUG_EXPECT_SEGFAULT(false); for (; undo < end; undo++) { if (undo->type == TYPE_POSITION_MARKER) @@ -57,18 +63,19 @@ uintptr_t current_page_num = ((uintptr_t)oslice) / 4096; if (pagenum == -1) { - if (get_page_status_in(STM_SEGMENT->segment_num, - current_page_num) == PAGE_NO_ACCESS) + if (get_page_status_in(my_segnum, current_page_num) == PAGE_NO_ACCESS) continue; - } - else { - if (current_page_num != pagenum) - continue; + } else if (pagenum != current_page_num) { + continue; } - if (from_segnum == -2 && _stm_was_read(obj) && (obj->stm_flags & GCFLAG_WB_EXECUTED)) { + if (from_segnum == -2 + && _stm_was_read(obj) + && (get_page_status_in(my_segnum, (uintptr_t)obj / 4096) == PAGE_ACCESSIBLE) + && (obj->stm_flags & GCFLAG_WB_EXECUTED)) { /* called from stm_validate(): > if not was_read(), we certainly didn't modify + > if obj->stm_flags is not accessible, WB_EXECUTED cannot be set > if not WB_EXECUTED, we may have read from the obj in a different page but did not modify it (should not occur right now, but future proof!) only the WB_EXECUTED alone is not enough, since we may have imported from a @@ -79,8 +86,7 @@ /* XXX: if the next assert is always true, we should never get a segfault in this function at all. So the DEBUG_EXPECT_SEGFAULT is correct. */ - assert((get_page_status_in(STM_SEGMENT->segment_num, - current_page_num) != PAGE_NO_ACCESS)); + assert((get_page_status_in(my_segnum, current_page_num) != PAGE_NO_ACCESS)); /* dprintf(("import slice seg=%d obj=%p off=%lu sz=%d pg=%lu\n", */ /* from_segnum, obj, SLICE_OFFSET(undo->slice), */ @@ -95,7 +101,8 @@ if (src_segment_base == NULL && SLICE_OFFSET(undo->slice) == 0) { /* check that restored obj doesn't have WB_EXECUTED */ - assert(!(obj->stm_flags & GCFLAG_WB_EXECUTED)); + assert((get_page_status_in(my_segnum, (uintptr_t)obj / 4096) == PAGE_NO_ACCESS) + || !(obj->stm_flags & GCFLAG_WB_EXECUTED)); } } DEBUG_EXPECT_SEGFAULT(true); @@ -1107,16 +1114,6 @@ STM_PSEGMENT->shadowstack_at_start_of_transaction = tl->shadowstack; STM_PSEGMENT->threadlocal_at_start_of_transaction = tl->thread_local_obj; - enter_safe_point_if_requested(); - dprintf(("> start_transaction\n")); - - s_mutex_unlock(); // XXX it's probably possible to not acquire this here - - uint8_t old_rv = STM_SEGMENT->transaction_read_version; - STM_SEGMENT->transaction_read_version = old_rv + 1; - if (UNLIKELY(old_rv == 0xff)) { - reset_transaction_read_version(); - } assert(list_is_empty(STM_PSEGMENT->modified_old_objects)); assert(list_is_empty(STM_PSEGMENT->large_overflow_objects)); @@ -1135,6 +1132,25 @@ check_nursery_at_transaction_start(); + /* Change read-version here, because if we do stm_validate in the + safe-point below, we should not see our old reads from the last + transaction. */ + uint8_t rv = STM_SEGMENT->transaction_read_version; + if (rv < 0xff) /* else, rare (maybe impossible?) case: we did already */ + rv++; /* incr it but enter_safe_point_if_requested() aborted */ + STM_SEGMENT->transaction_read_version = rv; + + /* Warning: this safe-point may run light finalizers and register + commit/abort callbacks if a major GC is triggered here */ + enter_safe_point_if_requested(); + dprintf(("> start_transaction\n")); + + s_mutex_unlock(); // XXX it's probably possible to not acquire this here + + if (UNLIKELY(rv == 0xff)) { + reset_transaction_read_version(); + } + stm_validate(); } diff --git a/rpython/translator/stm/src_stm/stm/core.h b/rpython/translator/stm/src_stm/stm/core.h --- a/rpython/translator/stm/src_stm/stm/core.h +++ b/rpython/translator/stm/src_stm/stm/core.h @@ -222,11 +222,10 @@ static void free_cle(struct stm_commit_log_entry_s *e); -#ifndef STM_TESTS -static char *stm_object_pages; -#else -char *stm_object_pages; -#endif +extern char *stm_object_pages; +extern long _stm_segment_nb_pages; +extern int _stm_nb_segments; +extern int _stm_psegment_ofs; static stm_thread_local_t *stm_all_thread_locals = NULL; diff --git a/rpython/translator/stm/src_stm/stm/nursery.c b/rpython/translator/stm/src_stm/stm/nursery.c --- a/rpython/translator/stm/src_stm/stm/nursery.c +++ b/rpython/translator/stm/src_stm/stm/nursery.c @@ -461,16 +461,20 @@ } OPT_ASSERT((nursery_used & 7) == 0); - -#if _STM_NURSERY_ZEROED +#ifndef NDEBUG /* reset the nursery by zeroing it */ char *realnursery; realnursery = REAL_ADDRESS(pseg->pub.segment_base, _stm_nursery_start); +#if _STM_NURSERY_ZEROED memset(realnursery, 0, nursery_used); /* assert that the rest of the nursery still contains only zeroes */ assert_memset_zero(realnursery + nursery_used, (NURSERY_END - _stm_nursery_start) - nursery_used); + +#else + memset(realnursery, 0xa0, nursery_used); +#endif #endif pseg->pub.nursery_current = (stm_char *)_stm_nursery_start; @@ -636,6 +640,11 @@ #if _STM_NURSERY_ZEROED memset(REAL_ADDRESS(STM_SEGMENT->segment_base, o), 0, size_rounded_up); #else + +#ifndef NDEBUG + memset(REAL_ADDRESS(STM_SEGMENT->segment_base, o), 0xb0, size_rounded_up); +#endif + o->stm_flags = 0; /* make all pages of 'o' accessible as synchronize_obj_flush() in minor collections assumes all young objs are fully accessible. */ diff --git a/rpython/translator/stm/src_stm/stm/sync.c b/rpython/translator/stm/src_stm/stm/sync.c --- a/rpython/translator/stm/src_stm/stm/sync.c +++ b/rpython/translator/stm/src_stm/stm/sync.c @@ -103,6 +103,24 @@ /************************************************************/ +void stm_wait_for_current_inevitable_transaction(void) +{ + struct stm_commit_log_entry_s *current = STM_PSEGMENT->last_commit_log_entry; + + /* XXX: don't do busy-waiting */ + while (1) { + if (current->next == NULL) { + break; + } else if (current->next == INEV_RUNNING) { + usleep(10); + continue; + } + current = current->next; + } +} + + + static bool acquire_thread_segment(stm_thread_local_t *tl) { /* This function acquires a segment for the currently running thread, diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h --- a/rpython/translator/stm/src_stm/stmgc.h +++ b/rpython/translator/stm/src_stm/stmgc.h @@ -304,6 +304,11 @@ void stm_start_inevitable_transaction(stm_thread_local_t *tl); void stm_commit_transaction(void); + +/* Temporary fix? Call this outside a transaction. If there is an + inevitable transaction running somewhere else, wait until it finishes. */ +void stm_wait_for_current_inevitable_transaction(void); + void stm_abort_transaction(void) __attribute__((noreturn)); void stm_collect(long level); From noreply at buildbot.pypy.org Mon Mar 30 13:56:31 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 30 Mar 2015 13:56:31 +0200 (CEST) Subject: [pypy-commit] pypy default: _hashlib.openssl_md_meth_names is not a set, but a frozenset in cpython Message-ID: <20150330115631.AD9D01C01D0@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76638:dbe9bd42d773 Date: 2015-03-30 13:56 +0200 http://bitbucket.org/pypy/pypy/changeset/dbe9bd42d773/ Log: _hashlib.openssl_md_meth_names is not a set, but a frozenset in cpython diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py --- a/pypy/module/_hashlib/interp_hashlib.py +++ b/pypy/module/_hashlib/interp_hashlib.py @@ -27,14 +27,14 @@ try: space = global_name_fetcher.space w_name = space.wrap(rffi.charp2str(obj_name[0].c_name)) - space.call_method(global_name_fetcher.w_meth_names, "add", w_name) + global_name_fetcher.meth_names.append(w_name) except OperationError, e: global_name_fetcher.w_error = e class NameFetcher: def setup(self, space): self.space = space - self.w_meth_names = space.call_function(space.w_set) + self.meth_names = [] self.w_error = None def _cleanup_(self): self.__dict__.clear() @@ -47,7 +47,9 @@ hash_name_mapper_callback, None) if global_name_fetcher.w_error: raise global_name_fetcher.w_error - return global_name_fetcher.w_meth_names + meth_names = global_name_fetcher.meth_names + global_name_fetcher.meth_names = None + return space.call_function(space.w_frozenset, space.newlist(meth_names)) class W_Hash(W_Root): NULL_CTX = lltype.nullptr(ropenssl.EVP_MD_CTX.TO) diff --git a/pypy/module/_hashlib/test/test_hashlib.py b/pypy/module/_hashlib/test/test_hashlib.py --- a/pypy/module/_hashlib/test/test_hashlib.py +++ b/pypy/module/_hashlib/test/test_hashlib.py @@ -5,7 +5,7 @@ def test_method_names(self): import _hashlib - assert isinstance(_hashlib.openssl_md_meth_names, set) + assert isinstance(_hashlib.openssl_md_meth_names, frozenset) assert "md5" in _hashlib.openssl_md_meth_names def test_simple(self): From noreply at buildbot.pypy.org Mon Mar 30 16:05:27 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 30 Mar 2015 16:05:27 +0200 (CEST) Subject: [pypy-commit] extradoc extradoc: more details Message-ID: <20150330140527.55BE01C02AD@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r5520:eb519f972b4f Date: 2015-03-30 16:05 +0200 http://bitbucket.org/pypy/extradoc/changeset/eb519f972b4f/ Log: more details diff --git a/blog/draft/stm-mar2015.txt b/blog/draft/stm-mar2015.txt --- a/blog/draft/stm-mar2015.txt +++ b/blog/draft/stm-mar2015.txt @@ -13,10 +13,10 @@ http://pypy.readthedocs.org/en/latest/stm.html This is a special version of PyPy that contains the "Software -Transactional Memory" (STM) core called stmgc-c7. It gives a +Transactional Memory" (STM) plug-in called stmgc-c7. It gives a replacement for Python's classical Global Interpreter Lock (GIL). The current version scales only up to around 4 cores; the next version of -the core, stmgc-c8, is in development and should address that +the plug-in, stmgc-c8, is in development and should address that limitation. Both versions only support 64-bit Linux for now (contributions welcome). @@ -46,24 +46,28 @@ get very far. It probably felt like a toy: on very small examples it would nicely scale, but on any larger example it would not scale at all. You didn't get any feedback about why, but the underlying reason -for that is that, in any large example, there are a few unexpected STM -conflicts that occur all the time. This prevents any parallelization. +is that, in a typical large example, there are some STM conflicts that +occur all the time and that won't be immediately found just by +thinking. This prevents any parallelization. Now PyPy-STM is no longer a black box: you have a way to learn about -these conflicts and fix them. The tl;dr version is to run:: +these conflicts, fix them, and try again. The tl;dr version is to run:: - PYPYSTM=stmlog pypy-stm yourprogr.py - print_stm_log.py stmlog + PYPYSTM=stmlog ./pypy-stm yourprogr.py + ./print_stm_log.py stmlog + +More details in `the STM user guide`_. Performance ----------- The performance is now more stable than it used to be. More -precisely, the best case is "25%-40% single-core slow-down with -very good scaling up to 4 threads", and the average performance seems +precisely, the best case is still "25%-40% single-core slow-down with +very good scaling up to 4 threads", but the average performance seems not too far from that. There are still dark spots --- notably, the -JIT is still slower to warm up, though it was improved a lot. Apart from +JIT is still slower to warm up, though it was improved a lot. These +are documented in the `current status`_ section. Apart from that, we should not get more than 2x single-core slow-down in the worst case. Please report such cases as bugs! @@ -71,15 +75,15 @@ TransactionQueue ---------------- -PyPy-STM is more than "just" a Python without GIL. It is a Python in -which you can do minor tweaks to your existing, *non-multithreaded* -programs and get them to use multiple cores. The basic idea is to -identify medium- or large-sized, likely-independent parts of the code -and to ask PyPy-STM to run these parts in parallel. An example would be -every iteration of some outermost loop over all items of a dictionary. -This is done with a ``transaction.TransactionQueue`` instance. See -``help(TransactionQueue)`` or read more about it in `the STM user -guide`_. +As explained before, PyPy-STM is more than "just" a Python without +GIL. It is a Python in which you can do minor tweaks to your +existing, *non-multithreaded* programs and get them to use multiple +cores. You identify medium- or large-sized, likely-independent parts +of the code and to ask PyPy-STM to run these parts in parallel. An +example would be every iteration of some outermost loop over all items +of a dictionary. This is done with a new API: +``transaction.TransactionQueue()``. See ``help(TransactionQueue)`` or +read more about it in `the STM user guide`_. This is not a 100% mechanical change: very likely, you need to hunt for and fix "STM conflicts" that prevent parallel execution (see From noreply at buildbot.pypy.org Mon Mar 30 16:07:24 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 30 Mar 2015 16:07:24 +0200 (CEST) Subject: [pypy-commit] pypy release-2.5.x: Apply 180f86a2e1fb and 5f90d0283413 from trunk. They are needed in case Message-ID: <20150330140724.C07EF1C02AD@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: release-2.5.x Changeset: r76639:ce191ec5b5f6 Date: 2015-03-30 15:50 +0200 http://bitbucket.org/pypy/pypy/changeset/ce191ec5b5f6/ Log: Apply 180f86a2e1fb and 5f90d0283413 from trunk. They are needed in case ll2ctypes fails to import libssl during translation, but the final compilation is still correctly configured to find libssl. diff --git a/pypy/module/_hashlib/__init__.py b/pypy/module/_hashlib/__init__.py --- a/pypy/module/_hashlib/__init__.py +++ b/pypy/module/_hashlib/__init__.py @@ -1,11 +1,10 @@ from pypy.interpreter.mixedmodule import MixedModule -from pypy.module._hashlib.interp_hashlib import algorithms +from pypy.module._hashlib.interp_hashlib import algorithms, fetch_names class Module(MixedModule): interpleveldefs = { 'new' : 'interp_hashlib.new', - 'openssl_md_meth_names': 'interp_hashlib.get(space).w_meth_names' } appleveldefs = { @@ -15,5 +14,5 @@ interpleveldefs['openssl_' + name] = 'interp_hashlib.new_' + name def startup(self, space): - from rpython.rlib.ropenssl import init_digests - init_digests() + w_meth_names = fetch_names(space) + space.setattr(self, space.wrap('openssl_md_meth_names'), w_meth_names) diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py --- a/pypy/module/_hashlib/interp_hashlib.py +++ b/pypy/module/_hashlib/interp_hashlib.py @@ -16,8 +16,6 @@ algorithms = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512') def hash_name_mapper_callback(obj_name, userdata): - state = global_state[0] - assert state is not None if not obj_name: return # Ignore aliased names, they pollute the list and OpenSSL appears @@ -27,36 +25,29 @@ if obj_name[0].c_alias: return try: - w_name = state.space.wrap(rffi.charp2str(obj_name[0].c_name)) - state.space.call_method(state.w_meth_names, "add", w_name) + space = global_name_fetcher.space + w_name = space.wrap(rffi.charp2str(obj_name[0].c_name)) + space.call_method(global_name_fetcher.w_meth_names, "add", w_name) except OperationError, e: - state.w_error = e + global_name_fetcher.w_error = e -# XXX make it threadlocal? -global_state = [None] +class NameFetcher: + def setup(self, space): + self.space = space + self.w_meth_names = space.call_function(space.w_set) + self.w_error = None + def _cleanup_(self): + self.__dict__.clear() +global_name_fetcher = NameFetcher() -class State: - def __init__(self, space): - self.space = space - self.generate_method_names(space) - - def generate_method_names(self, space): - if not we_are_translated(): - ropenssl.init_digests() - self.w_error = None - try: - global_state[0] = self - self.w_meth_names = space.call_function(space.w_set) - ropenssl.OBJ_NAME_do_all( - ropenssl.OBJ_NAME_TYPE_MD_METH, - hash_name_mapper_callback, None) - finally: - global_state[0] = None - if self.w_error: - raise self.w_error - -def get(space): - return space.fromcache(State) +def fetch_names(space): + global_name_fetcher.setup(space) + ropenssl.init_digests() + ropenssl.OBJ_NAME_do_all(ropenssl.OBJ_NAME_TYPE_MD_METH, + hash_name_mapper_callback, None) + if global_name_fetcher.w_error: + raise global_name_fetcher.w_error + return global_name_fetcher.w_meth_names class W_Hash(W_Root): NULL_CTX = lltype.nullptr(ropenssl.EVP_MD_CTX.TO) From noreply at buildbot.pypy.org Mon Mar 30 16:15:09 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 30 Mar 2015 16:15:09 +0200 (CEST) Subject: [pypy-commit] extradoc extradoc: tweaks Message-ID: <20150330141509.B09C61C02AD@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r5521:78e31ca9b3bb Date: 2015-03-30 16:14 +0200 http://bitbucket.org/pypy/extradoc/changeset/78e31ca9b3bb/ Log: tweaks diff --git a/blog/draft/stm-mar2015.txt b/blog/draft/stm-mar2015.txt --- a/blog/draft/stm-mar2015.txt +++ b/blog/draft/stm-mar2015.txt @@ -2,9 +2,9 @@ PyPy-STM 2.5.1 - Mawhrin-Skel ============================= -We're pleased to announce PyPy-STM 2.5.1, Mawhrin-Skel. This is the -second official release of PyPy-STM. You can download this release -here: +We're pleased to announce PyPy-STM 2.5.1, codenamed Mawhrin-Skel. +This is the second official release of PyPy-STM. You can download +this release here (64-bit Linux only): http://pypy.org/download.html @@ -17,8 +17,8 @@ replacement for Python's classical Global Interpreter Lock (GIL). The current version scales only up to around 4 cores; the next version of the plug-in, stmgc-c8, is in development and should address that -limitation. Both versions only support 64-bit Linux for now -(contributions welcome). +limitation. Both versions only support 64-bit Linux; we'd welcome +someone to port stmgc-c8 to other (64-bit) platforms. This release passes all regular PyPy tests, except for a few special cases. In other words, you should be able to drop in From noreply at buildbot.pypy.org Mon Mar 30 16:24:19 2015 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 30 Mar 2015 16:24:19 +0200 (CEST) Subject: [pypy-commit] extradoc extradoc: add intro to intro Message-ID: <20150330142419.D00A51C0EAB@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: extradoc Changeset: r5522:e35d13a2125b Date: 2015-03-30 16:24 +0200 http://bitbucket.org/pypy/extradoc/changeset/e35d13a2125b/ Log: add intro to intro diff --git a/blog/draft/stm-mar2015.txt b/blog/draft/stm-mar2015.txt --- a/blog/draft/stm-mar2015.txt +++ b/blog/draft/stm-mar2015.txt @@ -12,6 +12,14 @@ http://pypy.readthedocs.org/en/latest/stm.html +PyPy is an implementation of the Python programming language which focuses +on performance. So far we've been relentlessly optimizing for the single +core/process scenario. PyPy STM brings to the table a version of PyPy +that does not have the infamous Global Interpreter Lock, hence can run +multiple threads on multiple cores. Additionally it comes with a set +of primitives that make writing multithreaded applications a lot easier, +see the documentation for details. + This is a special version of PyPy that contains the "Software Transactional Memory" (STM) plug-in called stmgc-c7. It gives a replacement for Python's classical Global Interpreter Lock (GIL). The From noreply at buildbot.pypy.org Mon Mar 30 16:26:20 2015 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 30 Mar 2015 16:26:20 +0200 (CEST) Subject: [pypy-commit] extradoc extradoc: tone down a bit Message-ID: <20150330142620.E1FAC1C02AD@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: extradoc Changeset: r5523:94d238600883 Date: 2015-03-30 16:26 +0200 http://bitbucket.org/pypy/extradoc/changeset/94d238600883/ Log: tone down a bit diff --git a/blog/draft/stm-mar2015.txt b/blog/draft/stm-mar2015.txt --- a/blog/draft/stm-mar2015.txt +++ b/blog/draft/stm-mar2015.txt @@ -23,7 +23,8 @@ This is a special version of PyPy that contains the "Software Transactional Memory" (STM) plug-in called stmgc-c7. It gives a replacement for Python's classical Global Interpreter Lock (GIL). The -current version scales only up to around 4 cores; the next version of +current version comes with a modest single-core overhead and scales only +up to around 4 cores on some examples; the next version of the plug-in, stmgc-c8, is in development and should address that limitation. Both versions only support 64-bit Linux; we'd welcome someone to port stmgc-c8 to other (64-bit) platforms. From noreply at buildbot.pypy.org Mon Mar 30 16:46:17 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 30 Mar 2015 16:46:17 +0200 (CEST) Subject: [pypy-commit] pypy.org extradoc: release pypy-stm 2.5.1 Message-ID: <20150330144617.C4A9D1C01D0@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r587:3ff3436a1b7a Date: 2015-03-30 16:46 +0200 http://bitbucket.org/pypy/pypy.org/changeset/3ff3436a1b7a/ Log: release pypy-stm 2.5.1 diff --git a/download.html b/download.html --- a/download.html +++ b/download.html @@ -74,8 +74,9 @@ performance improvements.

    We provide binaries for x86 and ARM Linux, Mac OS/X and Windows for:

    • Download
    +
    +

    PyPy-STM 2.5.1

    +

    This is a special version of PyPy! See the Software Transactional +Memory (STM) documentation.

    + +

    Other versions

    The other versions of PyPy are:

    diff --git a/source/download.txt b/source/download.txt --- a/source/download.txt +++ b/source/download.txt @@ -14,9 +14,11 @@ We provide binaries for x86 and ARM Linux, Mac OS/X and Windows for: -* the Python2.7 compatible release — **PyPy 2.5.1** — (`what's new in PyPy 2.5.1?`_ ) +* the Python2.7 compatible release — **PyPy 2.5.1** — (`what's new in PyPy 2.5.1?`_) * the Python3.2.5 compatible release — **PyPy3 2.4.0** — (`what's new in PyPy3 2.4.0?`_). +* the Python2.7 Software Transactional Memory special release — **PyPy-STM 2.5.1** (Linux x86-64 only) + .. _what's new in PyPy 2.5.1?: http://doc.pypy.org/en/latest/release-2.5.1.html .. _what's new in PyPy3 2.4.0?: http://doc.pypy.org/en/latest/release-pypy3-2.4.0.html @@ -138,6 +140,18 @@ `portable Linux binaries`_. +PyPy-STM 2.5.1 +------------------------------ + +This is a special version of PyPy! See the `Software Transactional +Memory`_ (STM) documentation. + +* `PyPy-STM Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 14.04)`__ + +.. _`Software Transactional Memory`: http://doc.pypy.org/en/latest/stm.html +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-stm-2.5.1-linux64.tar.bz2 + + .. _`Other versions (without a JIT)`: Other versions From noreply at buildbot.pypy.org Mon Mar 30 16:52:22 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 30 Mar 2015 16:52:22 +0200 (CEST) Subject: [pypy-commit] extradoc extradoc: Change paragraph #2 to flow better after the new paragraph #1. Message-ID: <20150330145222.A67641C11E0@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r5524:6ce2a5c637aa Date: 2015-03-30 16:52 +0200 http://bitbucket.org/pypy/extradoc/changeset/6ce2a5c637aa/ Log: Change paragraph #2 to flow better after the new paragraph #1. diff --git a/blog/draft/stm-mar2015.txt b/blog/draft/stm-mar2015.txt --- a/blog/draft/stm-mar2015.txt +++ b/blog/draft/stm-mar2015.txt @@ -18,16 +18,15 @@ that does not have the infamous Global Interpreter Lock, hence can run multiple threads on multiple cores. Additionally it comes with a set of primitives that make writing multithreaded applications a lot easier, -see the documentation for details. +as explained below (see TransactionQueue) and in the documentation. -This is a special version of PyPy that contains the "Software -Transactional Memory" (STM) plug-in called stmgc-c7. It gives a -replacement for Python's classical Global Interpreter Lock (GIL). The -current version comes with a modest single-core overhead and scales only -up to around 4 cores on some examples; the next version of -the plug-in, stmgc-c8, is in development and should address that -limitation. Both versions only support 64-bit Linux; we'd welcome -someone to port stmgc-c8 to other (64-bit) platforms. +Internally, PyPy-STM is based on the Software Transactional Memory +plug-in called stmgc-c7. This version comes with a relatively +reasonable single-core overhead but scales only up to around 4 cores +on some examples; the next version of the plug-in, stmgc-c8, is in +development and should address that limitation (as well as reduce the +overhead). These versions only support 64-bit Linux; we'd welcome +someone to port the upcoming stmgc-c8 to other (64-bit) platforms. This release passes all regular PyPy tests, except for a few special cases. In other words, you should be able to drop in From noreply at buildbot.pypy.org Mon Mar 30 17:40:37 2015 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 30 Mar 2015 17:40:37 +0200 (CEST) Subject: [pypy-commit] pypy default: fix the case when alias is too small Message-ID: <20150330154037.55A131C0EBB@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r76640:0f6bac0956ac Date: 2015-03-30 17:40 +0200 http://bitbucket.org/pypy/pypy/changeset/0f6bac0956ac/ Log: fix the case when alias is too small diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py --- a/pypy/module/_hashlib/interp_hashlib.py +++ b/pypy/module/_hashlib/interp_hashlib.py @@ -22,7 +22,7 @@ # to have a its own definition of alias as the resulting list # still contains duplicate and alternate names for several # algorithms. - if obj_name[0].c_alias: + if rffi.cast(lltype.Signed, obj_name[0].c_alias): return try: space = global_name_fetcher.space From noreply at buildbot.pypy.org Mon Mar 30 18:01:28 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 30 Mar 2015 18:01:28 +0200 (CEST) Subject: [pypy-commit] pypy default: Add a ztranslation test for '_hashlib'. Minor fixes to the fake objspace to make it work. Message-ID: <20150330160128.490BA1C0EBB@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76641:82e73bc3274e Date: 2015-03-30 18:01 +0200 http://bitbucket.org/pypy/pypy/changeset/82e73bc3274e/ Log: Add a ztranslation test for '_hashlib'. Minor fixes to the fake objspace to make it work. diff --git a/pypy/module/_hashlib/test/test_ztranslation.py b/pypy/module/_hashlib/test/test_ztranslation.py new file mode 100644 --- /dev/null +++ b/pypy/module/_hashlib/test/test_ztranslation.py @@ -0,0 +1,4 @@ +from pypy.objspace.fake.checkmodule import checkmodule + +def test_checkmodule(): + checkmodule('_hashlib') diff --git a/pypy/module/cpyext/test/test_ztranslation.py b/pypy/module/cpyext/test/test_ztranslation.py --- a/pypy/module/cpyext/test/test_ztranslation.py +++ b/pypy/module/cpyext/test/test_ztranslation.py @@ -1,4 +1,4 @@ from pypy.objspace.fake.checkmodule import checkmodule def test_cpyext_translates(): - checkmodule('cpyext', '_rawffi') + checkmodule('cpyext', '_rawffi', translate_startup=False) diff --git a/pypy/objspace/fake/checkmodule.py b/pypy/objspace/fake/checkmodule.py --- a/pypy/objspace/fake/checkmodule.py +++ b/pypy/objspace/fake/checkmodule.py @@ -2,16 +2,19 @@ from pypy.config.pypyoption import get_pypy_config -def checkmodule(*modnames): +def checkmodule(*modnames, **kwds): + translate_startup = kwds.pop('translate_startup', True) + assert not kwds config = get_pypy_config(translating=True) space = FakeObjSpace(config) seeobj_w = [] + modules = [] for modname in modnames: mod = __import__('pypy.module.%s' % modname, None, None, ['__doc__']) # force computation and record what we wrap module = mod.Module(space, W_Root()) module.setup_after_space_initialization() - module.startup(space) + modules.append(module) for name in module.loaders: seeobj_w.append(module._load_lazily(space, name)) if hasattr(module, 'submodules'): @@ -20,5 +23,11 @@ for name in submod.loaders: seeobj_w.append(submod._load_lazily(space, name)) # - space.translates(seeobj_w=seeobj_w, + def func(): + for mod in modules: + mod.startup(space) + if not translate_startup: + func() # call it now + func = None + space.translates(func, seeobj_w=seeobj_w, **{'translation.list_comprehension_operations': True}) diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py --- a/pypy/objspace/fake/objspace.py +++ b/pypy/objspace/fake/objspace.py @@ -113,7 +113,7 @@ BUILTIN_TYPES = ['int', 'str', 'float', 'long', 'tuple', 'list', 'dict', 'unicode', 'complex', 'slice', 'bool', 'basestring', 'object', - 'bytearray', 'buffer'] + 'bytearray', 'buffer', 'set', 'frozenset'] class FakeObjSpace(ObjSpace): def __init__(self, config=None): From noreply at buildbot.pypy.org Mon Mar 30 18:08:04 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 30 Mar 2015 18:08:04 +0200 (CEST) Subject: [pypy-commit] pypy release-2.5.x: fix the case when alias is too small Message-ID: <20150330160804.084871C0EAB@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: release-2.5.x Changeset: r76642:af9d5af2e8e3 Date: 2015-03-30 18:08 +0200 http://bitbucket.org/pypy/pypy/changeset/af9d5af2e8e3/ Log: fix the case when alias is too small (grafted from 0f6bac0956ac1fee924f863d0a0594f6b5441f95) diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py --- a/pypy/module/_hashlib/interp_hashlib.py +++ b/pypy/module/_hashlib/interp_hashlib.py @@ -22,7 +22,7 @@ # to have a its own definition of alias as the resulting list # still contains duplicate and alternate names for several # algorithms. - if obj_name[0].c_alias: + if rffi.cast(lltype.Signed, obj_name[0].c_alias): return try: space = global_name_fetcher.space From noreply at buildbot.pypy.org Mon Mar 30 18:27:56 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Mon, 30 Mar 2015 18:27:56 +0200 (CEST) Subject: [pypy-commit] pypy install-rpython: Create a setup.py that installs rpython Message-ID: <20150330162756.633871C01D0@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: install-rpython Changeset: r76643:34784ec27d15 Date: 2015-03-30 17:27 +0100 http://bitbucket.org/pypy/pypy/changeset/34784ec27d15/ Log: Create a setup.py that installs rpython diff --git a/setup.py b/setup.py new file mode 100644 --- /dev/null +++ b/setup.py @@ -0,0 +1,39 @@ +""" +Setup.py script for RPython +""" +from setuptools import setup, find_packages +# To use a consistent encoding +from codecs import open +from os import path + +here = path.abspath(path.dirname(__file__)) +with open(path.join(here, 'README.rst'), encoding='utf-8') as f: + long_description = f.read() # XXX + +setup( + name='rpython', + version='0.1', + description='RPython', + long_description=long_description, + + url='https://rpython.readthedocs.org', + author='The PyPy team', + author_email='pypy-dev at python.org', + license='MIT', + + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Developers', + 'Topic :: Software Development :: Build Tools', + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + ], + keywords='development', + + packages=find_packages(exclude=[ + '_pytest', 'ctypes_configure', 'include', 'lib-python', 'lib-pypy', + 'py', 'pypy', 'site-packages', 'testrunner']), + install_requires=['pytest'], +) From noreply at buildbot.pypy.org Mon Mar 30 18:49:45 2015 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 30 Mar 2015 18:49:45 +0200 (CEST) Subject: [pypy-commit] pypy vmprof: mark non-jitted functions Message-ID: <20150330164945.EB2711C01D0@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76644:0a137723eabf Date: 2015-03-30 18:49 +0200 http://bitbucket.org/pypy/pypy/changeset/0a137723eabf/ Log: mark non-jitted functions diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py --- a/pypy/module/_vmprof/interp_vmprof.py +++ b/pypy/module/_vmprof/interp_vmprof.py @@ -96,7 +96,7 @@ gc_frame = cast_instance_to_gcref(frame) gc_inputvalue = cast_instance_to_gcref(w_inputvalue) gc_operr = cast_instance_to_gcref(operr) - unique_id = frame.pycode._unique_id + unique_id = frame.pycode._unique_id | 1 gc_result = pypy_execute_frame_trampoline(gc_frame, gc_inputvalue, gc_operr, unique_id) return cast_base_ptr_to_instance(W_Root, gc_result) From noreply at buildbot.pypy.org Mon Mar 30 19:07:57 2015 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 30 Mar 2015 19:07:57 +0200 (CEST) Subject: [pypy-commit] pypy vmprof: we use +1 everywhere, so make this -1 so we hit the same id Message-ID: <20150330170757.C05401C01D0@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76645:4ea3a0805b5a Date: 2015-03-30 19:07 +0200 http://bitbucket.org/pypy/pypy/changeset/4ea3a0805b5a/ Log: we use +1 everywhere, so make this -1 so we hit the same id diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py --- a/pypy/module/_vmprof/interp_vmprof.py +++ b/pypy/module/_vmprof/interp_vmprof.py @@ -96,7 +96,7 @@ gc_frame = cast_instance_to_gcref(frame) gc_inputvalue = cast_instance_to_gcref(w_inputvalue) gc_operr = cast_instance_to_gcref(operr) - unique_id = frame.pycode._unique_id | 1 + unique_id = frame.pycode._unique_id - 1 gc_result = pypy_execute_frame_trampoline(gc_frame, gc_inputvalue, gc_operr, unique_id) return cast_base_ptr_to_instance(W_Root, gc_result) From noreply at buildbot.pypy.org Mon Mar 30 19:19:38 2015 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 30 Mar 2015 19:19:38 +0200 (CEST) Subject: [pypy-commit] pypy object-dtype2: merge default into branch Message-ID: <20150330171938.E7FFC1C0EAB@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: object-dtype2 Changeset: r76646:6e392eb89276 Date: 2015-03-29 22:46 +0300 http://bitbucket.org/pypy/pypy/changeset/6e392eb89276/ Log: merge default into branch diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -3,20 +3,10 @@ d8ac7d23d3ec5f9a0fa1264972f74a010dbfd07f release-1.6 ff4af8f318821f7f5ca998613a60fca09aa137da release-1.7 07e08e9c885ca67d89bcc304e45a32346daea2fa release-2.0-beta-1 -9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm -9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm ab0dd631c22015ed88e583d9fdd4c43eebf0be21 pypy-2.1-beta1-arm 20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0 394146e9bb673514c61f0150ab2013ccf78e8de7 release-2.3 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1 10f1b29a2bd21f837090286174a9ca030b8680b2 release-2.5.0 -8e24dac0b8e2db30d46d59f2c4daa3d4aaab7861 release-2.5.1 -8e24dac0b8e2db30d46d59f2c4daa3d4aaab7861 release-2.5.1 -0000000000000000000000000000000000000000 release-2.5.1 -0000000000000000000000000000000000000000 release-2.5.1 -e3d046c43451403f5969580fc1c41d5df6c4082a release-2.5.1 -e3d046c43451403f5969580fc1c41d5df6c4082a release-2.5.1 -0000000000000000000000000000000000000000 release-2.5.1 -0000000000000000000000000000000000000000 release-2.5.1 9c4588d731b7fe0b08669bd732c2b676cb0a8233 release-2.5.1 diff --git a/pypy/doc/stm.rst b/pypy/doc/stm.rst --- a/pypy/doc/stm.rst +++ b/pypy/doc/stm.rst @@ -29,7 +29,8 @@ ==================== ``pypy-stm`` is a variant of the regular PyPy interpreter. (This -version supports Python 2.7; see below for `Python 3`_.) With caveats_ +version supports Python 2.7; see below for `Python 3, CPython, +and others`_.) With caveats_ listed below, it should be in theory within 20%-50% slower than a regular PyPy, comparing the JIT version in both cases (but see below!). It is called @@ -178,8 +179,8 @@ -Python 3 -======== +Python 3, CPython, and others +============================= In this document I describe "pypy-stm", which is based on PyPy's Python 2.7 interpreter. Supporting Python 3 should take about half an @@ -194,6 +195,29 @@ framework, although the amount of work to put there might vary, because the STM framework within RPython is currently targeting the PyPy interpreter and other ones might have slightly different needs. +But in general, all the tedious transformations are done by RPython +and you're only left with the (hopefully few) hard and interesting bits. + +The core of STM works as a library written in C (see `reference to +implementation details`_ below). It means that it can be used on +other interpreters than the ones produced by RPython. Duhton_ is an +early example of that. At this point, you might think about adapting +this library for CPython. You're warned, though: as far as I can +tell, it is a doomed idea. I had a hard time debugging Duhton, and +that's infinitely simpler than CPython. Even ignoring that, you can +see in the C sources of Duhton that many core design decisions are +different than in CPython: no refcounting; limited support for +prebuilt "static" objects; ``stm_read()`` and ``stm_write()`` macro +calls everywhere (and getting very rare and very obscure bugs if you +forget one); and so on. You could imagine some custom special-purpose +extension of the C language, which you would preprocess to regular C. +In my opinion that's starting to look a lot like RPython itself, but +maybe you'd prefer this approach. Of course you still have to worry +about each and every C extension module you need, but maybe you'd have +a way forward. + +.. _Duhton: https://bitbucket.org/pypy/duhton + User Guide @@ -372,18 +396,49 @@ and ``y`` that are thread-local: reading or writing them from concurrently-running transactions will return independent results. (Any other attributes of ``Foo`` instances will be globally visible - from all threads, as usual.) The optional argument to - ``threadlocalproperty()`` is the default value factory: in case no - value was assigned in the current thread yet, the factory is called - and its result becomes the value in that thread (like - ``collections.defaultdict``). If no default value factory is - specified, uninitialized reads raise ``AttributeError``. Note that - with ``TransactionQueue`` you get a pool of a fixed number of - threads, each running the transactions one after the other; such - thread-local properties will have the value last stored in them in - the same thread,, which may come from a random previous transaction. - This means that ``threadlocalproperty`` is useful mainly to avoid - conflicts from cache-like data structures. + from all threads, as usual.) This is useful together with + ``TransactionQueue`` for these two cases: + + - For attributes of long-lived objects that change during one + transaction, but should always be reset to some initial value + around transaction (for example, initialized to 0 at the start of + a transaction; or, if used for a list of pending things to do + within this transaction, it will always be empty at the end of one + transaction). + + - For general caches across transactions. With ``TransactionQueue`` + you get a pool of a fixed number N of threads, each running the + transactions serially. A thread-local property will have the + value last stored in it by the same thread, which may come from a + random previous transaction. Basically, you get N copies of the + property's value, and each transaction accesses a random copy. It + works fine for caches. + + In more details, the optional argument to ``threadlocalproperty()`` + is the default value factory: in case no value was assigned in the + current thread yet, the factory is called and its result becomes the + value in that thread (like ``collections.defaultdict``). If no + default value factory is specified, uninitialized reads raise + ``AttributeError``. + +* In addition to all of the above, there are cases where write-write + conflicts are caused by writing the same value to an attribute again + and again. See for example ea2e519614ab_: this fixes two such + issues where we write an object field without first checking if we + already did it. The ``dont_change_any_more`` field is a flag set to + ``True`` in that part of the code, but usually this + ``rtyper_makekey()`` method will be called many times for the same + object; the code used to repeatedly set the flag to ``True``, but + now it first checks and only does the write if it is ``False``. + Similarly, in the second half of the checkin, the method + ``setup_block_entry()`` used to both assign the ``concretetype`` + fields and return a list, but its two callers were different: one + would really need the ``concretetype`` fields initialized, whereas + the other would only need to get its result list --- the + ``concretetype`` field in that case might already be set or not, but + that would not matter. + +.. _ea2e519614ab: https://bitbucket.org/pypy/pypy/commits/ea2e519614ab Note that Python is a complicated language; there are a number of less common cases that may cause conflict (of any kind) where we might not diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -5,3 +5,9 @@ .. this is a revision shortly after release-2.5.1 .. startrev: 397b96217b85 +.. branch: gc-incminimark-pinning-improve +Object Pinning is now used in `bz2` and `rzlib` (therefore also affects +Python's `zlib`). In case the data to compress/decompress is inside the nursery +(incminimark) it no longer needs to create a non-moving copy of it. This saves +one `malloc` and copying the data. Additionally a new GC environment variable +is introduced (`PYPY_GC_MAX_PINNED`) primarily for debugging purposes. diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -141,7 +141,7 @@ res = _pypy_execute_source(source) before = rffi.aroundstate.before if before: before() - return rffi.cast(rffi.INT, res) + return rffi.cast(rffi.INT, res) @entrypoint('main', [], c_name='pypy_init_threads') def pypy_init_threads(): @@ -312,7 +312,7 @@ w_dict = app.getwdict(space) entry_point, _ = create_entry_point(space, w_dict) - return entry_point, None, PyPyAnnotatorPolicy(single_space = space) + return entry_point, None, PyPyAnnotatorPolicy() def interface(self, ns): for name in ['take_options', 'handle_config', 'print_help', 'target', diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py --- a/pypy/module/_hashlib/interp_hashlib.py +++ b/pypy/module/_hashlib/interp_hashlib.py @@ -144,10 +144,10 @@ with self.lock: ropenssl.EVP_MD_CTX_copy(ctx, self.ctx) digest_size = self.digest_size - with lltype.scoped_alloc(rffi.CCHARP.TO, digest_size) as digest: - ropenssl.EVP_DigestFinal(ctx, digest, None) + with rffi.scoped_alloc_buffer(digest_size) as buf: + ropenssl.EVP_DigestFinal(ctx, buf.raw, None) ropenssl.EVP_MD_CTX_cleanup(ctx) - return rffi.charpsize2str(digest, digest_size) + return buf.str(digest_size) W_Hash.typedef = TypeDef( diff --git a/pypy/module/_minimal_curses/interp_curses.py b/pypy/module/_minimal_curses/interp_curses.py --- a/pypy/module/_minimal_curses/interp_curses.py +++ b/pypy/module/_minimal_curses/interp_curses.py @@ -13,7 +13,7 @@ def __init__(self, msg): self.msg = msg -from rpython.annotator.classdef import FORCE_ATTRIBUTES_INTO_CLASSES +from rpython.annotator.description import FORCE_ATTRIBUTES_INTO_CLASSES from rpython.annotator.model import SomeString # this is necessary due to annmixlevel diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -33,7 +33,8 @@ PY_SSL_CLIENT, PY_SSL_SERVER = 0, 1 (PY_SSL_VERSION_SSL2, PY_SSL_VERSION_SSL3, - PY_SSL_VERSION_SSL23, PY_SSL_VERSION_TLS1) = range(4) + PY_SSL_VERSION_SSL23, PY_SSL_VERSION_TLS1, PY_SSL_VERSION_TLS1_1, + PY_SSL_VERSION_TLS1_2) = range(6) SOCKET_IS_NONBLOCKING, SOCKET_IS_BLOCKING = 0, 1 SOCKET_HAS_TIMED_OUT, SOCKET_HAS_BEEN_CLOSED = 2, 3 @@ -72,6 +73,9 @@ constants["PROTOCOL_SSLv3"] = PY_SSL_VERSION_SSL3 constants["PROTOCOL_SSLv23"] = PY_SSL_VERSION_SSL23 constants["PROTOCOL_TLSv1"] = PY_SSL_VERSION_TLS1 +if HAVE_TLSv1_2: + constants["PROTOCOL_TLSv1_1"] = PY_SSL_VERSION_TLS1_1 + constants["PROTOCOL_TLSv1_2"] = PY_SSL_VERSION_TLS1_2 constants["OP_ALL"] = SSL_OP_ALL &~SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS constants["OP_NO_SSLv2"] = SSL_OP_NO_SSLv2 @@ -140,7 +144,7 @@ def __del__(self): rffi.free_nonmovingbuffer( - self.protos, self.buf, self.pinned, self.is_raw) + self.protos, self.buf, self.pinned, self.is_raw) @staticmethod def advertiseNPN_cb(s, data_ptr, len_ptr, args): @@ -162,7 +166,7 @@ client_len = len(npn.protos) else: client = lltype.nullptr(rffi.CCHARP.TO) - client_len = 0 + client_len = 0 libssl_SSL_select_next_proto(out_ptr, outlen_ptr, server, server_len, @@ -593,14 +597,14 @@ CB_MAXLEN = 128 with lltype.scoped_alloc(rffi.CCHARP.TO, CB_MAXLEN) as buf: - if (libssl_SSL_session_reused(self.ssl) ^ + if (libssl_SSL_session_reused(self.ssl) ^ (self.socket_type == PY_SSL_CLIENT)): # if session is resumed XOR we are the client length = libssl_SSL_get_finished(self.ssl, buf, CB_MAXLEN) else: # if a new session XOR we are the server length = libssl_SSL_get_peer_finished(self.ssl, buf, CB_MAXLEN) - + if length > 0: return space.wrap(rffi.charpsize2str(buf, intmask(length))) @@ -1107,7 +1111,7 @@ except OperationError as e: if not e.match(space, space.w_TypeError): raise - raise oefmt(space.w_TypeError, + raise oefmt(space.w_TypeError, "password callback must return a string") except OperationError as e: pw_info.operationerror = e @@ -1196,6 +1200,10 @@ method = libssl_SSLv2_method() elif protocol == PY_SSL_VERSION_SSL23: method = libssl_SSLv23_method() + elif protocol == PY_SSL_VERSION_TLS1_1 and HAVE_TLSv1_2: + method = libssl_TLSv1_1_method() + elif protocol == PY_SSL_VERSION_TLS1_2 and HAVE_TLSv1_2: + method = libssl_TLSv1_2_method() else: raise oefmt(space.w_ValueError, "invalid protocol version") ctx = libssl_SSL_CTX_new(method) @@ -1348,7 +1356,7 @@ except OperationError as e: if not e.match(space, space.w_TypeError): raise - raise oefmt(space.w_TypeError, + raise oefmt(space.w_TypeError, "password should be a string or callable") libssl_SSL_CTX_set_default_passwd_cb( @@ -1452,7 +1460,7 @@ if cadata is not None: with rffi.scoped_nonmovingbuffer(cadata) as buf: self._add_ca_certs(space, buf, len(cadata), ca_file_type) - + # load cafile or capath if cafile is not None or capath is not None: ret = libssl_SSL_CTX_load_verify_locations( diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py --- a/pypy/module/bz2/interp_bz2.py +++ b/pypy/module/bz2/interp_bz2.py @@ -562,10 +562,7 @@ in_bufsize = datasize with OutBuffer(self.bzs) as out: - with lltype.scoped_alloc(rffi.CCHARP.TO, in_bufsize) as in_buf: - - for i in range(datasize): - in_buf[i] = data[i] + with rffi.scoped_nonmovingbuffer(data) as in_buf: self.bzs.c_next_in = in_buf rffi.setintfield(self.bzs, 'c_avail_in', in_bufsize) @@ -663,9 +660,7 @@ in_bufsize = len(data) - with lltype.scoped_alloc(rffi.CCHARP.TO, in_bufsize) as in_buf: - for i in range(in_bufsize): - in_buf[i] = data[i] + with rffi.scoped_nonmovingbuffer(data) as in_buf: self.bzs.c_next_in = in_buf rffi.setintfield(self.bzs, 'c_avail_in', in_bufsize) @@ -716,9 +711,7 @@ with lltype.scoped_alloc(bz_stream.TO, zero=True) as bzs: in_bufsize = len(data) - with lltype.scoped_alloc(rffi.CCHARP.TO, in_bufsize) as in_buf: - for i in range(in_bufsize): - in_buf[i] = data[i] + with rffi.scoped_nonmovingbuffer(data) as in_buf: bzs.c_next_in = in_buf rffi.setintfield(bzs, 'c_avail_in', in_bufsize) @@ -758,9 +751,7 @@ return space.wrap("") with lltype.scoped_alloc(bz_stream.TO, zero=True) as bzs: - with lltype.scoped_alloc(rffi.CCHARP.TO, in_bufsize) as in_buf: - for i in range(in_bufsize): - in_buf[i] = data[i] + with rffi.scoped_nonmovingbuffer(data) as in_buf: bzs.c_next_in = in_buf rffi.setintfield(bzs, 'c_avail_in', in_bufsize) diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py --- a/pypy/module/pypyjit/test_pypy_c/model.py +++ b/pypy/module/pypyjit/test_pypy_c/model.py @@ -271,6 +271,7 @@ @classmethod def parse_ops(cls, src): ops = [cls.parse_op(line) for line in src.splitlines()] + ops.append(('--end--', None, [], '...', True)) return [op for op in ops if op is not None] @classmethod @@ -403,6 +404,10 @@ raise InvalidMatch(message, frame=sys._getframe(1)) def match_op(self, op, (exp_opname, exp_res, exp_args, exp_descr, _)): + if exp_opname == '--end--': + self._assert(op == '--end--', 'got more ops than expected') + return + self._assert(op != '--end--', 'got less ops than expected') self._assert(op.name == exp_opname, "operation mismatch") self.match_var(op.res, exp_res) if exp_args[-1:] == ['...']: # exp_args ends with '...' @@ -415,18 +420,15 @@ self.match_descr(op.descr, exp_descr) - def _next_op(self, iter_ops, assert_raises=False, ignore_ops=set()): + def _next_op(self, iter_ops, ignore_ops=set()): try: while True: op = iter_ops.next() if op.name not in ignore_ops: break except StopIteration: - self._assert(assert_raises, "not enough operations") - return - else: - self._assert(not assert_raises, "operation list too long") - return op + return '--end--' + return op def try_match(self, op, exp_op): try: @@ -493,16 +495,17 @@ continue else: op = self._next_op(iter_ops, ignore_ops=ignore_ops) - self.match_op(op, exp_op) - except InvalidMatch, e: - if type(exp_op) is not str and exp_op[4] is False: # optional operation + try: + self.match_op(op, exp_op) + except InvalidMatch: + if type(exp_op) is str or exp_op[4] is not False: + raise + #else: optional operation iter_ops.revert_one() continue # try to match with the next exp_op + except InvalidMatch, e: e.opindex = iter_ops.index - 1 raise - # - # make sure we exhausted iter_ops - self._next_op(iter_ops, assert_raises=True, ignore_ops=ignore_ops) def match(self, expected_src, ignore_ops=[]): def format(src, opindex=None): @@ -545,9 +548,9 @@ return self def next(self): index = self.index - if index == len(self.sequence): + self.index = index + 1 + if index >= len(self.sequence): raise StopIteration - self.index = index + 1 return self.sequence[index] def revert_one(self): self.index -= 1 diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py --- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py +++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py @@ -384,6 +384,25 @@ """ assert not self.match(loop, expected) + def test_match_optional_op(self): + loop = """ + i1 = int_add(i0, 1) + """ + expected = """ + guard_not_invalidated? + i1 = int_add(i0, 1) + """ + assert self.match(loop, expected) + # + loop = """ + i1 = int_add(i0, 1) + """ + expected = """ + i1 = int_add(i0, 1) + guard_not_invalidated? + """ + assert self.match(loop, expected) + class TestRunPyPyC(BaseTestPyPyC): def test_run_function(self): diff --git a/pypy/module/pypyjit/test_pypy_c/test_array.py b/pypy/module/pypyjit/test_pypy_c/test_array.py --- a/pypy/module/pypyjit/test_pypy_c/test_array.py +++ b/pypy/module/pypyjit/test_pypy_c/test_array.py @@ -94,13 +94,25 @@ guard_not_invalidated(descr=...) # the bound check guard on img has been killed (thanks to the asserts) i14 = getarrayitem_raw(i10, i8, descr=) + # advanced: the following int_add cannot overflow, because: + # - i14 fits inside 32 bits + # - i9 fits inside 33 bits, because: + # - it comes from the previous iteration's i15 + # - prev i19 = prev i18 + prev i15 + # - prev i18 fits inside 32 bits + # - prev i19 is guarded to fit inside 32 bits + # - so as a consequence, prev i15 fits inside 33 bits + # the new i15 thus fits inside "33.5" bits, which is enough to + # guarantee that the next int_add(i18, i15) cannot overflow either... i15 = int_add(i9, i14) i17 = int_sub(i8, 640) # the bound check guard on intimg has been killed (thanks to the asserts) i18 = getarrayitem_raw(i11, i17, descr=) i19 = int_add(i18, i15) - # on 64bit, there is a guard checking that i19 actually fits into 32bit - ... + # guard checking that i19 actually fits into 32bit + i20 = int_signext(i19, 4) + i65 = int_ne(i20, i19) + guard_false(i65, descr=...) setarrayitem_raw(i11, i8, _, descr=) i28 = int_add(i8, 1) --TICK-- diff --git a/pypy/tool/ann_override.py b/pypy/tool/ann_override.py --- a/pypy/tool/ann_override.py +++ b/pypy/tool/ann_override.py @@ -13,13 +13,12 @@ class PyPyAnnotatorPolicy(AnnotatorPolicy): - def __init__(pol, single_space=None): - pol.lookups = {} - pol.lookups_where = {} - pol.pypytypes = {} - pol.single_space = single_space + def __init__(self): + self.lookups = {} + self.lookups_where = {} + self.pypytypes = {} - def specialize__wrap(pol, funcdesc, args_s): + def specialize__wrap(self, funcdesc, args_s): from pypy.interpreter.baseobjspace import W_Root from rpython.annotator.classdef import ClassDef W_Root_def = funcdesc.bookkeeper.getuniqueclassdef(W_Root) @@ -51,102 +50,102 @@ typ = (None, str) return funcdesc.cachedgraph(typ) - def _remember_immutable(pol, t, cached): + def _remember_immutable(self, t, cached): # for jit benefit if cached not in t._immutable_fields_: # accessed this way just # for convenience t._immutable_fields_.append(cached) - def attach_lookup(pol, t, attr): + def attach_lookup(self, t, attr): cached = "cached_%s" % attr if not t.is_heaptype() and not t.is_cpytype(): - pol._remember_immutable(t, cached) + self._remember_immutable(t, cached) setattr(t, cached, t._lookup(attr)) return True return False - def attach_lookup_in_type_where(pol, t, attr): + def attach_lookup_in_type_where(self, t, attr): cached = "cached_where_%s" % attr if not t.is_heaptype() and not t.is_cpytype(): - pol._remember_immutable(t, cached) + self._remember_immutable(t, cached) setattr(t, cached, t._lookup_where(attr)) return True return False - def consider_lookup(pol, bookkeeper, attr): + def consider_lookup(self, bookkeeper, attr): from rpython.annotator.classdef import InstanceSource - assert attr not in pol.lookups + assert attr not in self.lookups from pypy.objspace.std import typeobject cached = "cached_%s" % attr clsdef = bookkeeper.getuniqueclassdef(typeobject.W_TypeObject) classdesc = clsdef.classdesc classdesc.classdict[cached] = Constant(None) clsdef.add_source_for_attribute(cached, classdesc) - for t in pol.pypytypes: - if pol.attach_lookup(t, attr): + for t in self.pypytypes: + if self.attach_lookup(t, attr): source = InstanceSource(bookkeeper, t) clsdef.add_source_for_attribute(cached, source) - pol.lookups[attr] = True + self.lookups[attr] = True - def consider_lookup_in_type_where(pol, bookkeeper, attr): + def consider_lookup_in_type_where(self, bookkeeper, attr): from rpython.annotator.classdef import InstanceSource - assert attr not in pol.lookups_where + assert attr not in self.lookups_where from pypy.objspace.std import typeobject cached = "cached_where_%s" % attr clsdef = bookkeeper.getuniqueclassdef(typeobject.W_TypeObject) classdesc = clsdef.classdesc classdesc.classdict[cached] = Constant((None, None)) clsdef.add_source_for_attribute(cached, classdesc) - for t in pol.pypytypes: - if pol.attach_lookup_in_type_where(t, attr): + for t in self.pypytypes: + if self.attach_lookup_in_type_where(t, attr): source = InstanceSource(bookkeeper, t) clsdef.add_source_for_attribute(cached, source) - pol.lookups_where[attr] = True + self.lookups_where[attr] = True - def specialize__lookup(pol, funcdesc, args_s): + def specialize__lookup(self, funcdesc, args_s): s_space, s_obj, s_name = args_s if s_name.is_constant(): attr = s_name.const def builder(translator, func): #print "LOOKUP", attr - pol.consider_lookup(funcdesc.bookkeeper, attr) + self.consider_lookup(funcdesc.bookkeeper, attr) d = {'__name__': ''} exec CACHED_LOOKUP % {'attr': attr} in d return translator.buildflowgraph(d['lookup_'+attr]) return funcdesc.cachedgraph(attr, builder=builder) else: - pol.lookups[None] = True + self.lookups[None] = True return funcdesc.cachedgraph(None) # don't specialize - def specialize__lookup_in_type_where(pol, funcdesc, args_s): + def specialize__lookup_in_type_where(self, funcdesc, args_s): s_space, s_obj, s_name = args_s if s_name.is_constant(): attr = s_name.const def builder(translator, func): #print "LOOKUP_IN_TYPE_WHERE", attr - pol.consider_lookup_in_type_where(funcdesc.bookkeeper, attr) + self.consider_lookup_in_type_where(funcdesc.bookkeeper, attr) d = {'__name__': ''} exec CACHED_LOOKUP_IN_TYPE_WHERE % {'attr': attr} in d return translator.buildflowgraph(d['lookup_in_type_where_'+attr]) return funcdesc.cachedgraph(attr, builder=builder) else: - pol.lookups_where[None] = True + self.lookups_where[None] = True return funcdesc.cachedgraph(None) - def event(pol, bookkeeper, what, x): + def event(self, bookkeeper, what, x): from pypy.objspace.std import typeobject if isinstance(x, typeobject.W_TypeObject): from rpython.annotator.classdef import InstanceSource clsdef = bookkeeper.getuniqueclassdef(typeobject.W_TypeObject) - pol.pypytypes[x] = True + self.pypytypes[x] = True #print "TYPE", x - for attr in pol.lookups: - if attr and pol.attach_lookup(x, attr): + for attr in self.lookups: + if attr and self.attach_lookup(x, attr): cached = "cached_%s" % attr source = InstanceSource(bookkeeper, x) clsdef.add_source_for_attribute(cached, source) - for attr in pol.lookups_where: - if attr and pol.attach_lookup_in_type_where(x, attr): + for attr in self.lookups_where: + if attr and self.attach_lookup_in_type_where(x, attr): cached = "cached_where_%s" % attr source = InstanceSource(bookkeeper, x) clsdef.add_source_for_attribute(cached, source) diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py --- a/rpython/annotator/annrpython.py +++ b/rpython/annotator/annrpython.py @@ -12,6 +12,7 @@ from rpython.annotator import model as annmodel, signature from rpython.annotator.argument import simple_args from rpython.annotator.bookkeeper import Bookkeeper +from rpython.rtyper.normalizecalls import perform_normalizations import py log = py.log.Producer("annrpython") @@ -317,6 +318,8 @@ graphs[graph] = True for graph in graphs: simplify.eliminate_empty_blocks(graph) + if block_subset is None: + perform_normalizations(self) #___ flowing annotations in blocks _____________________ diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py --- a/rpython/annotator/bookkeeper.py +++ b/rpython/annotator/bookkeeper.py @@ -42,7 +42,7 @@ def __setstate__(self, dic): self.__dict__.update(dic) # normal action - delayed_imports() + self.register_builtins() def __init__(self, annotator): self.annotator = annotator @@ -67,7 +67,13 @@ self.needs_generic_instantiate = {} self.thread_local_fields = set() - delayed_imports() + self.register_builtins() + + def register_builtins(self): + import rpython.annotator.builtin # for side-effects + from rpython.annotator.exception import standardexceptions + for cls in standardexceptions: + self.getuniqueclassdef(cls) def enter(self, position_key): """Start of an operation. @@ -105,8 +111,7 @@ for pbc, args_s in self.emulated_pbc_calls.itervalues(): args = simple_args(args_s) - self.consider_call_site_for_pbc(pbc, args, - s_ImpossibleValue, None) + pbc.consider_call_site(args, s_ImpossibleValue, None) self.emulated_pbc_calls = {} finally: self.leave() @@ -157,15 +162,7 @@ if s_result is None: s_result = s_ImpossibleValue args = call_op.build_args(args_s) - self.consider_call_site_for_pbc(s_callable, args, - s_result, call_op) - - def consider_call_site_for_pbc(self, s_callable, args, s_result, - call_op): - descs = list(s_callable.descriptions) - family = descs[0].getcallfamily() - s_callable.getKind().consider_call_site(self, family, descs, args, - s_result, call_op) + s_callable.consider_call_site(args, s_result, call_op) def getuniqueclassdef(self, cls): """Get the ClassDef associated with the given user cls. @@ -605,6 +602,3 @@ def immutablevalue(x): return getbookkeeper().immutablevalue(x) - -def delayed_imports(): - import rpython.annotator.builtin diff --git a/rpython/annotator/classdef.py b/rpython/annotator/classdef.py --- a/rpython/annotator/classdef.py +++ b/rpython/annotator/classdef.py @@ -2,8 +2,7 @@ Type inference for user-defined classes. """ from rpython.annotator.model import ( - SomePBC, s_ImpossibleValue, unionof, s_None, SomeInteger, - SomeTuple, SomeString, AnnotatorError, SomeUnicodeString) + SomePBC, s_ImpossibleValue, unionof, s_None, AnnotatorError) from rpython.annotator import description @@ -437,18 +436,3 @@ class NoSuchAttrError(AnnotatorError): """Raised when an attribute is found on a class where __slots__ or _attrs_ forbits it.""" - -# ____________________________________________________________ - -FORCE_ATTRIBUTES_INTO_CLASSES = { - EnvironmentError: {'errno': SomeInteger(), - 'strerror': SomeString(can_be_None=True), - 'filename': SomeString(can_be_None=True)}, -} - -try: - WindowsError -except NameError: - pass -else: - FORCE_ATTRIBUTES_INTO_CLASSES[WindowsError] = {'winerror': SomeInteger()} diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py --- a/rpython/annotator/description.py +++ b/rpython/annotator/description.py @@ -7,7 +7,7 @@ from rpython.annotator.argument import rawshape, ArgErr from rpython.tool.sourcetools import valid_identifier, func_with_new_name from rpython.tool.pairtype import extendabletype -from rpython.annotator.model import AnnotatorError +from rpython.annotator.model import AnnotatorError, SomeInteger, SomeString class CallFamily(object): """A family of Desc objects that could be called from common call sites. @@ -329,9 +329,10 @@ name) @staticmethod - def consider_call_site(bookkeeper, family, descs, args, s_result, op): + def consider_call_site(descs, args, s_result, op): shape = rawshape(args) row = FunctionDesc.row_to_consider(descs, args, op) + family = descs[0].getcallfamily() family.calltable_add_row(shape, row) @staticmethod @@ -404,6 +405,8 @@ name=None, basedesc=None, classdict=None, specialize=None): super(ClassDesc, self).__init__(bookkeeper, cls) + if '__NOT_RPYTHON__' in cls.__dict__: + raise AnnotatorError('Bad class') if name is None: name = cls.__module__ + '.' + cls.__name__ @@ -477,8 +480,7 @@ if (self.is_builtin_exception_class() and self.all_enforced_attrs is None): - from rpython.annotator import classdef - if cls not in classdef.FORCE_ATTRIBUTES_INTO_CLASSES: + if cls not in FORCE_ATTRIBUTES_INTO_CLASSES: self.all_enforced_attrs = [] # no attribute allowed def add_source_attribute(self, name, value, mixin=False): @@ -573,8 +575,7 @@ try: return self._classdefs[key] except KeyError: - from rpython.annotator.classdef import ( - ClassDef, FORCE_ATTRIBUTES_INTO_CLASSES) + from rpython.annotator.classdef import ClassDef classdef = ClassDef(self.bookkeeper, self) self.bookkeeper.classdefs.append(classdef) self._classdefs[key] = classdef @@ -760,7 +761,7 @@ return s_result # common case @staticmethod - def consider_call_site(bookkeeper, family, descs, args, s_result, op): + def consider_call_site(descs, args, s_result, op): from rpython.annotator.model import SomeInstance, SomePBC, s_None if len(descs) == 1: # call to a single class, look at the result annotation @@ -795,17 +796,14 @@ "unexpected dynamic __init__?") initfuncdesc, = s_init.descriptions if isinstance(initfuncdesc, FunctionDesc): - initmethdesc = bookkeeper.getmethoddesc(initfuncdesc, - classdef, - classdef, - '__init__') + from rpython.annotator.bookkeeper import getbookkeeper + initmethdesc = getbookkeeper().getmethoddesc( + initfuncdesc, classdef, classdef, '__init__') initdescs.append(initmethdesc) # register a call to exactly these __init__ methods if initdescs: initdescs[0].mergecallfamilies(*initdescs[1:]) - initfamily = initdescs[0].getcallfamily() - MethodDesc.consider_call_site(bookkeeper, initfamily, initdescs, - args, s_None, op) + MethodDesc.consider_call_site(initdescs, args, s_None, op) def getallbases(self): desc = self @@ -897,10 +895,11 @@ flags) @staticmethod - def consider_call_site(bookkeeper, family, descs, args, s_result, op): + def consider_call_site(descs, args, s_result, op): cnt, keys, star = rawshape(args) shape = cnt + 1, keys, star # account for the extra 'self' row = FunctionDesc.row_to_consider(descs, args, op) + family = descs[0].getcallfamily() family.calltable_add_row(shape, row) def rowkey(self): @@ -1058,10 +1057,11 @@ return self.funcdesc.pycall(schedule, args, s_previous_result, op) @staticmethod - def consider_call_site(bookkeeper, family, descs, args, s_result, op): + def consider_call_site(descs, args, s_result, op): cnt, keys, star = rawshape(args) shape = cnt + 1, keys, star # account for the extra 'self' row = FunctionDesc.row_to_consider(descs, args, op) + family = descs[0].getcallfamily() family.calltable_add_row(shape, row) def rowkey(self): @@ -1077,3 +1077,18 @@ MemberDescriptorTypes.append(type(OSError.errno)) except AttributeError: # on CPython <= 2.4 pass + +# ____________________________________________________________ + +FORCE_ATTRIBUTES_INTO_CLASSES = { + EnvironmentError: {'errno': SomeInteger(), + 'strerror': SomeString(can_be_None=True), + 'filename': SomeString(can_be_None=True)}, +} + +try: + WindowsError +except NameError: + pass +else: + FORCE_ATTRIBUTES_INTO_CLASSES[WindowsError] = {'winerror': SomeInteger()} diff --git a/rpython/annotator/exception.py b/rpython/annotator/exception.py new file mode 100644 --- /dev/null +++ b/rpython/annotator/exception.py @@ -0,0 +1,7 @@ +from rpython.rlib import rstackovf + +# the exceptions that can be implicitely raised by some operations +standardexceptions = set([TypeError, OverflowError, ValueError, + ZeroDivisionError, MemoryError, IOError, OSError, StopIteration, KeyError, + IndexError, AssertionError, RuntimeError, UnicodeDecodeError, + UnicodeEncodeError, NotImplementedError, rstackovf._StackOverflow]) diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py --- a/rpython/annotator/model.py +++ b/rpython/annotator/model.py @@ -495,6 +495,10 @@ if len(self.descriptions) > 1: kind.simplify_desc_set(self.descriptions) + def consider_call_site(self, args, s_result, call_op): + descs = list(self.descriptions) + self.getKind().consider_call_site(descs, args, s_result, call_op) + def can_be_none(self): return self.can_be_None @@ -588,7 +592,7 @@ class SomeProperty(SomeObject): - # used for union error only + # used for union error only immutable = True knowntype = type(property) diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -47,6 +47,11 @@ too slow for normal use. Values are 0 (off), 1 (on major collections) or 2 (also on minor collections). + + PYPY_GC_MAX_PINNED The maximal number of pinned objects at any point + in time. Defaults to a conservative value depending + on nursery size and maximum object size inside the + nursery. Useful for debugging by setting it to 0. """ # XXX Should find a way to bound the major collection threshold by the # XXX total addressable size. Maybe by keeping some minimarkpage arenas @@ -56,6 +61,7 @@ # XXX try merging old_objects_pointing_to_pinned into # XXX old_objects_pointing_to_young (IRC 2014-10-22, fijal and gregor_w) import sys +import os from rpython.rtyper.lltypesystem import lltype, llmemory, llarena, llgroup from rpython.rtyper.lltypesystem.lloperation import llop from rpython.rtyper.lltypesystem.llmemory import raw_malloc_usage @@ -463,9 +469,19 @@ self.nursery_size = newsize self.allocate_nursery() # - # Estimate this number conservatively - bigobj = self.nonlarge_max + 1 - self.max_number_of_pinned_objects = self.nursery_size / (bigobj * 2) + env_max_number_of_pinned_objects = os.environ.get('PYPY_GC_MAX_PINNED') + if env_max_number_of_pinned_objects: + try: + env_max_number_of_pinned_objects = int(env_max_number_of_pinned_objects) + except ValueError: + env_max_number_of_pinned_objects = 0 + # + if env_max_number_of_pinned_objects >= 0: # 0 allows to disable pinning completely + self.max_number_of_pinned_objects = env_max_number_of_pinned_objects + else: + # Estimate this number conservatively + bigobj = self.nonlarge_max + 1 + self.max_number_of_pinned_objects = self.nursery_size / (bigobj * 2) def _nursery_memory_size(self): extra = self.nonlarge_max + 1 diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -36,7 +36,10 @@ return graphanalyze.BoolGraphAnalyzer.analyze_direct_call(self, graph, seen) def analyze_external_call(self, op, seen=None): - funcobj = op.args[0].value._obj + try: + funcobj = op.args[0].value._obj + except lltype.DelayedPointer: + return True if getattr(funcobj, 'random_effects_on_gcobjs', False): return True return graphanalyze.BoolGraphAnalyzer.analyze_external_call(self, op, @@ -248,6 +251,8 @@ annhelper.finish() # at this point, annotate all mix-level helpers annhelper.backend_optimize() + self.check_custom_trace_funcs(gcdata.gc, translator.rtyper) + self.collect_analyzer = CollectAnalyzer(self.translator) self.collect_analyzer.analyze_all() @@ -537,6 +542,24 @@ self.gcdata._has_got_custom_trace(self.get_type_id(TP)) specialize.arg(2)(func) + def check_custom_trace_funcs(self, gc, rtyper): + # detect if one of the custom trace functions uses the GC + # (it must not!) + for TP, func in rtyper.custom_trace_funcs: + def no_op_callback(obj, arg): + pass + def ll_check_no_collect(obj): + func(gc, obj, no_op_callback, None) + annhelper = annlowlevel.MixLevelHelperAnnotator(rtyper) + graph1 = annhelper.getgraph(ll_check_no_collect, [SomeAddress()], + annmodel.s_None) + annhelper.finish() + collect_analyzer = CollectAnalyzer(self.translator) + if collect_analyzer.analyze_direct_call(graph1): + raise Exception( + "the custom trace hook %r for %r can cause " + "the GC to be called!" % (func, TP)) + def consider_constant(self, TYPE, value): self.layoutbuilder.consider_constant(TYPE, value, self.gcdata.gc) diff --git a/rpython/memory/gctransform/test/test_framework.py b/rpython/memory/gctransform/test/test_framework.py --- a/rpython/memory/gctransform/test/test_framework.py +++ b/rpython/memory/gctransform/test/test_framework.py @@ -143,6 +143,31 @@ expected = "'no_collect' function can trigger collection: = 0x10001000 + def external(name, argtypes, restype, **kw): kw['compilation_info'] = eci @@ -284,6 +286,9 @@ ssl_external('SSL_get_SSL_CTX', [SSL], SSL_CTX) ssl_external('SSL_set_SSL_CTX', [SSL, SSL_CTX], SSL_CTX) ssl_external('TLSv1_method', [], SSL_METHOD) +if HAVE_TLSv1_2: + ssl_external('TLSv1_1_method', [], SSL_METHOD) + ssl_external('TLSv1_2_method', [], SSL_METHOD) ssl_external('SSLv2_method', [], SSL_METHOD) ssl_external('SSLv3_method', [], SSL_METHOD) ssl_external('SSLv23_method', [], SSL_METHOD) diff --git a/rpython/rlib/rzlib.py b/rpython/rlib/rzlib.py --- a/rpython/rlib/rzlib.py +++ b/rpython/rlib/rzlib.py @@ -365,10 +365,8 @@ """Common code for compress() and decompress(). """ # Prepare the input buffer for the stream - with lltype.scoped_alloc(rffi.CCHARP.TO, len(data)) as inbuf: - # XXX (groggi) should be possible to improve this with pinning by - # not performing the 'copy_string_to_raw' if non-movable/pinned - copy_string_to_raw(llstr(data), inbuf, 0, len(data)) + assert data is not None # XXX seems to be sane assumption, however not for sure + with rffi.scoped_nonmovingbuffer(data) as inbuf: stream.c_next_in = rffi.cast(Bytefp, inbuf) rffi.setintfield(stream, 'c_avail_in', len(data)) diff --git a/rpython/rtyper/annlowlevel.py b/rpython/rtyper/annlowlevel.py --- a/rpython/rtyper/annlowlevel.py +++ b/rpython/rtyper/annlowlevel.py @@ -41,9 +41,10 @@ __repr__ = __str__ class LowLevelAnnotatorPolicy(AnnotatorPolicy): - def __init__(pol, rtyper=None): - pol.rtyper = rtyper + def __init__(self, rtyper=None): + self.rtyper = rtyper + @staticmethod def lowlevelspecialize(funcdesc, args_s, key_for_args): args_s, key1, builder = flatten_star_args(funcdesc, args_s) key = [] @@ -73,21 +74,20 @@ flowgraph = funcdesc.cachedgraph(key, builder=builder) args_s[:] = new_args_s return flowgraph - lowlevelspecialize = staticmethod(lowlevelspecialize) + @staticmethod def default_specialize(funcdesc, args_s): return LowLevelAnnotatorPolicy.lowlevelspecialize(funcdesc, args_s, {}) - default_specialize = staticmethod(default_specialize) specialize__ll = default_specialize + @staticmethod def specialize__ll_and_arg(funcdesc, args_s, *argindices): keys = {} for i in argindices: keys[i] = args_s[i].const return LowLevelAnnotatorPolicy.lowlevelspecialize(funcdesc, args_s, keys) - specialize__ll_and_arg = staticmethod(specialize__ll_and_arg) def annotate_lowlevel_helper(annotator, ll_function, args_s, policy=None): if policy is None: @@ -99,24 +99,23 @@ class MixLevelAnnotatorPolicy(LowLevelAnnotatorPolicy): - def __init__(pol, annhelper): - pol.annhelper = annhelper - pol.rtyper = annhelper.rtyper + def __init__(self, annhelper): + self.rtyper = annhelper.rtyper - def default_specialize(pol, funcdesc, args_s): + def default_specialize(self, funcdesc, args_s): name = funcdesc.name if name.startswith('ll_') or name.startswith('_ll_'): # xxx can we do better? - return super(MixLevelAnnotatorPolicy, pol).default_specialize( + return super(MixLevelAnnotatorPolicy, self).default_specialize( funcdesc, args_s) else: return AnnotatorPolicy.default_specialize(funcdesc, args_s) - def specialize__arglltype(pol, funcdesc, args_s, i): - key = pol.rtyper.getrepr(args_s[i]).lowleveltype + def specialize__arglltype(self, funcdesc, args_s, i): + key = self.rtyper.getrepr(args_s[i]).lowleveltype alt_name = funcdesc.name+"__for_%sLlT" % key._short_name() return funcdesc.cachedgraph(key, alt_name=valid_identifier(alt_name)) - def specialize__genconst(pol, funcdesc, args_s, i): + def specialize__genconst(self, funcdesc, args_s, i): # XXX this is specific to the JIT TYPE = annotation_to_lltype(args_s[i], 'genconst') args_s[i] = lltype_to_annotation(TYPE) diff --git a/rpython/rtyper/exceptiondata.py b/rpython/rtyper/exceptiondata.py --- a/rpython/rtyper/exceptiondata.py +++ b/rpython/rtyper/exceptiondata.py @@ -1,15 +1,9 @@ from rpython.annotator import model as annmodel +from rpython.annotator.exception import standardexceptions from rpython.rtyper.llannotation import SomePtr -from rpython.rlib import rstackovf from rpython.rtyper.rclass import ( ll_issubclass, ll_type, ll_cast_to_object, getclassrepr, getinstancerepr) -# the exceptions that can be implicitely raised by some operations -standardexceptions = set([TypeError, OverflowError, ValueError, - ZeroDivisionError, MemoryError, IOError, OSError, StopIteration, KeyError, - IndexError, AssertionError, RuntimeError, UnicodeDecodeError, - UnicodeEncodeError, NotImplementedError, rstackovf._StackOverflow]) - class UnknownException(Exception): pass @@ -20,7 +14,6 @@ standardexceptions = standardexceptions def __init__(self, rtyper): - self.make_standard_exceptions(rtyper) # (NB. rclass identifies 'Exception' and 'object') r_type = rtyper.rootclass_repr r_instance = getinstancerepr(rtyper, None) @@ -32,11 +25,6 @@ self.lltype_of_exception_value = r_instance.lowleveltype self.rtyper = rtyper - def make_standard_exceptions(self, rtyper): - bk = rtyper.annotator.bookkeeper - for cls in self.standardexceptions: - bk.getuniqueclassdef(cls) - def finish(self, rtyper): bk = rtyper.annotator.bookkeeper for cls in self.standardexceptions: diff --git a/rpython/rtyper/extfunc.py b/rpython/rtyper/extfunc.py --- a/rpython/rtyper/extfunc.py +++ b/rpython/rtyper/extfunc.py @@ -157,12 +157,11 @@ r_result = rtyper.getrepr(s_result) ll_result = r_result.lowleveltype name = getattr(self, 'name', None) or self.instance.__name__ - method_name = rtyper.type_system.name[:2] + 'typeimpl' fake_method_name = rtyper.type_system.name[:2] + 'typefakeimpl' - impl = getattr(self, method_name, None) - fakeimpl = getattr(self, fake_method_name, self.instance) + impl = getattr(self, 'lltypeimpl', None) + fakeimpl = getattr(self, 'lltypefakeimpl', self.instance) if impl: - if hasattr(self, fake_method_name): + if hasattr(self, 'lltypefakeimpl'): # If we have both an llimpl and an llfakeimpl, # we need a wrapper that selects the proper one and calls it from rpython.tool.sourcetools import func_with_new_name diff --git a/rpython/rtyper/lltypesystem/rbuilder.py b/rpython/rtyper/lltypesystem/rbuilder.py --- a/rpython/rtyper/lltypesystem/rbuilder.py +++ b/rpython/rtyper/lltypesystem/rbuilder.py @@ -401,18 +401,6 @@ def empty(self): return nullptr(self.lowleveltype.TO) - @classmethod - def ll_new(cls, init_size): - # Clamp 'init_size' to be a value between 0 and 1280. - # Negative values are mapped to 1280. - init_size = intmask(min(r_uint(init_size), r_uint(1280))) - ll_builder = lltype.malloc(cls.lowleveltype.TO) - ll_builder.current_buf = ll_builder.mallocfn(init_size) - ll_builder.current_pos = 0 - ll_builder.current_end = init_size - ll_builder.total_size = init_size - return ll_builder - ll_append = staticmethod(ll_append) ll_append_char = staticmethod(ll_append_char) ll_append_slice = staticmethod(ll_append_slice) @@ -431,6 +419,19 @@ lltype.Ptr(lltype.Array(lltype.Char, hints={'nolength': True})) ) + @staticmethod + def ll_new(init_size): + # Clamp 'init_size' to be a value between 0 and 1280. + # Negative values are mapped to 1280. + init_size = intmask(min(r_uint(init_size), r_uint(1280))) + ll_builder = lltype.malloc(STRINGBUILDER) + ll_builder.current_buf = ll_builder.mallocfn(init_size) + ll_builder.current_pos = 0 + ll_builder.current_end = init_size + ll_builder.total_size = init_size + return ll_builder + + class UnicodeBuilderRepr(BaseStringBuilderRepr): lowleveltype = lltype.Ptr(UNICODEBUILDER) basetp = UNICODE @@ -440,5 +441,18 @@ lltype.Ptr(lltype.Array(lltype.UniChar, hints={'nolength': True})) ) + @staticmethod + def ll_new(init_size): + # Clamp 'init_size' to be a value between 0 and 1280. + # Negative values are mapped to 1280. + init_size = intmask(min(r_uint(init_size), r_uint(1280))) + ll_builder = lltype.malloc(UNICODEBUILDER) + ll_builder.current_buf = ll_builder.mallocfn(init_size) + ll_builder.current_pos = 0 + ll_builder.current_end = init_size + ll_builder.total_size = init_size + return ll_builder + + unicodebuilder_repr = UnicodeBuilderRepr() stringbuilder_repr = StringBuilderRepr() diff --git a/rpython/rtyper/rmodel.py b/rpython/rtyper/rmodel.py --- a/rpython/rtyper/rmodel.py +++ b/rpython/rtyper/rmodel.py @@ -25,6 +25,7 @@ """ __metaclass__ = extendabletype _initialized = setupstate.NOTINITIALIZED + __NOT_RPYTHON__ = True def __repr__(self): return '<%s %s>' % (self.__class__.__name__, self.lowleveltype) diff --git a/rpython/rtyper/rtyper.py b/rpython/rtyper/rtyper.py --- a/rpython/rtyper/rtyper.py +++ b/rpython/rtyper/rtyper.py @@ -26,7 +26,6 @@ attachRuntimeTypeInfo, Primitive) from rpython.rtyper.rmodel import Repr, inputconst, BrokenReprTyperError from rpython.rtyper.typesystem import LowLevelTypeSystem, getfunctionptr -from rpython.rtyper.normalizecalls import perform_normalizations from rpython.rtyper import rclass from rpython.rtyper.rclass import RootClassRepr from rpython.tool.pairtype import pair @@ -55,8 +54,6 @@ self.concrete_calltables = {} self.cache_dummy_values = {} self.lltype2vtable = {} - self.typererrors = [] - self.typererror_count = 0 # make the primitive_to_repr constant mapping self.primitive_to_repr = {} self.isinstance_helpers = {} @@ -169,22 +166,16 @@ def specialize(self, dont_simplify_again=False): """Main entry point: specialize all annotated blocks of the program.""" # specialize depends on annotator simplifications - assert dont_simplify_again in (False, True) # safety check if not dont_simplify_again: self.annotator.simplify() - - # first make sure that all functions called in a group have exactly - # the same signature, by hacking their flow graphs if needed - perform_normalizations(self.annotator) self.exceptiondata.finish(self) # new blocks can be created as a result of specialize_block(), so # we need to be careful about the loop here. self.already_seen = {} self.specialize_more_blocks() - if self.exceptiondata is not None: - self.exceptiondata.make_helpers(self) - self.specialize_more_blocks() # for the helpers just made + self.exceptiondata.make_helpers(self) + self.specialize_more_blocks() # for the helpers just made def getannmixlevel(self): if self.annmixlevel is not None: @@ -231,18 +222,11 @@ percentage = 100 * n // total if percentage >= previous_percentage + 5: previous_percentage = percentage - if self.typererror_count: - error_report = " but %d errors" % self.typererror_count - else: - error_report = '' - self.log.event('specializing: %d / %d blocks (%d%%)%s' % - (n, total, percentage, error_report)) + self.log.event('specializing: %d / %d blocks (%d%%)' % + (n, total, percentage)) # make sure all reprs so far have had their setup() called self.call_all_setups() - if self.typererrors: - self.dump_typererrors(to_log=True) - raise TyperError("there were %d error" % len(self.typererrors)) self.log.event('-=- specialized %d%s blocks -=-' % ( blockcount, newtext)) annmixlevel = self.annmixlevel @@ -250,29 +234,6 @@ if annmixlevel is not None: annmixlevel.finish() - def dump_typererrors(self, num=None, minimize=True, to_log=False): - c = 0 - bc = 0 - for err in self.typererrors[:num]: - c += 1 - if minimize and isinstance(err, BrokenReprTyperError): - bc += 1 - continue - graph, block, position = err.where - errmsg = ("TyperError-%d: %s\n" % (c, graph) + - str(err) + - "\n") - if to_log: - self.log.ERROR(errmsg) - else: - print errmsg - if bc: - minmsg = "(minimized %d errors away for this dump)" % (bc,) - if to_log: - self.log.ERROR(minmsg) - else: - print minmsg - def call_all_setups(self): # make sure all reprs so far have had their setup() called must_setup_more = [] @@ -324,9 +285,9 @@ # give the best possible types to the input args try: self.setup_block_entry(block) - except TyperError, e: - self.gottypererror(e, block, "block-entry", None) - return # cannot continue this block + except TyperError as e: + self.gottypererror(e, block, "block-entry") + raise # specialize all the operations, as far as possible @@ -341,9 +302,9 @@ try: hop.setup() # this is called from here to catch TyperErrors... self.translate_hl_to_ll(hop, varmapping) - except TyperError, e: - self.gottypererror(e, block, hop.spaceop, newops) - return # cannot continue this block: no op.result.concretetype + except TyperError as e: + self.gottypererror(e, block, hop.spaceop) + raise block.operations[:] = newops block.renamevariables(varmapping) @@ -432,9 +393,9 @@ continue # no conversion needed try: new_a1 = newops.convertvar(a1, r_a1, r_a2) - except TyperError, e: - self.gottypererror(e, block, link, newops) - continue # try other args + except TyperError as e: + self.gottypererror(e, block, link) + raise if new_a1 != a1: newlinkargs[i] = new_a1 @@ -516,14 +477,10 @@ "has no return value" % op.opname) op.result.concretetype = Void - def gottypererror(self, e, block, position, llops): - """Record a TyperError without crashing immediately. - Put a 'TyperError' operation in the graph instead. - """ + def gottypererror(self, exc, block, position): + """Record information about the location of a TyperError""" graph = self.annotator.annotated.get(block) - e.where = (graph, block, position) - self.typererror_count += 1 - raise + exc.where = (graph, block, position) # __________ regular operations __________ diff --git a/rpython/rtyper/test/test_llinterp.py b/rpython/rtyper/test/test_llinterp.py --- a/rpython/rtyper/test/test_llinterp.py +++ b/rpython/rtyper/test/test_llinterp.py @@ -25,22 +25,12 @@ py.log._setstate(mod.logstate) - -def timelog(prefix, call, *args, **kwds): - #import time - #print prefix, "...", - #start = time.time() - res = call(*args, **kwds) - #elapsed = time.time() - start - #print "%.2f secs" % (elapsed,) - return res - def gengraph(func, argtypes=[], viewbefore='auto', policy=None, backendopt=False, config=None, **extraconfigopts): t = TranslationContext(config=config) t.config.set(**extraconfigopts) a = t.buildannotator(policy=policy) - timelog("annotating", a.build_types, func, argtypes, main_entry_point=True) + a.build_types(func, argtypes, main_entry_point=True) a.validate() if viewbefore == 'auto': viewbefore = getattr(option, 'view', False) @@ -49,13 +39,13 @@ t.view() global typer # we need it for find_exception typer = t.buildrtyper() - timelog("rtyper-specializing", typer.specialize) + typer.specialize() #t.view() - timelog("checking graphs", t.checkgraphs) + t.checkgraphs() if backendopt: from rpython.translator.backendopt.all import backend_optimizations backend_optimizations(t) - timelog("checking graphs", t.checkgraphs) + t.checkgraphs() if viewbefore: t.view() desc = t.annotator.bookkeeper.getdesc(func) diff --git a/rpython/translator/backendopt/all.py b/rpython/translator/backendopt/all.py --- a/rpython/translator/backendopt/all.py +++ b/rpython/translator/backendopt/all.py @@ -151,8 +151,6 @@ inline_heuristic, call_count_pred=None, inline_graph_from_anywhere=False): - - type_system = translator.rtyper.type_system.name # inline functions in each other if inline_threshold: log.inlining("phase with threshold factor: %s" % inline_threshold) @@ -171,7 +169,7 @@ # vaporize mallocs if config.mallocs: log.malloc("starting malloc removal") - remove_mallocs(translator, graphs, type_system) + remove_mallocs(translator, graphs) if config.print_statistics: print "after malloc removal:" diff --git a/rpython/translator/backendopt/malloc.py b/rpython/translator/backendopt/malloc.py --- a/rpython/translator/backendopt/malloc.py +++ b/rpython/translator/backendopt/malloc.py @@ -536,17 +536,17 @@ raise AssertionError(op.opname) -def remove_simple_mallocs(graph, type_system='lltypesystem', verbose=True): +def remove_simple_mallocs(graph, verbose=True): remover = LLTypeMallocRemover(verbose) return remover.remove_simple_mallocs(graph) -def remove_mallocs(translator, graphs=None, type_system="lltypesystem"): +def remove_mallocs(translator, graphs=None): if graphs is None: graphs = translator.graphs tot = 0 for graph in graphs: - count = remove_simple_mallocs(graph, type_system=type_system, verbose=translator.config.translation.verbose) + count = remove_simple_mallocs(graph, verbose=translator.config.translation.verbose) if count: # remove typical leftovers from malloc removal removenoops.remove_same_as(graph) diff --git a/rpython/translator/backendopt/test/test_all.py b/rpython/translator/backendopt/test/test_all.py --- a/rpython/translator/backendopt/test/test_all.py +++ b/rpython/translator/backendopt/test/test_all.py @@ -42,7 +42,6 @@ HUGE_THRESHOLD = 100*INLINE_THRESHOLD_FOR_TEST class TestLLType(object): - type_system = 'lltype' check_malloc_removed = MallocRemovalTest.check_malloc_removed def translateopt(self, func, sig, **optflags): diff --git a/rpython/translator/backendopt/test/test_inline.py b/rpython/translator/backendopt/test/test_inline.py --- a/rpython/translator/backendopt/test/test_inline.py +++ b/rpython/translator/backendopt/test/test_inline.py @@ -47,8 +47,6 @@ self.data2 = 456 class TestInline(BaseRtypingTest): - type_system = 'lltype' - def translate(self, func, argtypes): t = TranslationContext() t.buildannotator().build_types(func, argtypes) diff --git a/rpython/translator/backendopt/test/test_malloc.py b/rpython/translator/backendopt/test/test_malloc.py --- a/rpython/translator/backendopt/test/test_malloc.py +++ b/rpython/translator/backendopt/test/test_malloc.py @@ -10,7 +10,6 @@ from rpython.conftest import option class TestMallocRemoval(object): - type_system = 'lltype' MallocRemover = LLTypeMallocRemover def check_malloc_removed(cls, graph): diff --git a/rpython/translator/backendopt/test/test_mallocv.py b/rpython/translator/backendopt/test/test_mallocv.py --- a/rpython/translator/backendopt/test/test_mallocv.py +++ b/rpython/translator/backendopt/test/test_mallocv.py @@ -17,8 +17,6 @@ class TestMallocRemoval(object): - type_system = 'lltype' - def check_malloc_removed(cls, graph, expected_mallocs, expected_calls): count_mallocs = 0 count_calls = 0 diff --git a/rpython/translator/backendopt/test/test_storesink.py b/rpython/translator/backendopt/test/test_storesink.py --- a/rpython/translator/backendopt/test/test_storesink.py +++ b/rpython/translator/backendopt/test/test_storesink.py @@ -7,8 +7,6 @@ from rpython.conftest import option class TestStoreSink(object): - type_system = 'lltype' - def translate(self, func, argtypes): t = TranslationContext() t.buildannotator().build_types(func, argtypes) diff --git a/rpython/translator/backendopt/test/test_writeanalyze.py b/rpython/translator/backendopt/test/test_writeanalyze.py --- a/rpython/translator/backendopt/test/test_writeanalyze.py +++ b/rpython/translator/backendopt/test/test_writeanalyze.py @@ -7,8 +7,6 @@ class BaseTest(object): - - type_system = 'lltype' Analyzer = WriteAnalyzer def translate(self, func, sig): diff --git a/rpython/translator/test/test_exceptiontransform.py b/rpython/translator/test/test_exceptiontransform.py --- a/rpython/translator/test/test_exceptiontransform.py +++ b/rpython/translator/test/test_exceptiontransform.py @@ -27,8 +27,6 @@ return interp.eval_graph(graph, values) class TestExceptionTransform: - type_system = 'lltype' - def compile(self, fn, inputargs): from rpython.translator.c.test.test_genc import compile return compile(fn, inputargs) @@ -239,7 +237,7 @@ etrafo.create_exception_handling(g) ops = dict.fromkeys([o.opname for b, o in g.iterblockops()]) assert 'zero_gc_pointers_inside' in ops - + def test_llexternal(self): from rpython.rtyper.lltypesystem.rffi import llexternal from rpython.rtyper.lltypesystem import lltype From noreply at buildbot.pypy.org Mon Mar 30 20:07:13 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 30 Mar 2015 20:07:13 +0200 (CEST) Subject: [pypy-commit] pypy object-dtype2: In-progress: define and use rgc.ll_writebarrier() where needed Message-ID: <20150330180713.702AC1C11DF@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: object-dtype2 Changeset: r76647:99280110fed2 Date: 2015-03-30 20:06 +0200 http://bitbucket.org/pypy/pypy/changeset/99280110fed2/ Log: In-progress: define and use rgc.ll_writebarrier() where needed diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -1649,14 +1649,18 @@ return boxes.W_ObjectBox(w_item) def store(self, arr, i, offset, box): - self._write(arr.storage, i, offset, self.unbox(box)) + self._write(arr.storage, i, offset, self.unbox(box), + arr.gcstruct) def read(self, arr, i, offset, dtype=None): return self.box(self._read(arr.storage, i, offset)) @jit.dont_look_inside - def _write(self, storage, i, offset, w_obj): + def _write(self, storage, i, offset, w_obj, gcstruct): + # no GC anywhere in this function! if we_are_translated(): + from rpython.rlib import rgc + rgc.ll_writebarrier(gcstruct) value = rffi.cast(lltype.Signed, cast_instance_to_gcref(w_obj)) else: value = len(_all_objs_for_tests) @@ -1676,7 +1680,7 @@ def fill(self, storage, width, box, start, stop, offset): value = self.unbox(box) for i in xrange(start, stop, width): - self._write(storage, i, offset, value) + self._write(storage, i, offset, value, XXX) def unbox(self, box): assert isinstance(box, self.BoxType) diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -681,6 +681,14 @@ call, for internal reasons. """ + at specialize.ll() +def ll_writebarrier(gc_obj): + """Use together with custom tracers. When you update some object pointer + stored in raw memory, you must call this function on 'gc_obj', which must + be the object of type TP with the custom tracer (*not* the value stored!). + This makes sure that the custom hook will be called again.""" + llop.gc_writebarrier(lltype.Void, gc_obj) + class RegisterGcTraceEntry(ExtRegistryEntry): _about_ = register_custom_trace_hook From noreply at buildbot.pypy.org Mon Mar 30 21:42:14 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 30 Mar 2015 21:42:14 +0200 (CEST) Subject: [pypy-commit] pypy default: 'transaction.py' can be used on cpython or regular pypy Message-ID: <20150330194214.653D41C0207@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76648:f588efa4658d Date: 2015-03-30 21:42 +0200 http://bitbucket.org/pypy/pypy/changeset/f588efa4658d/ Log: 'transaction.py' can be used on cpython or regular pypy diff --git a/pypy/doc/stm.rst b/pypy/doc/stm.rst --- a/pypy/doc/stm.rst +++ b/pypy/doc/stm.rst @@ -564,6 +564,15 @@ Miscellaneous functions ----------------------- +* First, note that the ``transaction`` module is found in the file + ``lib_pypy/transaction.py``. This file can be copied around to + execute the same programs on CPython or on non-STM PyPy, with + fall-back behavior. (One case where the behavior differs is + ``atomic``, which is in this fall-back case just a regular lock; so + ``with atomic`` only prevent other threads from entering other + ``with atomic`` sections, but won't prevent other threads from + running non-atomic code.) + * ``transaction.getsegmentlimit()``: return the number of "segments" in this pypy-stm. This is the limit above which more threads will not be able to execute on more cores. (Right now it is limited to 4 due to From noreply at buildbot.pypy.org Tue Mar 31 00:19:30 2015 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 31 Mar 2015 00:19:30 +0200 (CEST) Subject: [pypy-commit] pypy object-dtype2: missing import Message-ID: <20150330221930.481FC1C01D0@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: object-dtype2 Changeset: r76649:45f5b098008b Date: 2015-03-30 21:59 +0300 http://bitbucket.org/pypy/pypy/changeset/45f5b098008b/ Log: missing import diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -687,6 +687,7 @@ stored in raw memory, you must call this function on 'gc_obj', which must be the object of type TP with the custom tracer (*not* the value stored!). This makes sure that the custom hook will be called again.""" + from rpython.rtyper.lltypesystem.lloperation import llop llop.gc_writebarrier(lltype.Void, gc_obj) class RegisterGcTraceEntry(ExtRegistryEntry): From noreply at buildbot.pypy.org Tue Mar 31 00:19:31 2015 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 31 Mar 2015 00:19:31 +0200 (CEST) Subject: [pypy-commit] pypy object-dtype2: refactor fill, fix for value of pointers (arigato), take care around __del__, instantiate gcstruct in __init__ Message-ID: <20150330221931.7A8231C01D0@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: object-dtype2 Changeset: r76650:8258a094da18 Date: 2015-03-31 01:20 +0300 http://bitbucket.org/pypy/pypy/changeset/8258a094da18/ Log: refactor fill, fix for value of pointers (arigato), take care around __del__, instantiate gcstruct in __init__ diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py --- a/pypy/module/micronumpy/concrete.py +++ b/pypy/module/micronumpy/concrete.py @@ -346,8 +346,8 @@ def customtrace(gc, obj, callback, arg): #debug_print('in customtrace w/obj', obj) - length = rffi.cast(lltype.Signed, obj + offset_of_length) - step = rffi.cast(lltype.Signed, obj + offset_of_step) + length = rffi.cast(rffi.SIGNEDP, obj + offset_of_length)[0] + step = rffi.cast(rffi.SIGNEDP, obj + offset_of_step)[0] storage = obj + offset_of_storage debug_print('tracing', length, 'objects in ndarray.storage') i = 0 @@ -362,7 +362,6 @@ lambda_customtrace = lambda: customtrace def _setup(): - print 'registering custom trace for OBJECTSTORE' rgc.register_custom_trace_hook(OBJECTSTORE, lambda_customtrace) @jit.dont_look_inside @@ -393,7 +392,7 @@ def fill(self, space, box): self.dtype.itemtype.fill(self.storage, self.dtype.elsize, - box, 0, self.size, 0) + box, 0, self.size, 0, self.gcstruct) def set_shape(self, space, orig_array, new_shape): strides, backstrides = calc_strides(new_shape, self.dtype, @@ -423,14 +422,18 @@ gcstruct = lltype.nullptr(OBJECTSTORE) if storage == lltype.nullptr(RAW_STORAGE): length = support.product(shape) - storage = dtype.itemtype.malloc(length * dtype.elsize, zero=zero) if dtype.num == NPY.OBJECT: + storage = dtype.itemtype.malloc(length * dtype.elsize, zero=True) gcstruct = _create_objectstore(storage, length, dtype.elsize) + else: + storage = dtype.itemtype.malloc(length * dtype.elsize, zero=zero) ConcreteArrayNotOwning.__init__(self, shape, dtype, order, strides, backstrides, storage) self.gcstruct = gcstruct def __del__(self): + if self.gcstruct: + self.gcstruct.length = 0 free_raw_storage(self.storage, track_allocation=False) @@ -469,6 +472,7 @@ parent = parent.parent # one level only self.parent = parent self.storage = parent.storage + self.gcstruct = parent.gcstruct self.order = parent.order self.dtype = dtype self.size = support.product(shape) * self.dtype.elsize @@ -526,6 +530,7 @@ class VoidBoxStorage(BaseConcreteArray): def __init__(self, size, dtype): self.storage = alloc_raw_storage(size) + self.gcstruct = lltype.nullptr(OBJECTSTORE) self.dtype = dtype self.size = size diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -196,7 +196,7 @@ with arr as storage: self._write(storage, i, offset, self.unbox(box)) - def fill(self, storage, width, box, start, stop, offset): + def fill(self, storage, width, box, start, stop, offset, gcstruct): value = self.unbox(box) for i in xrange(start, stop, width): self._write(storage, i, offset, value) @@ -1157,7 +1157,7 @@ with arr as storage: self._write(storage, i, offset, self.unbox(box)) - def fill(self, storage, width, box, start, stop, offset): + def fill(self, storage, width, box, start, stop, offset, gcstruct): value = self.unbox(box) for i in xrange(start, stop, width): self._write(storage, i, offset, value) @@ -1677,10 +1677,10 @@ w_obj = _all_objs_for_tests[res] return w_obj - def fill(self, storage, width, box, start, stop, offset): + def fill(self, storage, width, box, start, stop, offset, gcstruct): value = self.unbox(box) for i in xrange(start, stop, width): - self._write(storage, i, offset, value, XXX) + self._write(storage, i, offset, value, gcstruct) def unbox(self, box): assert isinstance(box, self.BoxType) @@ -1851,7 +1851,7 @@ def bool(self, v): return bool(self.to_str(v)) - def fill(self, storage, width, box, start, stop, offset): + def fill(self, storage, width, box, start, stop, offset, gcstruct): for i in xrange(start, stop, width): self._store(storage, i, offset, box, width) @@ -2006,7 +2006,7 @@ for k in range(size): storage[k + i + ofs] = box_storage[k + box.ofs] - def fill(self, storage, width, box, start, stop, offset): + def fill(self, storage, width, box, start, stop, offset, gcstruct): assert isinstance(box, boxes.W_VoidBox) assert width == box.dtype.elsize for i in xrange(start, stop, width): From noreply at buildbot.pypy.org Tue Mar 31 02:56:38 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Tue, 31 Mar 2015 02:56:38 +0200 (CEST) Subject: [pypy-commit] pypy install-rpython: Add a README for rpython Message-ID: <20150331005638.406431C1D63@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: install-rpython Changeset: r76651:df86e704d421 Date: 2015-03-31 01:18 +0100 http://bitbucket.org/pypy/pypy/changeset/df86e704d421/ Log: Add a README for rpython diff --git a/README-rpython.rst b/README-rpython.rst new file mode 100644 --- /dev/null +++ b/README-rpython.rst @@ -0,0 +1,22 @@ +RPython +======= + +RPython is a translation and support framework for producing implementations of +dynamic languages, emphasizing a clean separation between language +specification and implementation aspects. + +By separating concerns in this way, our implementation of Python - and other +dynamic languages - is able to automatically generate a Just-in-Time compiler +for any dynamic language. It also allows a mix-and-match approach to +implementation decisions, including many that have historically been outside of +a user's control, such as target platform, memory and threading models, garbage +collection strategies, and optimizations applied, including whether or not to +have a JIT in the first place. + +Links +----- + +* `Documentation `_ +* `Bug tracker `_ +* `PyPy project `_ +* Mailing list: pypy-dev at python.org diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -7,8 +7,16 @@ from os import path here = path.abspath(path.dirname(__file__)) -with open(path.join(here, 'README.rst'), encoding='utf-8') as f: - long_description = f.read() # XXX +with open(path.join(here, 'README-rpython.rst'), encoding='utf-8') as f: + long_description = f.read() + +long_description += """ +Warning +------- + +This is an experimental release of a randomly chosen, untested version of +RPython. Packaging issues are likely, feedback is welcome. +""" setup( name='rpython', From noreply at buildbot.pypy.org Tue Mar 31 02:56:39 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Tue, 31 Mar 2015 02:56:39 +0200 (CEST) Subject: [pypy-commit] pypy install-rpython: Add a manifest Message-ID: <20150331005639.7209D1C1D63@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: install-rpython Changeset: r76652:9a44ae723d06 Date: 2015-03-31 01:54 +0100 http://bitbucket.org/pypy/pypy/changeset/9a44ae723d06/ Log: Add a manifest diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,9 @@ +include README-rpython.rst +exclude README.rst +prune _pytest +prune ctypes_configure +prune include +prune lib-python +prune lib_pypy +prune py +prune pypy diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -40,8 +40,6 @@ ], keywords='development', - packages=find_packages(exclude=[ - '_pytest', 'ctypes_configure', 'include', 'lib-python', 'lib-pypy', - 'py', 'pypy', 'site-packages', 'testrunner']), + packages=find_packages(), # MANIFEST.in filters out all the pypy stuff install_requires=['pytest'], ) From noreply at buildbot.pypy.org Tue Mar 31 03:01:21 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Tue, 31 Mar 2015 03:01:21 +0200 (CEST) Subject: [pypy-commit] pypy install-rpython: version bump Message-ID: <20150331010121.8B9661C1D60@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: install-rpython Changeset: r76653:1b4d5355eb02 Date: 2015-03-31 02:01 +0100 http://bitbucket.org/pypy/pypy/changeset/1b4d5355eb02/ Log: version bump diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ setup( name='rpython', - version='0.1', + version='0.1.1', description='RPython', long_description=long_description, From noreply at buildbot.pypy.org Tue Mar 31 03:33:36 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Tue, 31 Mar 2015 03:33:36 +0200 (CEST) Subject: [pypy-commit] pypy install-rpython: try again, including C files this time Message-ID: <20150331013336.5997D1C0207@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: install-rpython Changeset: r76654:cf6c8c6a12d4 Date: 2015-03-31 02:19 +0100 http://bitbucket.org/pypy/pypy/changeset/cf6c8c6a12d4/ Log: try again, including C files this time diff --git a/MANIFEST.in b/MANIFEST.in --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,6 @@ include README-rpython.rst exclude README.rst +recursive-include rpython/translator/c/src *.c *.h prune _pytest prune ctypes_configure prune include diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ setup( name='rpython', - version='0.1.1', + version='0.1.2', description='RPython', long_description=long_description, @@ -41,5 +41,6 @@ keywords='development', packages=find_packages(), # MANIFEST.in filters out all the pypy stuff + package_data={'rpython': ['translator/c/src/*.c', 'translator/c/src/*.h']}, install_requires=['pytest'], ) From noreply at buildbot.pypy.org Tue Mar 31 09:43:32 2015 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 31 Mar 2015 09:43:32 +0200 (CEST) Subject: [pypy-commit] pypy default: simplify the code and kill @specialize.memo function that makes no sense any Message-ID: <20150331074332.0EF841C04A7@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r76655:40afa1747889 Date: 2015-03-31 09:43 +0200 http://bitbucket.org/pypy/pypy/changeset/40afa1747889/ Log: simplify the code and kill @specialize.memo function that makes no sense any more diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -24,13 +24,9 @@ self.end = end self.reason = reason - at specialize.memo() -def rpy_encode_error_handler(): - # A RPython version of the "strict" error handler. - def raise_unicode_exception_encode(errors, encoding, msg, u, - startingpos, endingpos): - raise RUnicodeEncodeError(encoding, u, startingpos, endingpos, msg) - return raise_unicode_exception_encode +def raise_unicode_exception_encode(errors, encoding, msg, u, + startingpos, endingpos): + raise RUnicodeEncodeError(encoding, u, startingpos, endingpos, msg) # ____________________________________________________________ @@ -67,5 +63,5 @@ # This is not the case with Python3. return runicode.unicode_encode_utf_8( uni, len(uni), "strict", - errorhandler=rpy_encode_error_handler(), + errorhandler=raise_unicode_exception_encode, allow_surrogates=True) diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -439,12 +439,12 @@ try: if encoding == 'ascii': u = space.unicode_w(w_object) - eh = unicodehelper.rpy_encode_error_handler() + eh = unicodehelper.raise_unicode_exception_encode return space.wrap(unicode_encode_ascii( u, len(u), None, errorhandler=eh)) if encoding == 'utf-8': u = space.unicode_w(w_object) - eh = unicodehelper.rpy_encode_error_handler() + eh = unicodehelper.raise_unicode_exception_encode return space.wrap(unicode_encode_utf_8( u, len(u), None, errorhandler=eh, allow_surrogates=True)) From noreply at buildbot.pypy.org Tue Mar 31 11:03:12 2015 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 31 Mar 2015 11:03:12 +0200 (CEST) Subject: [pypy-commit] pypy default: this is hopeless - more specialization Message-ID: <20150331090312.126AC1C11BF@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r76656:a6a90d0533db Date: 2015-03-31 11:02 +0200 http://bitbucket.org/pypy/pypy/changeset/a6a90d0533db/ Log: this is hopeless - more specialization diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py --- a/rpython/rlib/runicode.py +++ b/rpython/rlib/runicode.py @@ -4,6 +4,7 @@ from rpython.rlib.rarithmetic import r_uint, intmask from rpython.rlib.unicodedata import unicodedb from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rlib import jit if rffi.sizeof(lltype.UniChar) == 4: @@ -1053,10 +1054,12 @@ return result.build() + at specialize.arg_or_var(3) def unicode_encode_latin_1(p, size, errors, errorhandler=None): res = unicode_encode_ucs1_helper(p, size, errors, errorhandler, 256) return res + at specialize.arg_or_var(3) def unicode_encode_ascii(p, size, errors, errorhandler=None): res = unicode_encode_ucs1_helper(p, size, errors, errorhandler, 128) return res From noreply at buildbot.pypy.org Tue Mar 31 13:03:35 2015 From: noreply at buildbot.pypy.org (hrc706) Date: Tue, 31 Mar 2015 13:03:35 +0200 (CEST) Subject: [pypy-commit] pypy default: bug fix for test_protocol_tlsv1_1 fail Message-ID: <20150331110335.CF5AD1C11E0@cobra.cs.uni-duesseldorf.de> Author: Ruochen Huang Branch: Changeset: r76657:bc3c28340cf6 Date: 2015-03-31 20:00 +0900 http://bitbucket.org/pypy/pypy/changeset/bc3c28340cf6/ Log: bug fix for test_protocol_tlsv1_1 fail diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -75,7 +75,9 @@ constants["PROTOCOL_TLSv1"] = PY_SSL_VERSION_TLS1 if HAVE_TLSv1_2: constants["PROTOCOL_TLSv1_1"] = PY_SSL_VERSION_TLS1_1 + constants["OP_NO_TLSv1_1"] = SSL_OP_NO_TLSv1_1 constants["PROTOCOL_TLSv1_2"] = PY_SSL_VERSION_TLS1_2 + constants["OP_NO_TLSv1_2"] = SSL_OP_NO_TLSv1_2 constants["OP_ALL"] = SSL_OP_ALL &~SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS constants["OP_NO_SSLv2"] = SSL_OP_NO_SSLv2 diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py --- a/rpython/rlib/ropenssl.py +++ b/rpython/rlib/ropenssl.py @@ -93,6 +93,9 @@ SSL_OP_NO_SSLv2 = rffi_platform.ConstantInteger("SSL_OP_NO_SSLv2") SSL_OP_NO_SSLv3 = rffi_platform.ConstantInteger("SSL_OP_NO_SSLv3") SSL_OP_NO_TLSv1 = rffi_platform.ConstantInteger("SSL_OP_NO_TLSv1") + if HAVE_TLSv1_2: + SSL_OP_NO_TLSv1_1 = rffi_platform.ConstantInteger("SSL_OP_NO_TLSv1_1") + SSL_OP_NO_TLSv1_2 = rffi_platform.ConstantInteger("SSL_OP_NO_TLSv1_2") SSL_OP_CIPHER_SERVER_PREFERENCE = rffi_platform.ConstantInteger( "SSL_OP_CIPHER_SERVER_PREFERENCE") SSL_OP_SINGLE_DH_USE = rffi_platform.ConstantInteger( From noreply at buildbot.pypy.org Tue Mar 31 13:09:01 2015 From: noreply at buildbot.pypy.org (hrc706) Date: Tue, 31 Mar 2015 13:09:01 +0200 (CEST) Subject: [pypy-commit] pypy default: a stupid false, fixed it Message-ID: <20150331110901.784F11C04A7@cobra.cs.uni-duesseldorf.de> Author: Ruochen Huang Branch: Changeset: r76658:07d8f21f1d7f Date: 2015-03-31 20:09 +0900 http://bitbucket.org/pypy/pypy/changeset/07d8f21f1d7f/ Log: a stupid false, fixed it diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py --- a/rpython/rlib/ropenssl.py +++ b/rpython/rlib/ropenssl.py @@ -77,6 +77,7 @@ else: ASN1_ITEM_EXP = ASN1_ITEM OPENSSL_VERSION_NUMBER = cconfig["OPENSSL_VERSION_NUMBER"] +HAVE_TLSv1_2 = OPENSSL_VERSION_NUMBER >= 0x10001000 class CConfig: _compilation_info_ = eci @@ -258,7 +259,6 @@ OPENSSL_VERSION_NUMBER != 0x00909000 if OPENSSL_VERSION_NUMBER < 0x0090800f and not OPENSSL_NO_ECDH: OPENSSL_NO_ECDH = True -HAVE_TLSv1_2 = OPENSSL_VERSION_NUMBER >= 0x10001000 def external(name, argtypes, restype, **kw): From noreply at buildbot.pypy.org Tue Mar 31 13:10:14 2015 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 31 Mar 2015 13:10:14 +0200 (CEST) Subject: [pypy-commit] pypy vmprof: cache some writes on the code object level Message-ID: <20150331111014.31CF91C1016@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76659:f4ad7a7e6e7a Date: 2015-03-31 13:10 +0200 http://bitbucket.org/pypy/pypy/changeset/f4ad7a7e6e7a/ Log: cache some writes on the code object level diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py --- a/pypy/module/_vmprof/interp_vmprof.py +++ b/pypy/module/_vmprof/interp_vmprof.py @@ -124,12 +124,14 @@ def try_cast_to_pycode(gcref): return rgc.try_cast_gcref_to_instance(PyCode, gcref) +MAX_CODES = 1000 + class VMProf(object): def __init__(self): self.is_enabled = False self.ever_enabled = False - self.mapping_so_far = [] # stored mapping in between runs self.fileno = -1 + self.current_codes = [] def enable(self, space, fileno, period): if self.is_enabled: @@ -175,20 +177,29 @@ if self.fileno == -1: raise OperationError(space.w_RuntimeError, space.wrap("vmprof not running")) - name = code._get_full_name() + if len(self.current_codes) < MAX_CODES: + self.current_codes.append(code) + else: + self._flush_codes(space) + + def _fluch_codes(self, space): b = StringBuilder() - b.append('\x02') - write_long_to_string_builder(code._unique_id, b) - write_long_to_string_builder(len(name), b) - b.append(name) + for code in self.current_codes: + name = code._get_full_name() + b.append('\x02') + write_long_to_string_builder(code._unique_id, b) + write_long_to_string_builder(len(name), b) + b.append(name) os.write(self.fileno, b.build()) + self.current_codes = [] def disable(self, space): if not self.is_enabled: raise oefmt(space.w_ValueError, "_vmprof not enabled") self.is_enabled = False + space.register_code_callback(None) + self._fluch_codes(space) self.fileno = -1 - space.register_code_callback(None) if we_are_translated(): # does not work untranslated res = vmprof_disable() From noreply at buildbot.pypy.org Tue Mar 31 13:20:40 2015 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 31 Mar 2015 13:20:40 +0200 (CEST) Subject: [pypy-commit] pypy vmprof: typo Message-ID: <20150331112040.8E78E1C11E0@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76660:e885d85c010a Date: 2015-03-31 13:20 +0200 http://bitbucket.org/pypy/pypy/changeset/e885d85c010a/ Log: typo diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py --- a/pypy/module/_vmprof/interp_vmprof.py +++ b/pypy/module/_vmprof/interp_vmprof.py @@ -182,7 +182,7 @@ else: self._flush_codes(space) - def _fluch_codes(self, space): + def _flush_codes(self, space): b = StringBuilder() for code in self.current_codes: name = code._get_full_name() From noreply at buildbot.pypy.org Tue Mar 31 13:33:38 2015 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 31 Mar 2015 13:33:38 +0200 (CEST) Subject: [pypy-commit] pypy vmprof: so many typos Message-ID: <20150331113338.EF5841C156B@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76661:f15aca12980f Date: 2015-03-31 13:33 +0200 http://bitbucket.org/pypy/pypy/changeset/f15aca12980f/ Log: so many typos diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py --- a/pypy/module/_vmprof/interp_vmprof.py +++ b/pypy/module/_vmprof/interp_vmprof.py @@ -198,7 +198,7 @@ raise oefmt(space.w_ValueError, "_vmprof not enabled") self.is_enabled = False space.register_code_callback(None) - self._fluch_codes(space) + self._flush_codes(space) self.fileno = -1 if we_are_translated(): # does not work untranslated From noreply at buildbot.pypy.org Tue Mar 31 16:52:16 2015 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 31 Mar 2015 16:52:16 +0200 (CEST) Subject: [pypy-commit] pypy vmprof: automatically gzip the output here Message-ID: <20150331145216.46DA11C1016@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76662:1e07ae277422 Date: 2015-03-31 16:52 +0200 http://bitbucket.org/pypy/pypy/changeset/1e07ae277422/ Log: automatically gzip the output here diff --git a/rpython/bin/rpython-vmprof b/rpython/bin/rpython-vmprof --- a/rpython/bin/rpython-vmprof +++ b/rpython/bin/rpython-vmprof @@ -17,11 +17,12 @@ print __doc__ sys.exit(1) -import _vmprof -x = open("vmprof.log", "w") -_vmprof.enable(x.fileno(), 1000) +import _vmprof, subprocess +x = subprocess.Popen('gzip > vmprof.log.gz', shell=True, stdin=subprocess.PIPE) +_vmprof.enable(x.stdin.fileno(), 1000) try: main() finally: _vmprof.disable() - x.close() + x.stdin.close() + x.wait() From noreply at buildbot.pypy.org Tue Mar 31 17:57:16 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 31 Mar 2015 17:57:16 +0200 (CEST) Subject: [pypy-commit] pypy default: A bug! Message-ID: <20150331155716.17BC31C0318@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76663:3df0542cbfa7 Date: 2015-03-31 17:57 +0200 http://bitbucket.org/pypy/pypy/changeset/3df0542cbfa7/ Log: A bug! diff --git a/rpython/rtyper/test/test_rpbc.py b/rpython/rtyper/test/test_rpbc.py --- a/rpython/rtyper/test/test_rpbc.py +++ b/rpython/rtyper/test/test_rpbc.py @@ -1656,6 +1656,23 @@ res = self.interpret(g, [2]) assert self.ll_to_string(res) == "ASub" + def test_bug_callfamily(self): + def cb1(): + pass + def cb2(): + pass + def g(cb): + pass + def h(cb): + cb() + def f(): + g(cb1) + g(cb2) + h(cb2) + return 42 + res = self.interpret(f, []) + assert res == 42 + # ____________________________________________________________ class TestRPBCExtra(BaseRtypingTest): From noreply at buildbot.pypy.org Tue Mar 31 18:15:12 2015 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 31 Mar 2015 18:15:12 +0200 (CEST) Subject: [pypy-commit] pypy vmprof: write header Message-ID: <20150331161512.EC1DA1C04A7@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76664:28eaabd5b1bb Date: 2015-03-31 18:15 +0200 http://bitbucket.org/pypy/pypy/changeset/28eaabd5b1bb/ Log: write header diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py --- a/pypy/module/_vmprof/interp_vmprof.py +++ b/pypy/module/_vmprof/interp_vmprof.py @@ -171,6 +171,9 @@ write_long_to_string_builder(0, b) write_long_to_string_builder(period_usec, b) write_long_to_string_builder(0, b) + b.append('\x04') # interp name + b.append(chr(len('pypy'))) + b.append('pypy') os.write(fileno, b.build()) def register_code(self, space, code): From noreply at buildbot.pypy.org Tue Mar 31 18:43:25 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 31 Mar 2015 18:43:25 +0200 (CEST) Subject: [pypy-commit] pypy default: Test and fix for a very rare case Message-ID: <20150331164325.2ADCD1C02AD@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76665:dac9223bca11 Date: 2015-03-31 18:43 +0200 http://bitbucket.org/pypy/pypy/changeset/dac9223bca11/ Log: Test and fix for a very rare case diff --git a/rpython/rtyper/rpbc.py b/rpython/rtyper/rpbc.py --- a/rpython/rtyper/rpbc.py +++ b/rpython/rtyper/rpbc.py @@ -6,7 +6,7 @@ from rpython.rtyper import rclass, callparse from rpython.rtyper.rclass import CLASSTYPE, OBJECT_VTABLE, OBJECTPTR from rpython.rtyper.error import TyperError -from rpython.rtyper.lltypesystem import lltype +from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rtyper.rmodel import (Repr, inputconst, CanBeNull, mangle, warning, impossible_repr) from rpython.tool.pairtype import pair, pairtype @@ -205,11 +205,15 @@ # this row llfn = self.rtyper.type_system.null_callable(row.fntype) llfns[row.attrname] = llfn - if not found_anything: - raise TyperError("%r not in %r" % (funcdesc, - self.s_pbc.descriptions)) if len(self.uniquerows) == 1: - result = llfn # from the loop above + if found_anything: + result = llfn # from the loop above + else: + # extremely rare case, shown only sometimes by + # test_bug_callfamily: don't emit NULL, because that + # would be interpreted as equal to None... It should + # never be called anyway. + result = rffi.cast(self.lowleveltype, ~len(self.funccache)) else: # build a Struct with all the values collected in 'llfns' result = self.create_specfunc() diff --git a/rpython/rtyper/test/test_rpbc.py b/rpython/rtyper/test/test_rpbc.py --- a/rpython/rtyper/test/test_rpbc.py +++ b/rpython/rtyper/test/test_rpbc.py @@ -1658,16 +1658,17 @@ def test_bug_callfamily(self): def cb1(): - pass + xxx # never actually called def cb2(): pass - def g(cb): - pass + def g(cb, result): + assert (cb is None) == (result == 0) def h(cb): cb() def f(): - g(cb1) - g(cb2) + g(None, 0) + g(cb1, 1) + g(cb2, 2) h(cb2) return 42 res = self.interpret(f, []) From noreply at buildbot.pypy.org Tue Mar 31 22:42:19 2015 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 31 Mar 2015 22:42:19 +0200 (CEST) Subject: [pypy-commit] pypy vmprof: I'm getting so confused, count every FOUR so I know where we are Message-ID: <20150331204219.1A9ED1C0318@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76666:32000f6d787e Date: 2015-03-31 22:42 +0200 http://bitbucket.org/pypy/pypy/changeset/32000f6d787e/ Log: I'm getting so confused, count every FOUR so I know where we are diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -129,7 +129,7 @@ ec = self.space.getexecutioncontext() self._unique_id = ec._code_unique_id - ec._code_unique_id += 2 # so we have one bit that we can mark stuff + ec._code_unique_id += 4 # so we have two bits that we can mark stuff # with def _get_full_name(self): diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py --- a/pypy/module/_vmprof/interp_vmprof.py +++ b/pypy/module/_vmprof/interp_vmprof.py @@ -96,7 +96,8 @@ gc_frame = cast_instance_to_gcref(frame) gc_inputvalue = cast_instance_to_gcref(w_inputvalue) gc_operr = cast_instance_to_gcref(operr) - unique_id = frame.pycode._unique_id - 1 + assert frame.pycode._unique_id & 3 == 0 + unique_id = frame.pycode._unique_id | 1 gc_result = pypy_execute_frame_trampoline(gc_frame, gc_inputvalue, gc_operr, unique_id) return cast_base_ptr_to_instance(W_Root, gc_result)