From ale at codespeak.net Tue May 1 08:55:46 2007 From: ale at codespeak.net (ale at codespeak.net) Date: Tue, 1 May 2007 08:55:46 +0200 (CEST) Subject: [pypy-svn] r42512 - pypy/dist/pypy/lib/pyontology/test Message-ID: <20070501065546.5EAA18076@code0.codespeak.net> Author: ale Date: Tue May 1 08:55:44 2007 New Revision: 42512 Modified: pypy/dist/pypy/lib/pyontology/test/test_sparql.py Log: Changed and added test for SPARQL. Modified: pypy/dist/pypy/lib/pyontology/test/test_sparql.py ============================================================================== --- pypy/dist/pypy/lib/pyontology/test/test_sparql.py (original) +++ pypy/dist/pypy/lib/pyontology/test/test_sparql.py Tue May 1 08:55:44 2007 @@ -69,7 +69,7 @@ assert len(res[0]) == len(res[1]) == 4 assert res[0][1] in ['http://example.org/ns#p', 'http://example.org/ns#q'] assert res[1][1] in ['http://example.org/ns#p', 'http://example.org/ns#q'] - assert result[3][0] == 'x<2' + assert result[3].formula == 'query_x<2' # There are 8 ways of having the triples in the query, if predicate is not a builtin owl predicate # @@ -113,6 +113,7 @@ O.attach_fd() O.finish() res = O.sparql(query) + assert len(res) == 1 assert res[0]['x'] == u'http://example.org/ns#sub' def test_case_2(): @@ -166,7 +167,6 @@ res = O.sparql(query) assert res[0]['x'] == u'http://example.org/ns#sub' assert res[0]['y'] == 123 - assert res[0]['x'] == u'http://example.org/ns#sub' def test_case_6(): """ return the values of p """ @@ -202,7 +202,88 @@ O.finish() res = O.sparql(query) assert res[0]['y'] == 123 + +query_x = """ + PREFIX ltw : + PREFIX owl : + PREFIX rdf : + SELECT ?x + WHERE { + ?x rdf:type ltw:Active_FundingOrganisation . + ?x ltw:organisationNameAbbreviation 'BMBF' . + }""" + +def test_queryx(): + O = Ontology() + O.add_file(datapath("testont2.rdf"), 'xml') + O.attach_fd() + + res = O.sparql(query_x) + assert len(res) == 1 + assert res[0]['x'] == u'http://www.lt-world.org/ltw.owl#obj_61128' + +query_y = """ + PREFIX ltw : + PREFIX owl : + PREFIX rdf : + PREFIX xsd: + SELECT ?project + WHERE { + ?project ltw:dateStart ?date_begin . + ?project ltw:dateEnd ?date_end . + FILTER ( ?date_begin < "2007-01-01T00:00:00Z"^^xsd:dateTime + && ?date_end >= "2006-01-01T00:00:00Z"^^xsd:dateTime) . + }""" + + +def test_queryy(): + py.test.skip("Needs special care for dateTime type") + O = Ontology() + O.add_file(datapath("testont2.rdf"), 'xml') + O.attach_fd() + + res = O.sparql(query_y) + assert len(res) == 1 + assert res[0]['project'] == u'http://www.lt-world.org/ltw.owl#obj_61128' + + +query1_a = """ + PREFIX ltw : + PREFIX owl : + PREFIX rdf : + SELECT ?person ?activity + WHERE { + ?activity rdf:type ltw:Active_Project . + ?person rdf:type ltw:Active_Person . + } + ORDER BY ?person""" +query1_b = """ + PREFIX ltw : + PREFIX owl : + PREFIX rdf : + SELECT ?person ?activity + WHERE { + ?activity ltw:hasParticipant ?person_obj . + ?person_obj ltw:personName ?person . + } + ORDER BY ?person""" +def test_query1_a(): + O = Ontology() + O.add_file(datapath("testont2.rdf"), 'xml') + O.attach_fd() + + res = O.sparql(query1_a) + assert len(res) == 1 + +def test_query1_b(): + O = Ontology() + O.add_file(datapath("testont2.rdf"), 'xml') + O.attach_fd() + + res = O.sparql(query1_b) + assert len(res) == 1 + query1 = """ PREFIX ltw : PREFIX owl : @@ -219,19 +300,22 @@ query2 = """ PREFIX ltw : PREFIX owl : - SELECT ?project ?date_begin + PREFIX rdf : + SELECT ?project ?date_begin ?x WHERE { - ?project ltw:supportedBy ltw:BMBF . + ?project ltw:supportedBy ?x . + ?x rdf:type ltw:Active_FundingOrganisation . + ?x ltw:organisationNameAbbreviation 'BMBF' . ?project ltw:dateStart ?date_begin . ?project ltw:dateEnd ?date_end . FILTER ( ?date_begin < 2007 && ?date_end >= 2006) . }""" #which project is funded in a technological area (i.e. Semantic web), -query3 = """ +query_3 = """ PREFIX ltw : PREFIX owl : PREFIX rdf : - SELECT ?project + SELECT ?project ?x ?y WHERE { ?project rdf:type ltw:Active_Project . ?project ltw:lt_technologicalMethod ?y . @@ -239,7 +323,47 @@ ?project ltw:supportedBy ?x . }""" -def test_query1(): +query3a = """ + PREFIX ltw : + PREFIX owl : + PREFIX rdf : + SELECT ?project ?x ?y + WHERE { + ?project rdf:type ltw:Active_Project . + ?project ltw:lt_technologicalMethod ?y . + }""" + +query3b = """ + PREFIX ltw : + PREFIX owl : + PREFIX rdf : + SELECT ?project ?x ?y + WHERE { + ?y rdf:type ltw:Semantic_Web . + ?project ltw:supportedBy ?x . + }""" + +def test_query3a(): + py.test.skip("WIP") + O = Ontology() + O.add_file(datapath("testont2.rdf"), 'xml') + O.attach_fd() + + res = O.sparql(query3a) + assert len(res) == 1 + assert res[0]['project'] == u'http://www.lt-world.org/ltw.owl#obj_59773' + +def test_query3b(): + py.test.skip("WIP") + O = Ontology() + O.add_file(datapath("testont2.rdf"), 'xml') + O.attach_fd() + + res = O.sparql(query3a) + assert len(res) == 1 + assert res[0]['project'] == u'http://www.lt-world.org/ltw.owl#obj_59773' + +def test_query_1(): O = Ontology() O.add_file(datapath("testont2.rdf"), 'xml') O.attach_fd() @@ -249,27 +373,26 @@ assert res[0]['activity'] == u'http://www.lt-world.org/ltw.owl#obj_59754' assert res[0]['person'] == u'\nKlara Vicsi' -def test_query2(): -# py.test.skip("Doesn't work yet") +def test_query_2(): + py.test.skip("Needs special care for dateTime type") O = Ontology() O.add_file(datapath("testont2.rdf"), 'xml') O.attach_fd() res = O.sparql(query2) - assert len(res) == 2 + assert len(res) == 1 assert res[0]['project'] == u'http://www.lt-world.org/ltw.owl#obj_59754' assert res[0]['date_begin'] == datetime.date(1998,9,1) -def test_query3(): - #py.test.skip("Doesn't work yet") +def test_query_3(): + py.test.skip("WIP") O = Ontology() O.add_file(datapath("testont2.rdf"), 'xml') O.attach_fd() - res = O.sparql(query3) + res = O.sparql(query_3) assert len(res) == 1 - assert res[0]['activity'] == u'http://www.lt-world.org/ltw.owl#obj_59754' - assert res[0]['person'] == u'\nKlara Vicsi' + assert res[0]['project'] == u'http://www.lt-world.org/ltw.owl#obj_59773' import xmlrpclib, socket, os, signal From ale at codespeak.net Tue May 1 08:57:34 2007 From: ale at codespeak.net (ale at codespeak.net) Date: Tue, 1 May 2007 08:57:34 +0200 (CEST) Subject: [pypy-svn] r42513 - pypy/dist/pypy/lib/pyontology/test Message-ID: <20070501065734.2E1958076@code0.codespeak.net> Author: ale Date: Tue May 1 08:57:33 2007 New Revision: 42513 Modified: pypy/dist/pypy/lib/pyontology/test/testont2.rdf Log: Added types needed for the tests. Added a section for 'Semantci Web' Modified: pypy/dist/pypy/lib/pyontology/test/testont2.rdf ============================================================================== --- pypy/dist/pypy/lib/pyontology/test/testont2.rdf (original) +++ pypy/dist/pypy/lib/pyontology/test/testont2.rdf Tue May 1 08:57:33 2007 @@ -1,5 +1,33 @@ + + + +Semantic Web + +Semantic Web + +Semantic Web + +The Semantic Web is a W3C-based initiative for representing knowledge on the World Wide Web in a machine-readable fashion, such that it can be understood and used by machines for intelligent applications. + + + +EUROPA!Europaeische Union!European Comission!Eurpean Commission!Eurpean Comission!European Commission - Directorate General + +EC + +EC + +European Commission + +European Commission + + + +Federal Ministry of Education and Research +BMBF + @@ -8,7 +36,8 @@ - + + From ale at codespeak.net Tue May 1 09:01:05 2007 From: ale at codespeak.net (ale at codespeak.net) Date: Tue, 1 May 2007 09:01:05 +0200 (CEST) Subject: [pypy-svn] r42514 - pypy/dist/pypy/lib/pyontology Message-ID: <20070501070105.BC1CA8076@code0.codespeak.net> Author: ale Date: Tue May 1 09:01:05 2007 New Revision: 42514 Modified: pypy/dist/pypy/lib/pyontology/sparql_grammar.py Log: Removed extra quoting of strings. Support for 'and' and 'or'. Construction of constraints from FILTER expression Modified: pypy/dist/pypy/lib/pyontology/sparql_grammar.py ============================================================================== --- pypy/dist/pypy/lib/pyontology/sparql_grammar.py (original) +++ pypy/dist/pypy/lib/pyontology/sparql_grammar.py Tue May 1 09:01:05 2007 @@ -28,11 +28,28 @@ def replace_string(s, loc, toks): - return [rdfliteral((toks[0]), datatype="http://www.w3.org/2001/XMLSchema#String")] + return [rdfliteral((toks[0][1:-1]), datatype="http://www.w3.org/2001/XMLSchema#String")] def replace_float(s, loc, toks): return [rdfliteral(float(toks[0]), datatype="http://www.w3.org/2001/XMLSchema#float")] +def replace_RDFLiteral(s, loc, toks): + + if toks.IRIref: + lit = toks[0] + lit.datatype = toks.IRIref + ret = [lit] + else: + ret = [rdfliteral(toks[0], datatype="http://www.w3.org/2001/XMLSchema#String")] + return ret + +def construct_constraint(s, loc, toks): + varlist = [] + for x in toks.Var: + varlist.append(x[0]) + varlist.append(toks[0]) + return [varlist] + class SPARQLGrammar(object): # All productions are declared Forward(). This is primarily @@ -75,8 +92,8 @@ lne = punctuation('ne') bnode = punctuation('_:') comma = punctuation(',') - lor = punctuation('||') - land = punctuation('&&') + lor = punctuation('||').setParseAction(lambda x,y,z: [' or ']) + land = punctuation('&&').setParseAction(lambda x,y,z: [' and ']) # keywords @@ -138,7 +155,7 @@ OptionalGraphPattern = production('OptionalGraphPattern') GraphGraphPattern = production('GraphGraphPattern') GroupOrUnionGraphPattern = production('GroupOrUnionGraphPattern') - Constraint = production('Constraint') + Constraint = production('Constraint').setParseAction(construct_constraint) ConstructTemplate = production('ConstructTemplate') Triples = production('Triples') Triples1 = production('Triples1') @@ -174,7 +191,7 @@ PrimaryExpression = production('PrimaryExpression') RDFTerm = production('RDFTerm') NumericLiteral = production('NumericLiteral') - RDFLiteral = production('RDFLiteral') + RDFLiteral = production('RDFLiteral').setParseAction(replace_RDFLiteral) BooleanLiteral = production('BooleanLiteral') String = production('String').setParseAction(replace_string) IRIref = production('IRIref') From ale at codespeak.net Tue May 1 09:03:54 2007 From: ale at codespeak.net (ale at codespeak.net) Date: Tue, 1 May 2007 09:03:54 +0200 (CEST) Subject: [pypy-svn] r42515 - pypy/dist/pypy/lib/pyontology Message-ID: <20070501070354.AE2B98076@code0.codespeak.net> Author: ale Date: Tue May 1 09:03:54 2007 New Revision: 42515 Modified: pypy/dist/pypy/lib/pyontology/constraint_classes.py Log: Corrected the constraints working on restrictions. Changed the solver mechanics to be able to solve the problem with respect to only some of the variables. Modified: pypy/dist/pypy/lib/pyontology/constraint_classes.py ============================================================================== --- pypy/dist/pypy/lib/pyontology/constraint_classes.py (original) +++ pypy/dist/pypy/lib/pyontology/constraint_classes.py Tue May 1 09:03:54 2007 @@ -31,6 +31,10 @@ print repository try: foundSolution = repository.consistency(verbose) +# This ties the repository to a special distributor - not nice + if not foundSolution: + dom = repository.getDomains() + foundSolution = [x for x in self._distributor.vars if dom[x].size() == 1] except ConsistencyFailure, exc: if verbose: print strftime('%H:%M:%S'), exc @@ -88,10 +92,16 @@ class MyDistributor(SplitDistributor): - def __init__(self): + def __init__(self, vars_to_distribute): SplitDistributor.__init__(self,2) + self.vars = vars_to_distribute self.to_split = None + def findSmallestDomain(self, domains): + dom_sizes = [(domains[v].size(),v) for v in self.vars if domains[v].size()>1] + dom_sizes.sort() + return dom_sizes[0][1] + def nb_subdomains(self, domains): """See AbstractDistributor""" self.to_split = self.findSmallestDomain(domains) @@ -258,7 +268,7 @@ class PropertyConstrain3(AbstractConstraint): cost = 1 def __init__(self, prop, variable, cls_or_restriction): - AbstractConstraint.__init__(self, [ prop]) + AbstractConstraint.__init__(self, [variable]) self.object = cls_or_restriction self.variable = variable self.prop = prop @@ -266,23 +276,24 @@ def narrow(self, domains): # Narrow the domains of object and variable to those values # that are connected by self.prop +# import pdb +# pdb.set_trace() dom = domains[self.prop] sub = domains[self.variable] obj = domains[self.object] - vals_dict = dom._dict + sub_= [x[0] for x in dom.getValues()] + obj_= [x[1] for x in dom.getValues()] - keep = set() sub_rem = [] + obj_rem = [] for v in sub.getValues(): - if not v in vals_dict: + if not v in sub_: sub_rem.append(v) - #sub.removeValue(v) - else: - for o in dom.getValuesPrKey(v): - keep.add(o) - remove = [x for x in obj.getValues() if not x in keep] + for v in obj.getValues(): + if not v in obj_: + obj_rem.append(v) sub.removeValues(sub_rem) - obj.removeValues(remove) + obj.removeValues(obj_rem) class MemberConstraint(AbstractConstraint): cost = 1 @@ -585,14 +596,25 @@ assert len(inter) > 0 cls = domains[self.variable].setValues(inter) -class SomeValueConstraint(OneofPropertyConstraint): +class SomeValueConstraint(AbstractConstraint): cost = 100 - + + def __init__(self, variable, property, list_of_vals): + AbstractConstraint.__init__(self, [variable ]) + self.variable = variable + self.property = property + self.List = list_of_vals + cost = 100 + + def estimateCost(self, domains): + return self.cost + + def narrow(self, domains): val = set(domains[self.List].getValues()) dom = domains[self.variable] - property = dom.property + property = self.property indi = dom.getValues() prop = Linkeddict(list(domains[property].getValues())) for v in indi: @@ -606,7 +628,7 @@ else: dom.removeValue(v) -class AllValueConstraint(OneofPropertyConstraint): +class AllValueConstraint(SomeValueConstraint): """ AllValuesfrom property restriction is used to define the class of individuals for which the values for the property (defined by the onProperty triple) all comes from the class description @@ -621,7 +643,7 @@ if not val: return dom = domains[self.variable] - property = dom.property + property = self.property indi = dom.getValues() prop = domains[property]._dict remove = [] @@ -635,14 +657,14 @@ remove.append(v) dom.removeValues(remove) -class HasvalueConstraint(OneofPropertyConstraint): +class HasvalueConstraint(SomeValueConstraint): cost = 100 def narrow(self, domains): val = self.List dom = domains[self.variable] - property = dom.property + property = self.property indi = dom.getValues() prop = Linkeddict(domains[property].getValues()) remove = [] From ale at codespeak.net Tue May 1 09:05:39 2007 From: ale at codespeak.net (ale at codespeak.net) Date: Tue, 1 May 2007 09:05:39 +0200 (CEST) Subject: [pypy-svn] r42516 - pypy/dist/pypy/lib/pyontology Message-ID: <20070501070539.27C858076@code0.codespeak.net> Author: ale Date: Tue May 1 09:05:38 2007 New Revision: 42516 Modified: pypy/dist/pypy/lib/pyontology/pyontology.py Log: Remember to add Literal to the domains. Lots of fixes and tweaks in SPARQL implementation Modified: pypy/dist/pypy/lib/pyontology/pyontology.py ============================================================================== --- pypy/dist/pypy/lib/pyontology/pyontology.py (original) +++ pypy/dist/pypy/lib/pyontology/pyontology.py Tue May 1 09:05:38 2007 @@ -145,7 +145,7 @@ variables[self.name].setValues([v[0] for v in prop.getValues()]) elif ('owl_Thing' in variables.keys() and isinstance(self, ClassDomain) and self.size() == 0): - variables[self.name].setValues(list(variables['owl_Thing'].getValues())) + variables[self.name].setValues(list(variables['owl_Thing'].getValues())+list(variables['owl_Literal'])) variables.update(self.domains) glob_constraints.extend(self.in_constraint) # assert self.size() != 0 @@ -533,6 +533,9 @@ continue self.variables[key].finish(self.variables, self.constraints) else: +# In case of an inconsistent ontology remove the comments to get more +# than one error + # try: constraint.narrow(self.variables) # except ConsistencyFailure, e: @@ -553,7 +556,7 @@ triples = where.GroupGraphPattern[0].Triples new = [] - vars = [] + vars = set() for trip in triples: case = 0 inc = 1 @@ -572,7 +575,7 @@ elif item.VAR1: var_uri = URIRef('query_'+item.VAR1[0][0]) newtrip.append(var_uri) - vars.append(var_uri) + vars.add(var_uri) case += trip_.index(item) + inc if inc == 2: inc = 1 @@ -583,6 +586,12 @@ newtrip.append(case) new.append(newtrip) constrain = where.GroupGraphPattern[0].Constraint + if constrain: + constrain = constrain[0] + expr = constrain.pop(-1) + for x in constrain: + expr = expr.replace(x, URIRef('query_'+x)) + constrain = Expression([URIRef('query_'+x) for x in constrain], expr) return new, prefixes, resvars, constrain, vars # There are 8 ways of having the triples in the query, if predicate is not a builtin owl predicate @@ -615,12 +624,22 @@ elif case == 1: # Add a HasValue constraint ns,pred = trip[1].split("#") - if ns in namespaces.values(): + if 0: #ns in namespaces.values(): + import pdb + pdb.set_trace() self.consider_triple(trip) else: var = self.make_var(Restriction, URIRef(trip[0])) - self.onProperty(var, URIRef(trip[1])) - self.hasValue(var, trip[2]) + prop = URIRef(trip[1]) + prop_name = self.mangle_name(prop) + self.onProperty(var, prop) + query_dom[prop_name] = self.variables[prop_name] + if trip[2] in self.variables['owl_Literal'] : + val = trip[2] + else: + val = self.variables[self.mangle_name(trip[2])] + query_constr.append(HasvalueConstraint(var, prop_name, val)) + #self.hasValue(var, trip[2]) elif case == 2: # for all p's return p if p[0]==s and p[1]==o @@ -672,18 +691,9 @@ prop = self.make_var(Property, URIRef(trip[1])) query_dom[prop] = self.variables[prop] p_vals = list(self.variables[prop].getValues()) - sub = self.make_var(Thing, trip[0]) - vals = set([v[0] for v in p_vals]) - if self.variables[sub].size(): - vals &= set(self.variables[sub].getValues()) - self.variables[sub].setValues(vals) - obj = self.make_var(Thing, trip[2]) - vals = set([v[1] for v in p_vals]) - if self.variables[obj].size(): - vals &= set(self.variables[obj].getValues()) - self.variables[obj].setValues(vals) + sub = self.make_var(ClassDomain, trip[0]) + obj = self.make_var(ClassDomain, trip[2]) con = PropertyConstrain3(prop, sub, obj) -# con = Expression([sub,prop,obj], "%s == (%s, %s)" %(prop, sub, obj)) query_constr.append(con) elif case == 6: @@ -709,13 +719,15 @@ _dom, _ = self.variables[self.mangle_name(v)].finish(self.variables, query_constr) #query_dom, query_constr) query_dom.update(_dom) # Build a repository with the variables in the query - dom = dict([(self.mangle_name(v),self.variables[self.mangle_name(v)]) - for v in vars]) - + solve_vars = [self.mangle_name(v) for v in vars] + dom = dict([(v, self.variables[v]) + for v in solve_vars]) dom.update(query_dom) +# import pdb +# pdb.set_trace() # solve the repository and return the solution - rep = Repository(dom.keys(), dom, query_constr) - res_s = Solver(MyDistributor()).solve(rep, verbose=0) + rep = Repository(solve_vars, dom, query_constr) + res_s = Solver(MyDistributor(solve_vars)).solve(rep, verbose=0) res = [] query_vars = dict([('query_%s_'%name,name) for name in resvars]) for d in res_s: @@ -1059,7 +1071,11 @@ Individuals that have a property with the value of var. To make an assertion we need to know for which class the restriction applies""" sub = self.make_var(Restriction, s) - cons = HasvalueConstraint(sub, var) + try: + prop = self.variables[sub].property + except KeyError: + prop = None + cons = HasvalueConstraint(sub, prop, var) self.constraints.append(cons) def allValuesFrom(self, s, var): From cfbolz at codespeak.net Tue May 1 13:43:39 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 1 May 2007 13:43:39 +0200 (CEST) Subject: [pypy-svn] r42520 - in pypy/extradoc/talk/badhonnef2007: . image Message-ID: <20070501114339.328A18069@code0.codespeak.net> Author: cfbolz Date: Tue May 1 13:43:38 2007 New Revision: 42520 Added: pypy/extradoc/talk/badhonnef2007/ pypy/extradoc/talk/badhonnef2007/image/ pypy/extradoc/talk/badhonnef2007/image/arch.pdf (contents, props changed) pypy/extradoc/talk/badhonnef2007/image/bench.png (contents, props changed) pypy/extradoc/talk/badhonnef2007/image/py-web.png (contents, props changed) pypy/extradoc/talk/badhonnef2007/talk.pdf (contents, props changed) pypy/extradoc/talk/badhonnef2007/talk.tex Log: talk about pypy and prolog, heavily stolen from the dls talk Added: pypy/extradoc/talk/badhonnef2007/image/arch.pdf ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/badhonnef2007/image/bench.png ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/badhonnef2007/image/py-web.png ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/badhonnef2007/talk.pdf ============================================================================== Binary file. No diff available. Added: pypy/extradoc/talk/badhonnef2007/talk.tex ============================================================================== --- (empty file) +++ pypy/extradoc/talk/badhonnef2007/talk.tex Tue May 1 13:43:38 2007 @@ -0,0 +1,465 @@ +\documentclass[utf8]{beamer} + +% This file is a solution template for: + +% - Talk at a conference/colloquium. +% - Talk length is about 20min. +% - Style is ornate. + +\mode +{ + \usetheme{Warsaw} + % or ... + + \setbeamercovered{transparent} + % or whatever (possibly just delete it) +} + + +\usepackage[english]{babel} + +\usepackage[utf8]{inputenc} +% or whatever + +\usepackage{times} +\usepackage[T1]{fontenc} +% Or whatever. Note that the encoding and the font should match. If T1 +% does not look nice, try deleting the line with the fontenc. + + +\title{A flexible Prolog interpreter in Python} + +\author{Carl Friedrich Bolz} +% - Give the names in the same order as the appear in the paper. +% - Use the \inst{?} command only if the authors have different +% affiliation. + +\institute[Heinrich-Heine-Universit?t D?sseldorf] +{ + Institut f?r Informatik\\ + Heinrich-Heine-Universit?t D?sseldorf +} + +\date{24. Workshop der GI-Fachgruppe Programmiersprachen und Rechenkonzepte, 4. Mai 2007} +% - Either use conference name or its abbreviation. +% - Not really informative to the audience, more for people (including +% yourself) who are reading the slides online + + +% If you have a file called "university-logo-filename.xxx", where xxx +% is a graphic format that can be processed by latex or pdflatex, +% resp., then you can add a logo as follows: + +\pgfdeclareimage[height=0.5cm]{pypy-logo}{image/py-web.png} +\logo{\pgfuseimage{pypy-logo}} + + + +% Delete this, if you do not want the table of contents to pop up at +% the beginning of each subsection: +%\AtBeginSubsection[] +%{ +% \begin{frame} +% \frametitle{Outline} +% \tableofcontents[currentsection,currentsubsection] +% \end{frame} +%} + + +% If you wish to uncover everything in a step-wise fashion, uncomment +% the following command: + +%\beamerdefaultoverlayspecification{<+->} + + +\begin{document} + +\begin{frame} + \titlepage +\end{frame} + +\begin{frame} + \frametitle{Outline} + \tableofcontents + % You might wish to add the option [pausesections] +\end{frame} + + +% Structuring a talk is a difficult task and the following structure +% may not be suitable. Here are some rules that apply for this +% solution: + +% - Exactly two or three sections (other than the summary). +% - At *most* three subsections per section. +% - Talk about 30s to 2min per frame. So there should be between about +% 15 and 30 frames, all told. + +% - A conference audience is likely to know very little of what you +% are going to talk about. So *simplify*! +% - In a 20min talk, getting the main ideas across is hard +% enough. Leave out details, even if it means being less precise than +% you think necessary. +% - If you omit details that are vital to the proof/implementation, +% just say so once. Everybody will be happy with that. + +\section{What is Pyrolog?} + +\begin{frame} + \frametitle{Pyrolog} + + \begin{itemize} + \item + Pyrolog is a Prolog interpreter written in RPython + \item + RPython is a subset of Python translatable to other languages + \item + translation part done with the help of the PyPy project + \end{itemize} +\end{frame} + +\section{The PyPy Approach to VM Construction} +\subsection{Overview} +\begin{frame} + \frametitle{What is PyPy?} + \begin{itemize} + \item + started as a Python VM implementation + in RPython (a well-chosen subset of Python) + \item + includes a translation tool-chain + \item + is becoming a general environment for writing interpreters (JavaScript, Prolog started) + \item + Open source project (MIT license) + \item + received EU funding for 2.5 years + \end{itemize} + +\end{frame} + +\subsection{Motivation} +\begin{frame} + \frametitle{VMs are still hard} + It is hard to achieve: + + \begin{itemize} + \item + flexibility + \item + maintainability + \item + performance (needs dynamic compilation techniques) + \end{itemize} + Especially with limited resources (like Open Source projects, research projects) +\end{frame} + + +\begin{frame} + \frametitle{The Python case (i)} + CPython (the reference implementation) is a straightforward, portable VM. + + \begin{itemize} + \item + Pervasive decisions: reference counting, single global lock ... + \item + No dynamic compilation +% \pause + \item + Extensions: + \begin{itemize} + \item + \alert{Stackless} (unlimited recursion, coroutines, serializable continuations) + \item + \alert{Psyco} (run-time specializer) + \item + \alert{Jython}, \alert{IronPython} + \end{itemize} + \end{itemize} +\end{frame} + + +\begin{frame} + \frametitle{The Python case (ii)} + \begin{itemize} + \item + Extensions have problems + \begin{itemize} + \item + need to keep track of CPython + \item + are hard to maintain + \item + Psyco very hard to port to other hardware architectures + \end{itemize} + \item + The community wants Python to run everywhere: + Jython (Java), IronPython (.NET). + Lots of effort and duplication. + + \item + At various points various incompatibilities between + the implementations + + \end{itemize} +\end{frame} + + +\begin{frame} + \frametitle{The Prolog case} + \begin{itemize} + \item + problem mitigated by the fact that Prolog the language does not change + \item + a lot of implementations out there + \item + well-tuned mature C implementations (Sictsus, XSB, SWI, GNU-Prolog) + \begin{itemize} + \item + have sometimes incompatible extensions to core Prolog + \item + interfacing with libraries is tedious + \item + changing the language to experiment is hard + \item + fixed implementation decisions (GC, how to generate code, etc.) + \end{itemize} + \item + on CLR (P\#) and JVM (Prolog Caf?, tuProlog) + \begin{itemize} + \item + interfacing with libraries of the platform mostly easy + \item + no extensions to core Prolog (like tabling, coroutines) + \item + slow, compared to good C implementations + \end{itemize} + \end{itemize} +\end{frame} + + +\subsection{Approach} +\begin{frame} + \frametitle{PyPy's Approach} + \alert{Goal:} generate VMs from a single high-level description of the + language, in a retargettable way. + \begin{itemize} + \item + Write an interpreter for a dynamic language (Python, Prolog, JavaScript, + whatever) in a high-level language (Python) + \item + Leave out low-level details + \item + Favour simplicity and flexibility + \item + Define a mapping to low-level targets + \item + Generate VMs from the interpreter + \end{itemize} +\end{frame} + + +\begin{frame} + \frametitle{Mapping to low-level targets} + \begin{itemize} + \item + Mechanically translate the interpreter to multiple + lower-level targets + \begin{itemize} + \item C-like + \item Java + \item .NET + \end{itemize} + \item + Insert low-level aspects into the code as required by + the target (Object layout, memory management) + \begin{itemize} + \item object layout + \item memory management + \end{itemize} \item + Optionally insert new pervasive features not expressed + in the source + \begin{itemize} + \item continuations + \item dynamic compilation + \end{itemize} + \end{itemize} +\end{frame} + + +\begin{frame} + \frametitle{Translation Steps} + \begin{columns}[c] + \begin{column}{5cm} + \includegraphics[width=5cm]{image/arch.pdf} + \end{column} + \begin{column}{7cm} + \begin{itemize} + \item + Generate flow graphs from the RPython program + \item + Peform global type inference on the flow graphs + \item + Transform flow graphs through several steps until they match the level of + the target environment + \item + Weave in translation aspects in the process + \end{itemize} + \end{column} + \end{columns} + +\end{frame} + + +\begin{frame} + \frametitle{Translation Aspects (i)} + Features not present in the source can be + added during translation. + + Example: memory management: + \begin{itemize} + \item + Boehm garbage collector + \item + mark-n-sweep written in RPython, with additional features + \item + reference counting + \end{itemize} +\end{frame} + + +\begin{frame} + \frametitle{Translation Aspects (ii)} + \begin{itemize} + \item + \alert{Stackless transformation}: continuation capture, implemented by + saving the low-level frames' local variables into the heap and back + \begin{itemize} + \item allows arbitrarily deep stack usage + \item uses the C stack as long as possible + \item has the consequence of making RPython do tail call elimination + \end{itemize} + \item + work in progress: turning an interpreter into a just-in-time compiler + is a translation aspect too + \end{itemize} +\end{frame} + +\section{The Prolog interpreter} +\begin{frame} + \frametitle{Prolog Interpreter Implementation} + \begin{itemize} + \item + naive, very simple interpreter + \begin{itemize} + \item uses "structure copying" + \item interprets Prolog terms directly, no bytecode + \end{itemize} + \item + uses continuation passing style similar to BinProlog + \item + Prolog calls mapped to RPython calls + \begin{itemize} + \item possible because stackless allows arbitrary deep recursion + \end{itemize} + \item + implements large parts of the ISO standard (some builtins missing) + \end{itemize} +\end{frame} + + + + +\begin{frame} + \frametitle{Builtins} + \begin{itemize} + \item + builtins implemented in Python + \item + easy to add new ones to interface with libraries + \item + application-specific builtins + \item + examples: + \begin{itemize} + \item functions to download and analyze webpages + \item an imperative hashmap + \end{itemize} + \end{itemize} +\end{frame} + + +\begin{frame} + \frametitle{Interpreter Facts} + \begin{itemize} + \item + 2500 lines of Python code in total + \item + 700 of those are for builtins + \item + after translation to C: 14000 line of C code + \item + part of the PyPy distribution at: + http://codespeak.net/pypy + \end{itemize} +\end{frame} + + + +\begin{frame} + \frametitle{Performance (i)} + \includegraphics[scale=0.4]{image/bench.png} +\end{frame} + + +\begin{frame} + \frametitle{Performance (ii)} + \begin{itemize} + \item + performance is quite bad compared to tuned C implementations + \item + performance is pretty good compared to Java and .NET implementations + \item + surprising, since those are often based on the WAM + \item + maybe it's hard to simulate the WAM on such a VM + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Title} + \begin{itemize} + \item + \end{itemize} +\end{frame} + + +\section*{Summary} +\subsection*{Summary} +\begin{frame} + \frametitle{Summary} + + % Keep the summary *very short*. + \begin{itemize} + \item + The construction of virtual machines gets easier when using high-level + languages + \item + XXX + \item + XXX + \end{itemize} +\end{frame} +\subsection*{Outlook} +\begin{frame} + \frametitle{Outlook} + % The following outlook is optional. + \begin{itemize} + \item + \end{itemize} +\end{frame} + + + +\end{document} + + From cfbolz at codespeak.net Tue May 1 23:36:17 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 1 May 2007 23:36:17 +0200 (CEST) Subject: [pypy-svn] r42554 - pypy/extradoc/talk/badhonnef2007 Message-ID: <20070501213617.48D098061@code0.codespeak.net> Author: cfbolz Date: Tue May 1 23:36:16 2007 New Revision: 42554 Modified: pypy/extradoc/talk/badhonnef2007/talk.pdf pypy/extradoc/talk/badhonnef2007/talk.tex Log: reworking the talk Modified: pypy/extradoc/talk/badhonnef2007/talk.pdf ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/badhonnef2007/talk.tex ============================================================================== --- pypy/extradoc/talk/badhonnef2007/talk.tex (original) +++ pypy/extradoc/talk/badhonnef2007/talk.tex Tue May 1 23:36:16 2007 @@ -111,9 +111,12 @@ \item Pyrolog is a Prolog interpreter written in RPython \item - RPython is a subset of Python translatable to other languages + RPython (``Restricted Python'') is a subset of Python translatable to other + languages + \item + RPython is designed to be significantly faster than regular Python \item - translation part done with the help of the PyPy project + translation tool-chain part of the PyPy project \end{itemize} \end{frame} @@ -123,14 +126,15 @@ \frametitle{What is PyPy?} \begin{itemize} \item - started as a Python VM implementation - in RPython (a well-chosen subset of Python) + started as a Python VM implementation in Python \item - includes a translation tool-chain + Python itself is too dynamic to be translatable to other languages, need a subset + \item + includes a translation tool-chain for RPython \item is becoming a general environment for writing interpreters (JavaScript, Prolog started) \item - Open source project (MIT license) + Open Source project (MIT license) \item received EU funding for 2.5 years \end{itemize} @@ -140,7 +144,7 @@ \subsection{Motivation} \begin{frame} \frametitle{VMs are still hard} - It is hard to achieve: + Hard to reconcile: \begin{itemize} \item @@ -160,12 +164,13 @@ \begin{itemize} \item - Pervasive decisions: reference counting, single global lock ... + Pervasive decisions: reference counting, global lock \dots \item No dynamic compilation % \pause - \item - Extensions: + \end{itemize} + \begin{block}{ + Extensions:} \begin{itemize} \item \alert{Stackless} (unlimited recursion, coroutines, serializable continuations) @@ -174,23 +179,20 @@ \item \alert{Jython}, \alert{IronPython} \end{itemize} - \end{itemize} + \end{block} \end{frame} \begin{frame} \frametitle{The Python case (ii)} - \begin{itemize} - \item - Extensions have problems + \begin{block}{Problems of Extensions:} \begin{itemize} - \item - need to keep track of CPython - \item - are hard to maintain - \item - Psyco very hard to port to other hardware architectures + \item hard to maintain: need to keep track of CPython + \item Psyco very hard to port to other hardware architectures + \item tedious to write them, Python semantics need to be re-implemented \end{itemize} + \end{block} + \begin{itemize} \item The community wants Python to run everywhere: Jython (Java), IronPython (.NET). @@ -205,26 +207,36 @@ \begin{frame} - \frametitle{The Prolog case} + \frametitle{The Prolog case (i)} \begin{itemize} \item problem mitigated by the fact that Prolog the language does not change \item + the core of Prolog is very simple (at least compared to Python) + \item a lot of implementations out there \item - well-tuned mature C implementations (Sictsus, XSB, SWI, GNU-Prolog) + well-tuned mature C implementations (Sicstus, XSB, SWI, GNU-Prolog) + \item + on CLR (P\#) and JVM (Prolog Caf?, tuProlog) + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{The Prolog case (ii)} + \begin{block}{mature C implementations} \begin{itemize} \item - have sometimes incompatible extensions to core Prolog - \item interfacing with libraries is tedious \item changing the language to experiment is hard \item + often extensions to core Prolog, incompatible between each other + \item fixed implementation decisions (GC, how to generate code, etc.) \end{itemize} - \item - on CLR (P\#) and JVM (Prolog Caf?, tuProlog) + \end{block} + \begin{block}{implementations on CLR and JVM} \begin{itemize} \item interfacing with libraries of the platform mostly easy @@ -233,10 +245,11 @@ \item slow, compared to good C implementations \end{itemize} - \end{itemize} + \end{block} \end{frame} + \subsection{Approach} \begin{frame} \frametitle{PyPy's Approach} @@ -279,72 +292,42 @@ Optionally insert new pervasive features not expressed in the source \begin{itemize} - \item continuations + \item continuations, ``micro-threads'' \item dynamic compilation \end{itemize} \end{itemize} \end{frame} -\begin{frame} - \frametitle{Translation Steps} - \begin{columns}[c] - \begin{column}{5cm} - \includegraphics[width=5cm]{image/arch.pdf} - \end{column} - \begin{column}{7cm} - \begin{itemize} - \item - Generate flow graphs from the RPython program - \item - Peform global type inference on the flow graphs - \item - Transform flow graphs through several steps until they match the level of - the target environment - \item - Weave in translation aspects in the process - \end{itemize} - \end{column} - \end{columns} - -\end{frame} - \begin{frame} \frametitle{Translation Aspects (i)} - Features not present in the source can be - added during translation. - - Example: memory management: + Features not present in the source can be added during translation. \begin{itemize} \item - Boehm garbage collector - \item - mark-n-sweep written in RPython, with additional features + \alert{memory management}: use different GC strategies (Boehm collector, custom mark-n-sweep) \item - reference counting + \alert{Stackless transformation}: allows program to control its stack (continuations, \dots) \end{itemize} \end{frame} \begin{frame} \frametitle{Translation Aspects (ii)} + A \alert{JIT compiler} as a translation aspect \begin{itemize} \item - \alert{Stackless transformation}: continuation capture, implemented by - saving the low-level frames' local variables into the heap and back - \begin{itemize} - \item allows arbitrarily deep stack usage - \item uses the C stack as long as possible - \item has the consequence of making RPython do tail call elimination - \end{itemize} + Transform the interpreter into a JIT compiler, using partial evaluation and + specialization techniques \item - work in progress: turning an interpreter into a just-in-time compiler - is a translation aspect too + Some hints in the interpreter source needed + \item + Current prototype applied to Python interpreter gives impressive speedups \end{itemize} \end{frame} \section{The Prolog interpreter} +\subsection{Interpreter} \begin{frame} \frametitle{Prolog Interpreter Implementation} \begin{itemize} @@ -355,12 +338,9 @@ \item interprets Prolog terms directly, no bytecode \end{itemize} \item - uses continuation passing style similar to BinProlog + uses continuation passing style inspired by BinProlog \item Prolog calls mapped to RPython calls - \begin{itemize} - \item possible because stackless allows arbitrary deep recursion - \end{itemize} \item implements large parts of the ISO standard (some builtins missing) \end{itemize} @@ -399,12 +379,12 @@ after translation to C: 14000 line of C code \item part of the PyPy distribution at: - http://codespeak.net/pypy + \texttt{http://codespeak.net/pypy} \end{itemize} \end{frame} - +\subsection{Performance} \begin{frame} \frametitle{Performance (i)} \includegraphics[scale=0.4]{image/bench.png} @@ -421,14 +401,7 @@ \item surprising, since those are often based on the WAM \item - maybe it's hard to simulate the WAM on such a VM - \end{itemize} -\end{frame} - -\begin{frame} - \frametitle{Title} - \begin{itemize} - \item + maybe the WAM model does not match these VMs very well? \end{itemize} \end{frame} @@ -436,23 +409,61 @@ \section*{Summary} \subsection*{Summary} \begin{frame} - \frametitle{Summary} + \frametitle{Summary} % Keep the summary *very short*. \begin{itemize} \item - The construction of virtual machines gets easier when using high-level - languages + Very simple Prolog interpreter in RPython can compete with interpreters on the JVM, CLR \item - XXX + Interpreter implementation eased by use of a high-level language \item - XXX + Low-level details abstracted away but re-inserted later \end{itemize} \end{frame} + \subsection*{Outlook} \begin{frame} - \frametitle{Outlook} - % The following outlook is optional. + \frametitle{Outlook} + \begin{itemize} + \item + complete the set of builtins + \item + tight language integration between Prolog and Python + \item + apply the dynamic compiler generator to the Prolog interpreter + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Backup slides} + \dots +\end{frame} + +\begin{frame} + \frametitle{Translation Steps} + \begin{columns}[c] + \begin{column}{5cm} + \includegraphics[width=5cm]{image/arch.pdf} + \end{column} + \begin{column}{7cm} + \begin{itemize} + \item + Generate flow graphs from the RPython program + \item + Peform global type inference on the flow graphs + \item + Transform flow graphs through several steps until they match the level of + the target environment + \item + Weave in translation aspects in the process + \end{itemize} + \end{column} + \end{columns} +\end{frame} + +\begin{frame} + \frametitle{Title} \begin{itemize} \item \end{itemize} From cfbolz at codespeak.net Tue May 1 23:41:30 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 1 May 2007 23:41:30 +0200 (CEST) Subject: [pypy-svn] r42556 - pypy/extradoc/eu-report Message-ID: <20070501214130.86AF3807E@code0.codespeak.net> Author: cfbolz Date: Tue May 1 23:41:29 2007 New Revision: 42556 Added: pypy/extradoc/eu-report/D08.2_JIT_Compiler_Architecture-2007-05-01.pdf (contents, props changed) Removed: pypy/extradoc/eu-report/D08.2_JIT_Compiler_Architecture-interim-2007-01-28.pdf Log: final D08.2 report Added: pypy/extradoc/eu-report/D08.2_JIT_Compiler_Architecture-2007-05-01.pdf ============================================================================== Binary file. No diff available. From cfbolz at codespeak.net Tue May 1 23:43:30 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 1 May 2007 23:43:30 +0200 (CEST) Subject: [pypy-svn] r42557 - pypy/dist/pypy/doc Message-ID: <20070501214330.E3141807C@code0.codespeak.net> Author: cfbolz Date: Tue May 1 23:43:30 2007 New Revision: 42557 Modified: pypy/dist/pypy/doc/index-report.txt Log: update link here Modified: pypy/dist/pypy/doc/index-report.txt ============================================================================== --- pypy/dist/pypy/doc/index-report.txt (original) +++ pypy/dist/pypy/doc/index-report.txt Tue May 1 23:43:30 2007 @@ -12,6 +12,9 @@ Reports of 2007 =============== +`D08.2 JIT Compiler Architecture`_ is a report about the Architecture and +working of our JIT compiler generator. *(2007-05-01)* + `D08.1 JIT Compiler Release`_ reports on our sucessfully including a JIT compiler for Python and the novel framework we used to automatically generate it in PyPy 1.0. *(2007-04-30)* @@ -82,10 +85,6 @@ feedback for it is welcome. *(2007-02-28)* -`Draft D08.2 JIT Compiler Architecture`_ is an interim version of a report -about the Architecture and working of our JIT compiler generator. The report -is still a draft, all feedback for it is welcome. *(2007-01-28)* - .. _`py-lib`: http://codespeak.net/py/current/doc/ .. _`py.test`: http://codespeak.net/py/current/doc/test.html @@ -146,7 +145,7 @@ .. _`D05.4 Encapsulating Low Level Aspects`: http://codespeak.net/svn/pypy/extradoc/eu-report/D05.4_Publish_on_encapsulating_low_level_language_aspects.pdf .. _`D06.1 Core Object Optimization Results`: http://codespeak.net/svn/pypy/extradoc/eu-report/D06.1_Core_Optimizations-2007-04-30.pdf .. _`D07.1 Massive Parallelism and Translation Aspects`: http://codespeak.net/pypy/extradoc/eu-report/D07.1_Massive_Parallelism_and_Translation_Aspects-2007-02-28.pdf -.. _`Draft D08.2 JIT Compiler Architecture`: http://codespeak.net/pypy/extradoc/eu-report/D08.2_JIT_Compiler_Architecture-interim-2007-01-28.pdf +.. _`D08.2 JIT Compiler Architecture`: http://codespeak.net/pypy/extradoc/eu-report/D08.2_JIT_Compiler_Architecture-2007-05-01.pdf .. _`D08.1 JIT Compiler Release`: http://codespeak.net/pypy/extradoc/eu-report/D08.1_JIT_Compiler_Release-2007-04-30.pdf .. _`D09.1 Constraint Solving and Semantic Web`: http://codespeak.net/pypy/extradoc/eu-report/D09.1_Constraint_Solving_and_Semantic_Web-interim-2007-02-28.pdf .. _`D10.1 Aspect-Oriented, Design-by-Contract Programming and RPython static checking`: http://codespeak.net/pypy/extradoc/eu-report/D10.1_Aspect_Oriented_Programming_in_PyPy-2007-03-22.pdf From cfbolz at codespeak.net Tue May 1 23:50:38 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 1 May 2007 23:50:38 +0200 (CEST) Subject: [pypy-svn] r42559 - pypy/extradoc/eu-report Message-ID: <20070501215038.55FB0807E@code0.codespeak.net> Author: cfbolz Date: Tue May 1 23:50:37 2007 New Revision: 42559 Added: pypy/extradoc/eu-report/D14.4_Report_About_Milestone_Phase_3-2007-2007-05-01.pdf - copied unchanged from r42558, pypy/eu-tracking/deliverable/D14_4_Report_About_Milestone_Phase_3/D14.4-deliverablereport.pdf Removed: pypy/extradoc/eu-report/D14.4_Report_About_Milestone_Phase_3-interim-2007-2007-04-05.pdf Log: final version of 14.4 From cfbolz at codespeak.net Tue May 1 23:52:50 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 1 May 2007 23:52:50 +0200 (CEST) Subject: [pypy-svn] r42560 - pypy/dist/pypy/doc Message-ID: <20070501215250.71818807C@code0.codespeak.net> Author: cfbolz Date: Tue May 1 23:52:49 2007 New Revision: 42560 Modified: pypy/dist/pypy/doc/index-report.txt Log: update link Modified: pypy/dist/pypy/doc/index-report.txt ============================================================================== --- pypy/dist/pypy/doc/index-report.txt (original) +++ pypy/dist/pypy/doc/index-report.txt Tue May 1 23:52:49 2007 @@ -12,6 +12,10 @@ Reports of 2007 =============== +`D14.4 PyPy-1.0 Milestone report`_ (for language developers and researchers) +summarizes research & technical results of the PyPy-1.0 release and discusses +related development process and community aspects. *(2007-05-01)* + `D08.2 JIT Compiler Architecture`_ is a report about the Architecture and working of our JIT compiler generator. *(2007-05-01)* @@ -19,11 +23,6 @@ JIT compiler for Python and the novel framework we used to automatically generate it in PyPy 1.0. *(2007-04-30)* -`D14.4 PyPy-1.0 Milestone report`_ (for language developers and researchers) -summarizes research & technical results of the PyPy-1.0 release and discusses -related development process and community aspects. This report is an interim -version, we may still incorporate feedback. *(2007-04-05)* - `D06.1 Core Object Optimization Results`_ documents the optimizations we implemented in the interpreter and object space: dictionary implementations, method call optimizations, etc. The report is still not final @@ -155,7 +154,7 @@ .. _`D14.1 Report about Milestone/Phase 1`: http://codespeak.net/svn/pypy/extradoc/eu-report/D14.1_Report_about_Milestone_Phase_1.pdf .. _`D14.2 Tutorials and Guide Through the PyPy Source Code`: http://codespeak.net/pypy/extradoc/eu-report/D14.2_Tutorials_and_Guide_Through_the_PyPy_Source_Code-2007-03-22.pdf .. _`D14.3 Report about Milestone/Phase 2`: http://codespeak.net/pypy/extradoc/eu-report/D14.3_Report_about_Milestone_Phase_2-final-2006-08-03.pdf -.. _`D14.4 PyPy-1.0 Milestone report`: http://codespeak.net/pypy/extradoc/eu-report/D14.4_Report_About_Milestone_Phase_3-interim-2007-2007-04-05.pdf +.. _`D14.4 PyPy-1.0 Milestone report`: http://codespeak.net/pypy/extradoc/eu-report/D14.4_Report_About_Milestone_Phase_3-2007-2007-05-01.pdf .. _`D14.5 Documentation of the development process`: http://codespeak.net/pypy/extradoc/eu-report/D14.5_Documentation_of_the_development_process-2007-03-30.pdf From cfbolz at codespeak.net Tue May 1 23:54:40 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 1 May 2007 23:54:40 +0200 (CEST) Subject: [pypy-svn] r42561 - pypy/extradoc/eu-report Message-ID: <20070501215440.5EF78807C@code0.codespeak.net> Author: cfbolz Date: Tue May 1 23:54:39 2007 New Revision: 42561 Added: pypy/extradoc/eu-report/D14.4_Report_About_Milestone_Phase_3-2007-05-01.pdf - copied unchanged from r42559, pypy/extradoc/eu-report/D14.4_Report_About_Milestone_Phase_3-2007-2007-05-01.pdf Removed: pypy/extradoc/eu-report/D14.4_Report_About_Milestone_Phase_3-2007-2007-05-01.pdf Log: remove double 2007 in filename ?! From cfbolz at codespeak.net Tue May 1 23:55:22 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 1 May 2007 23:55:22 +0200 (CEST) Subject: [pypy-svn] r42562 - pypy/dist/pypy/doc Message-ID: <20070501215522.0EC4C807F@code0.codespeak.net> Author: cfbolz Date: Tue May 1 23:55:21 2007 New Revision: 42562 Modified: pypy/dist/pypy/doc/index-report.txt Log: remove it here too Modified: pypy/dist/pypy/doc/index-report.txt ============================================================================== --- pypy/dist/pypy/doc/index-report.txt (original) +++ pypy/dist/pypy/doc/index-report.txt Tue May 1 23:55:21 2007 @@ -154,7 +154,7 @@ .. _`D14.1 Report about Milestone/Phase 1`: http://codespeak.net/svn/pypy/extradoc/eu-report/D14.1_Report_about_Milestone_Phase_1.pdf .. _`D14.2 Tutorials and Guide Through the PyPy Source Code`: http://codespeak.net/pypy/extradoc/eu-report/D14.2_Tutorials_and_Guide_Through_the_PyPy_Source_Code-2007-03-22.pdf .. _`D14.3 Report about Milestone/Phase 2`: http://codespeak.net/pypy/extradoc/eu-report/D14.3_Report_about_Milestone_Phase_2-final-2006-08-03.pdf -.. _`D14.4 PyPy-1.0 Milestone report`: http://codespeak.net/pypy/extradoc/eu-report/D14.4_Report_About_Milestone_Phase_3-2007-2007-05-01.pdf +.. _`D14.4 PyPy-1.0 Milestone report`: http://codespeak.net/pypy/extradoc/eu-report/D14.4_Report_About_Milestone_Phase_3-2007-05-01.pdf .. _`D14.5 Documentation of the development process`: http://codespeak.net/pypy/extradoc/eu-report/D14.5_Documentation_of_the_development_process-2007-03-30.pdf From santagada at codespeak.net Wed May 2 05:57:04 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Wed, 2 May 2007 05:57:04 +0200 (CEST) Subject: [pypy-svn] r42570 - in pypy/dist/pypy/lang/js: . test Message-ID: <20070502035704.D126C807C@code0.codespeak.net> Author: santagada Date: Wed May 2 05:57:03 2007 New Revision: 42570 Modified: pypy/dist/pypy/lang/js/jsgrammar.txt pypy/dist/pypy/lang/js/newparser.py pypy/dist/pypy/lang/js/test/test_new_parser.py Log: started testing of expressions... we are getting closer to the point where this parser is going to take the place of using narcissus + spidermonkey Modified: pypy/dist/pypy/lang/js/jsgrammar.txt ============================================================================== --- pypy/dist/pypy/lang/js/jsgrammar.txt (original) +++ pypy/dist/pypy/lang/js/jsgrammar.txt Wed May 2 05:57:03 2007 @@ -15,9 +15,9 @@ statement : | [";"] - | [ ";" ] + | | [";"] - | [";"] + | [";"]* | [";"] | [";"] | [";"] @@ -49,6 +49,9 @@ initialiser : ["="] assignmentexpression ; +emptystatement : ";" + ; + expressionstatement : expression ; @@ -88,7 +91,11 @@ ; # XXX this looks wrong to me -caseclauses : caseclause caseclauses +# ans: it was defined in the spec as +# CaseClauses : +# CaseClause +# CaseClauses CaseClause +caseclauses : caseclause+ ; caseclause : ["case"] expression [":"] statementlist? Modified: pypy/dist/pypy/lang/js/newparser.py ============================================================================== --- pypy/dist/pypy/lang/js/newparser.py (original) +++ pypy/dist/pypy/lang/js/newparser.py Wed May 2 05:57:03 2007 @@ -15,7 +15,7 @@ newrules = [Rule("hacked_first_symbol", [[start, "EOF"]])] + rules return newrules -parse = make_parse_function(regexs, setstartrule(rules, start="expression"), eof=True) +parse = make_parse_function(regexs, setstartrule(rules, start="statement"), eof=True) #parse = make_parse_function(regexs, rules) print rules[2].nonterminal Modified: pypy/dist/pypy/lang/js/test/test_new_parser.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_new_parser.py (original) +++ pypy/dist/pypy/lang/js/test/test_new_parser.py Wed May 2 05:57:03 2007 @@ -53,7 +53,7 @@ self.counts = {} def general_nonterminal_visit(self, node): - print node + print node.symbol self.counts[node.symbol] = self.counts.get(node.symbol, 0) + 1 for child in node.children: self.dispatch(child) @@ -157,6 +157,8 @@ ]) self.parse_and_compare("++5", 6) self.parse_and_compare("~3", -4) + self.parse("x = 3") + self.parse("x") def test_chained(self): self.parse_and_eval_all(["1 + 2 * 3", @@ -167,3 +169,39 @@ "30 | 3 & 5", ]) +class TestStatements(BaseGrammarTest): + def setup_class(cls): + cls.parse = parse_func('statement') + + def parse_count(self, s): + "parse the expression and return the CountingVisitor" + cv = CountingVisitor() + self.parse(s).visit(cv) + return cv.counts + + def test_block(self): + r = self.parse_count("{x;return;true;if(x);}") + assert r['block'] == 1 + + def test_vardecl(self): + r = self.parse_count("var x;") + assert r['variablestatement'] == 1 + + r = self.parse_count("var x = 2;") + assert r['variablestatement'] == 1 + + def test_empty(self): + for i in range(1,10): + r = self.parse_count('{%s}'%(';'*i)) + assert r['emptystatement'] == i + + def test_if(self): + r = self.parse_count("if(x)return;;") + assert r['ifstatement'] == 1 + assert r.get('emptystatement', 0) == 0 + r = self.parse_count("if(x)if(i)return;") + assert r['ifstatement'] == 2 + r = self.parse_count("if(x)return;else return;") + assert r['ifstatement'] == 1 + + From santagada at codespeak.net Wed May 2 06:03:30 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Wed, 2 May 2007 06:03:30 +0200 (CEST) Subject: [pypy-svn] r42571 - pypy/dist/pypy/lang/js Message-ID: <20070502040330.7C5D8807F@code0.codespeak.net> Author: santagada Date: Wed May 2 06:03:30 2007 New Revision: 42571 Modified: pypy/dist/pypy/lang/js/jsgrammar.txt Log: should be a ? not a *... but still I dunno if this is correct or not Modified: pypy/dist/pypy/lang/js/jsgrammar.txt ============================================================================== --- pypy/dist/pypy/lang/js/jsgrammar.txt (original) +++ pypy/dist/pypy/lang/js/jsgrammar.txt Wed May 2 06:03:30 2007 @@ -17,7 +17,7 @@ | [";"] | | [";"] - | [";"]* + | [";"]? | [";"] | [";"] | [";"] From antocuni at codespeak.net Wed May 2 23:47:08 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 2 May 2007 23:47:08 +0200 (CEST) Subject: [pypy-svn] r42600 - in pypy/dist/pypy/translator: . cli cli/test Message-ID: <20070502214708.7B0B2807F@code0.codespeak.net> Author: antocuni Date: Wed May 2 23:47:07 2007 New Revision: 42600 Added: pypy/dist/pypy/translator/cli/silverpython.py (contents, props changed) pypy/dist/pypy/translator/cli/test/test_silverpython.py (contents, props changed) Modified: pypy/dist/pypy/translator/cli/cts.py pypy/dist/pypy/translator/cli/entrypoint.py pypy/dist/pypy/translator/cli/function.py pypy/dist/pypy/translator/cli/gencli.py pypy/dist/pypy/translator/cli/rte.py pypy/dist/pypy/translator/cli/test/runtest.py pypy/dist/pypy/translator/driver.py Log: - add support to driver for building libraries instead of standalone executables (i.e., allowing more than one entry-point) - first steps to write a gencli's frontend which allow to compile a rpython module to a .NET dll; the draft name is SilveRPython, suggestions are welcome :-) Modified: pypy/dist/pypy/translator/cli/cts.py ============================================================================== --- pypy/dist/pypy/translator/cli/cts.py (original) +++ pypy/dist/pypy/translator/cli/cts.py Wed May 2 23:47:07 2007 @@ -196,6 +196,9 @@ ret_type, ret_var = self.llvar_to_cts(graph.getreturnvar()) func_name = func_name or graph.name func_name = self.escape_name(func_name) + namespace = getattr(graph.func, '_namespace_', None) + if namespace: + func_name = '%s::%s' % (namespace, func_name) args = [arg for arg in graph.getargs() if arg.concretetype is not ootype.Void] if is_method: Modified: pypy/dist/pypy/translator/cli/entrypoint.py ============================================================================== --- pypy/dist/pypy/translator/cli/entrypoint.py (original) +++ pypy/dist/pypy/translator/cli/entrypoint.py Wed May 2 23:47:07 2007 @@ -19,6 +19,12 @@ self.db = db self.cts = CTS(db) + def ilasm_flags(self): + return [] + + def output_filename(self, il_filename): + return il_filename.replace('.il', '.exe') + class StandaloneEntryPoint(BaseEntryPoint): """ This class produces a 'main' method that converts the argv in a @@ -59,3 +65,21 @@ ilasm.opcode('ret') ilasm.end_function() self.db.pending_function(self.graph) + +class DllEntryPoint(BaseEntryPoint): + def __init__(self, name, graphs): + self.name = name + self.graphs = graphs + + def get_name(self): + return self.name + + def ilasm_flags(self): + return ['/dll'] + + def output_filename(self, il_filename): + return il_filename.replace('.il', '.dll') + + def render(self, ilasm): + for graph in self.graphs: + self.db.pending_function(graph) Modified: pypy/dist/pypy/translator/cli/function.py ============================================================================== --- pypy/dist/pypy/translator/cli/function.py (original) +++ pypy/dist/pypy/translator/cli/function.py Wed May 2 23:47:07 2007 @@ -173,7 +173,18 @@ OOFunction.__init__(self, *args, **kwargs) self._set_args() self._set_locals() - + namespace = getattr(self.graph.func, '_namespace_', None) + str + if namespace: + if '.' in namespace: + self.namespace, self.classname = namespace.rsplit('.', 1) + else: + self.namespace = None + self.classname = namespace + else: + self.namespace = None + self.classname = None + def _create_generator(self, ilasm): return self # Function implements the Generator interface @@ -192,11 +203,20 @@ else: args = self.args meth_type = 'static' - self.ilasm.begin_function(self.name, args, returntype, self.is_entrypoint, meth_type) + + if self.namespace: + self.ilasm.begin_namespace(self.namespace) + if self.classname: + self.ilasm.begin_class(self.classname) + self.ilasm.begin_function(self.name, args, returntype, self.is_entrypoint, meth_type) self.ilasm.locals(self.locals) def end_render(self): self.ilasm.end_function() + if self.classname: + self.ilasm.end_class() + if self.namespace: + self.ilasm.end_namespace() def set_label(self, label): self.ilasm.label(label) Modified: pypy/dist/pypy/translator/cli/gencli.py ============================================================================== --- pypy/dist/pypy/translator/cli/gencli.py (original) +++ pypy/dist/pypy/translator/cli/gencli.py Wed May 2 23:47:07 2007 @@ -87,20 +87,20 @@ ilasm = SDK.ilasm() tmpfile = self.tmpfile.strpath - self._exec_helper(ilasm, tmpfile, + self._exec_helper(ilasm, [tmpfile]+self.entrypoint.ilasm_flags(), 'ilasm failed to assemble (%s):\n%s\n%s', timeout = 900) # Mono's ilasm occasionally deadlocks. We set a timer to avoid # blocking automated test runs forever. - exefile = tmpfile.replace('.il', '.exe') + self.outfile = self.entrypoint.output_filename(tmpfile) if getoption('verify'): peverify = SDK.peverify() - self._exec_helper(peverify, exefile, 'peverify failed to verify (%s):\n%s\n%s') - return exefile + self._exec_helper(peverify, [outfile], 'peverify failed to verify (%s):\n%s\n%s') + return self.outfile - def _exec_helper(self, helper, filename, msg, timeout=None): - args = [helper, filename] + def _exec_helper(self, helper, args, msg, timeout=None): + args = [helper] + args if timeout and not sys.platform.startswith('win'): import os from pypy.tool import autopath @@ -109,5 +109,5 @@ proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = proc.communicate() retval = proc.wait() - assert retval == 0, msg % (filename, stdout, stderr) + assert retval == 0, msg % (args[0], stdout, stderr) Modified: pypy/dist/pypy/translator/cli/rte.py ============================================================================== --- pypy/dist/pypy/translator/cli/rte.py (original) +++ pypy/dist/pypy/translator/cli/rte.py Wed May 2 23:47:07 2007 @@ -11,14 +11,9 @@ from py.compat import subprocess from pypy.translator.cli.sdk import SDK from pypy.tool.ansi_print import ansi_log -log = py.log.Producer("cli") -py.log.setconsumer("cli", ansi_log) +log = py.log.Producer("cli") +py.log.setconsumer("cli", ansi_log) -SRC_DIR = os.path.join(os.path.dirname(__file__), 'src/') - -def _filename(name, path=None): - rel_path = os.path.join(SRC_DIR, name) - return os.path.abspath(rel_path) class Target: SOURCES = [] @@ -26,6 +21,12 @@ ALIAS = None FLAGS = [] DEPENDENCIES = [] + SRC_DIR = os.path.join(os.path.dirname(__file__), 'src/') + + def _filename(cls, name, path=None): + rel_path = os.path.join(cls.SRC_DIR, name) + return os.path.abspath(rel_path) + _filename = classmethod(_filename) def get_COMPILER(cls): return SDK.csc() @@ -34,9 +35,9 @@ def get(cls): for dep in cls.DEPENDENCIES: dep.get() - sources = [_filename(src) for src in cls.SOURCES] - out = _filename(cls.OUTPUT) - alias = _filename(cls.ALIAS or cls.OUTPUT) + sources = [cls._filename(src) for src in cls.SOURCES] + out = cls._filename(cls.OUTPUT) + alias = cls._filename(cls.ALIAS or cls.OUTPUT) recompile = True try: src_mtime = max([os.stat(src).st_mtime for src in sources]) @@ -54,14 +55,14 @@ def compile(cls, sources, out): log.red("Compiling %s" % (cls.ALIAS or cls.OUTPUT)) oldcwd = os.getcwd() - os.chdir(SRC_DIR) + os.chdir(cls.SRC_DIR) compiler = subprocess.Popen([cls.get_COMPILER()] + cls.FLAGS + ['/out:%s' % out] + sources, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = compiler.communicate() retval = compiler.wait() assert retval == 0, 'Failed to compile %s: the compiler said:\n %s' % (cls.OUTPUT, stderr) if cls.ALIAS is not None: - alias = _filename(cls.ALIAS) + alias = cls._filename(cls.ALIAS) shutil.copy(out, alias) os.chdir(oldcwd) Added: pypy/dist/pypy/translator/cli/silverpython.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/cli/silverpython.py Wed May 2 23:47:07 2007 @@ -0,0 +1,44 @@ +from pypy.translator.driver import TranslationDriver +from pypy.translator.cli.entrypoint import DllEntryPoint + +class DllDef: + def __init__(self, name, namespace, functions=[], classes=[]): + self.name = name + self.namespace = namespace + self.functions = functions # [(function, annotation), ...] + + def add_function(self, func, inputtypes): + self.functions.append((func, inputtypes)) + + def get_entrypoint(self, bk): + graphs = [bk.getdesc(f).cachedgraph(None) for f, _ in self.functions] + return DllEntryPoint(self.name, graphs) + + def compile(self): + # add all functions to the appropriate namespace + for func, _ in self.functions: + if not hasattr(func, '_namespace_'): + func._namespace_ = self.namespace + driver = TranslationDriver() + driver.setup_library(self) + driver.proceed(['compile_cli']) + return driver + + +class MyClass: + def __init__(self, x): + self.x = x + + def foo(self): + return self.x + +def main(): + dll = DllDef('mylibrary', 'foo', [], [ + (MyClass, [int]), + ]) + driver = dll.compile() + driver.copy_cli_dll() + + +if __name__ == '__main__': + main() Modified: pypy/dist/pypy/translator/cli/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/runtest.py (original) +++ pypy/dist/pypy/translator/cli/test/runtest.py Wed May 2 23:47:07 2007 @@ -137,15 +137,15 @@ assert False, 'Input type %s not supported' % arg_type -def compile_function(func, annotation=[], graph=None, backend_opt={}): +def compile_function(func, annotation=[], graph=None, backendopt=True): olddefs = patch() - gen = _build_gen(func, annotation, graph, backend_opt) + gen = _build_gen(func, annotation, graph, backendopt) gen.generate_source() exe_name = gen.build_exe() unpatch(*olddefs) # restore original values return CliFunctionWrapper(exe_name) -def _build_gen(func, annotation, graph=None, backend_opt={}): +def _build_gen(func, annotation, graph=None, backendopt=True): try: func = func.im_func except AttributeError: @@ -165,8 +165,9 @@ t.view() t.buildrtyper(type_system="ootype").specialize() - check_virtual_methods(ootype.ROOT) - backend_optimizations(t) + if backendopt: + check_virtual_methods(ootype.ROOT) + backend_optimizations(t) main_graph = t.graphs[0] @@ -242,13 +243,13 @@ self._ann = None self._cli_func = None - def _compile(self, fn, args, ann=None): + def _compile(self, fn, args, ann=None, backendopt=True): if ann is None: ann = [lltype_to_annotation(typeOf(x)) for x in args] if self._func is fn and self._ann == ann: return self._cli_func else: - self._cli_func = compile_function(fn, ann) + self._cli_func = compile_function(fn, ann, backendopt=backendopt) self._func = fn self._ann = ann return self._cli_func @@ -261,8 +262,8 @@ if platform.processor() == 'powerpc': py.test.skip('PowerPC --> %s' % reason) - def interpret(self, fn, args, annotation=None): - f = self._compile(fn, args, annotation) + def interpret(self, fn, args, annotation=None, backendopt=True): + f = self._compile(fn, args, annotation, backendopt) res = f(*args) if isinstance(res, ExceptionWrapper): raise res Added: pypy/dist/pypy/translator/cli/test/test_silverpython.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/cli/test/test_silverpython.py Wed May 2 23:47:07 2007 @@ -0,0 +1,58 @@ +from pypy.tool import udir +from pypy.translator.cli.rte import Target +from pypy.translator.cli.silverpython import DllDef +from pypy.translator.cli.test.runtest import CliFunctionWrapper, CliTest + +TEMPLATE = """ +using System; +class SilveRPytonTest { + public static void Main() { + %s + } +} +""" + +class TestSilveRPython(CliTest): + + def _csharp(self, reference, source): + tmpfile = udir.udir.join('tmp.cs') + tmpfile.write(TEMPLATE % source) + if reference is None: + flags = [] + else: + flags = ['/r:%s' % reference] + + class MyTarget(Target): + SOURCES = [str(tmpfile)] + FLAGS = flags + OUTPUT = 'tmp.exe' + SRC_DIR = str(udir.udir) + + func = CliFunctionWrapper(MyTarget.get()) + return func() + + + def test_compilation(self): + res = self._csharp(None, 'Console.WriteLine(42);') + assert res == 42 + + def test_func_namespace(self): + def foo(x): + return x+1 + def bar(x): + return foo(x) + foo._namespace_ = 'MyNamespace.MyClass' + bar._namespace_ = 'MyClass' + res = self.interpret(bar, [41], backendopt=False) + assert res == 42 + + def test_simple_functions(self): + def foo(x): + return x+1 + def bar(x): + return x*2 + dll = DllDef('test', 'Test', [(foo, [int]), + (bar, [int])]) + dll.compile() + res = self._csharp('test', 'Console.WriteLine("{0}, {1}", Test.foo(42), Test.bar(42));') + assert res == (43, 84) Modified: pypy/dist/pypy/translator/driver.py ============================================================================== --- pypy/dist/pypy/translator/driver.py (original) +++ pypy/dist/pypy/translator/driver.py Wed May 2 23:47:07 2007 @@ -209,9 +209,14 @@ self.entry_point = entry_point self.translator = translator + self.libdef = None self.translator.driver_instrument_result = self.instrument_result + def setup_library(self, libdef, policy=None, extra={}, empty_translator=None): + self.setup(None, None, policy, extra, empty_translator) + self.libdef = libdef + def instrument_result(self, args): backend, ts = self.get_backend_and_type_system() if backend != 'c' or sys.platform == 'win32': @@ -275,16 +280,22 @@ annmodel.DEBUG = self.config.translation.debug annotator = translator.buildannotator(policy=policy) - - s = annotator.build_types(self.entry_point, self.inputtypes) - - self.sanity_check_annotation() - if self.standalone and s.knowntype != int: - raise Exception("stand-alone program entry point must return an " - "int (and not, e.g., None or always raise an " - "exception).") + + if self.entry_point: + s = annotator.build_types(self.entry_point, self.inputtypes) + + self.sanity_check_annotation() + if self.standalone and s.knowntype != int: + raise Exception("stand-alone program entry point must return an " + "int (and not, e.g., None or always raise an " + "exception).") + return s + else: + assert self.libdef is not None + for func, inputtypes in self.libdef.functions: + annotator.build_types(func, inputtypes) + self.sanity_check_annotation() annotator.simplify() - return s # task_annotate = taskdef(task_annotate, [], "Annotating&simplifying") @@ -619,9 +630,16 @@ from pypy.translator.cli.gencli import GenCli from pypy.translator.cli.entrypoint import get_entrypoint - entry_point_graph = self.translator.graphs[0] - self.gen = GenCli(udir, self.translator, get_entrypoint(entry_point_graph), - config=self.config) + if self.entry_point is not None: # executable mode + entry_point_graph = self.translator.graphs[0] + entry_point = get_entrypoint(entry_point_graph) + else: + # library mode + assert self.libdef is not None + bk = self.translator.annotator.bookkeeper + entry_point = self.libdef.get_entrypoint(bk) + + self.gen = GenCli(udir, self.translator, entry_point, config=self.config) filename = self.gen.generate_source() self.log.info("Wrote %s" % (filename,)) task_source_cli = taskdef(task_source_cli, ["?" + OOBACKENDOPT, OOTYPE], @@ -667,6 +685,15 @@ f.close() os.chmod(newexename, 0755) + def copy_cli_dll(self): + import os.path + import shutil + dllname = self.gen.outfile + usession_path, dll_name = os.path.split(dllname) + pypylib_dll = os.path.join(usession_path, 'pypylib.dll') + shutil.copy(dllname, '.') + shutil.copy(pypylib_dll, '.') + def task_run_cli(self): pass task_run_cli = taskdef(task_run_cli, ['compile_cli'], From antocuni at codespeak.net Wed May 2 23:56:39 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 2 May 2007 23:56:39 +0200 (CEST) Subject: [pypy-svn] r42601 - pypy/dist/pypy/translator Message-ID: <20070502215639.4C25B8080@code0.codespeak.net> Author: antocuni Date: Wed May 2 23:56:38 2007 New Revision: 42601 Modified: pypy/dist/pypy/translator/driver.py Log: oops! Modified: pypy/dist/pypy/translator/driver.py ============================================================================== --- pypy/dist/pypy/translator/driver.py (original) +++ pypy/dist/pypy/translator/driver.py Wed May 2 23:56:38 2007 @@ -289,13 +289,14 @@ raise Exception("stand-alone program entry point must return an " "int (and not, e.g., None or always raise an " "exception).") + annotator.simplify() return s else: assert self.libdef is not None for func, inputtypes in self.libdef.functions: annotator.build_types(func, inputtypes) self.sanity_check_annotation() - annotator.simplify() + annotator.simplify() # task_annotate = taskdef(task_annotate, [], "Annotating&simplifying") From arigo at codespeak.net Thu May 3 10:47:31 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 3 May 2007 10:47:31 +0200 (CEST) Subject: [pypy-svn] r42610 - pypy/extradoc/talk/badhonnef2007 Message-ID: <20070503084731.2BDCD807C@code0.codespeak.net> Author: arigo Date: Thu May 3 10:47:30 2007 New Revision: 42610 Modified: pypy/extradoc/talk/badhonnef2007/talk.tex Log: - repetition - typo Otherwise the talk looks very good to me. Modified: pypy/extradoc/talk/badhonnef2007/talk.tex ============================================================================== --- pypy/extradoc/talk/badhonnef2007/talk.tex (original) +++ pypy/extradoc/talk/badhonnef2007/talk.tex Thu May 3 10:47:30 2007 @@ -284,7 +284,7 @@ \end{itemize} \item Insert low-level aspects into the code as required by - the target (Object layout, memory management) + the target \begin{itemize} \item object layout \item memory management @@ -376,7 +376,7 @@ \item 700 of those are for builtins \item - after translation to C: 14000 line of C code + after translation to C: 14000 lines of C code \item part of the PyPy distribution at: \texttt{http://codespeak.net/pypy} From antocuni at codespeak.net Thu May 3 11:08:11 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 3 May 2007 11:08:11 +0200 (CEST) Subject: [pypy-svn] r42613 - pypy/dist/pypy/translator/cli/test Message-ID: <20070503090811.2A3CC807C@code0.codespeak.net> Author: antocuni Date: Thu May 3 11:08:10 2007 New Revision: 42613 Modified: pypy/dist/pypy/translator/cli/test/test_backendopt.py Log: typo Modified: pypy/dist/pypy/translator/cli/test/test_backendopt.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_backendopt.py (original) +++ pypy/dist/pypy/translator/cli/test/test_backendopt.py Thu May 3 11:08:10 2007 @@ -4,15 +4,11 @@ TestTypedOptimizedSwitchTestCase as c_TestTypedOptimizedSwitchTestCase class CTestCompat: - backend_opt = { - 'merge_if_blocks': True - } - def CodeGenerator(self): return self def getcompiled(self, fn, annotation): - return compile_function(fn, annotation, backend_opt=self.backend_opt) + return compile_function(fn, annotation, backendopt=True) class TestOptimizedSwitchTestCase(CTestCompat, c_TestTypedOptimizedSwitchTestCase): From afa at codespeak.net Thu May 3 22:38:23 2007 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 3 May 2007 22:38:23 +0200 (CEST) Subject: [pypy-svn] r42648 - pypy/dist/pypy/module/select/test Message-ID: <20070503203823.EF5C38075@code0.codespeak.net> Author: afa Date: Thu May 3 22:38:23 2007 New Revision: 42648 Modified: pypy/dist/pypy/module/select/test/test_select.py Log: Refactor test_select.py, to allow multiple kinds of selectable objects: - pipes created with os.pipe() - connected sockets, but without using socketpair(), to have a chance to make it work on win32. Modified: pypy/dist/pypy/module/select/test/test_select.py ============================================================================== --- pypy/dist/pypy/module/select/test/test_select.py (original) +++ pypy/dist/pypy/module/select/test/test_select.py Thu May 3 22:38:23 2007 @@ -1,14 +1,7 @@ import py, sys from pypy.conftest import gettestobjspace -class AppTestSelect: - def setup_class(cls): - if sys.platform == 'win': - py.test.skip("select() doesn't work with pipes, " - "we would need tests using sockets") - space = gettestobjspace(usemodules=('select',)) - cls.space = space - +class _AppTestSelect: def test_sleep(self): import time, select start = time.time() @@ -18,22 +11,22 @@ assert end - start > 0.25 def test_readable(self): - import os, select - readend, writeend = os.pipe() + import select + readend, writeend = getpair() try: iwtd, owtd, ewtd = select.select([readend], [], [], 0) assert iwtd == owtd == ewtd == [] - os.write(writeend, 'X') + writeend.send('X') iwtd, owtd, ewtd = select.select([readend], [], []) assert iwtd == [readend] assert owtd == ewtd == [] finally: - os.close(writeend) - os.close(readend) + writeend.close() + readend.close() def test_write_read(self): - import os, select - readend, writeend = os.pipe() + import select + readend, writeend = getpair() try: total_out = 0 while True: @@ -42,7 +35,7 @@ if owtd == []: break assert owtd == [writeend] - total_out += os.write(writeend, 'x' * 512) + total_out += writeend.send('x' * 512) total_in = 0 while True: iwtd, owtd, ewtd = select.select([readend], [], [], 0) @@ -50,45 +43,45 @@ if iwtd == []: break assert iwtd == [readend] - data = os.read(readend, 4096) + data = readend.recv(4096) assert len(data) > 0 assert data == 'x' * len(data) total_in += len(data) assert total_in == total_out finally: - os.close(writeend) - os.close(readend) + writeend.close() + readend.close() def test_close(self): - import os, select - readend, writeend = os.pipe() + import select + readend, writeend = getpair() try: try: - total_out = os.write(writeend, 'x' * 512) + total_out = writeend.send('x' * 512) finally: - os.close(writeend) + writeend.close() assert 1 <= total_out <= 512 total_in = 0 while True: iwtd, owtd, ewtd = select.select([readend], [], []) assert iwtd == [readend] assert owtd == ewtd == [] - data = os.read(readend, 4096) + data = readend.recv(4096) if len(data) == 0: break assert data == 'x' * len(data) total_in += len(data) assert total_in == total_out finally: - os.close(readend) + readend.close() def test_read_many(self): - import os, select + import select readends = [] writeends = [] try: for i in range(10): - fd1, fd2 = os.pipe() + fd1, fd2 = getpair() readends.append(fd1) writeends.append(fd2) iwtd, owtd, ewtd = select.select(readends, [], [], 0) @@ -96,24 +89,91 @@ for i in range(50): n = (i*3) % 10 - os.write(writeends[n], 'X') + writeends[n].send('X') iwtd, owtd, ewtd = select.select(readends, [], []) assert iwtd == [readends[n]] assert owtd == ewtd == [] - data = os.read(readends[n], 1) + data = readends[n].recv(1) assert data == 'X' finally: for fd in readends + writeends: - os.close(fd) + fd.close() def test_read_end_closed(self): - import os, select - readend, writeend = os.pipe() - os.close(readend) + import select + readend, writeend = getpair() + readend.close() try: iwtd, owtd, ewtd = select.select([], [writeend], []) assert owtd == [writeend] assert iwtd == ewtd == [] finally: - os.close(writeend) + writeend.close() + +class AppTestSelectWithPipes(_AppTestSelect): + "Use a pipe to get pairs of file descriptors" + def setup_class(cls): + if sys.platform == 'win32': + py.test.skip("select() doesn't work with pipes on win32") + space = gettestobjspace(usemodules=('select',)) + cls.space = space + + # Wraps a file descriptor in an socket-like object + space.exec_('''if 1: + import os + class FileAsSocket: + def __init__(self, fd): + self.fd = fd + def fileno(self): + return self.fd + def send(self, data): + return os.write(self.fd, data) + def recv(self, length): + return os.read(self.fd, length) + def close(self): + return os.close(self.fd) + def getpair(): + s1, s2 = os.pipe() + return FileAsSocket(s1), FileAsSocket(s2)''', + space.builtin.w_dict, space.builtin.w_dict) + +class AppTestSelectWithSockets(_AppTestSelect): + """Same tests with connected sockets. + socket.socketpair() does not exists on win32, + so we start our own server.""" + def setup_class(cls): + space = gettestobjspace(usemodules=('select',)) + cls.space = space + + space.setitem(space.builtin.w_dict, space.wrap('getpair'), + space.wrap(cls.getsocketpair)) + + import socket + cls.sock = socket.socket() + + try_ports = [1023] + range(20000, 30000, 437) + for port in try_ports: + print 'binding to port %d:' % (port,), + cls.sockaddress = ('127.0.0.1', port) + try: + cls.sock.bind(cls.sockaddress) + print 'works' + break + except socket.error, e: # should get a "Permission denied" + print e + else: + raise e + + @classmethod + def getsocketpair(cls): + """Helper method which returns a pair of connected sockets. + Note that they become faked objects at AppLevel""" + import thread, socket + + cls.sock.listen(1) + s2 = socket.socket() + thread.start_new_thread(s2.connect, (cls.sockaddress,)) + s1, addr2 = cls.sock.accept() + + return s1, s2 From afa at codespeak.net Thu May 3 23:17:35 2007 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 3 May 2007 23:17:35 +0200 (CEST) Subject: [pypy-svn] r42649 - in pypy/dist/pypy: module/select rlib Message-ID: <20070503211735.2E5128075@code0.codespeak.net> Author: afa Date: Thu May 3 23:17:33 2007 New Revision: 42649 Removed: pypy/dist/pypy/module/select/ctypes_select.py Modified: pypy/dist/pypy/module/select/__init__.py pypy/dist/pypy/module/select/interp_select.py pypy/dist/pypy/rlib/_rsocket_ctypes.py Log: Remove ctypes_select in favor of _rsocket_ctypes. Most of select structs and functions were actually already used by rsocket, and it will be easier to port to win32. Next step: work on Windows to write a poll() replacement. Modified: pypy/dist/pypy/module/select/__init__.py ============================================================================== --- pypy/dist/pypy/module/select/__init__.py (original) +++ pypy/dist/pypy/module/select/__init__.py Thu May 3 23:17:33 2007 @@ -13,9 +13,15 @@ } def buildloaders(cls): - from pypy.module.select import ctypes_select as _c - for constant, value in _c.constants.iteritems(): - Module.interpleveldefs[constant] = "space.wrap(%r)" % value + constantnames = ''' + POLLIN POLLPRI POLLOUT POLLERR POLLHUP POLLNVAL + POLLRDNORM POLLRDBAND POLLWRNORM POLLWEBAND POLLMSG'''.split() + + from pypy.rlib._rsocket_ctypes import constants + for name in constantnames: + if name in constants: + value = constants[name] + Module.interpleveldefs[name] = "space.wrap(%r)" % value super(Module, cls).buildloaders() buildloaders = classmethod(buildloaders) Modified: pypy/dist/pypy/module/select/interp_select.py ============================================================================== --- pypy/dist/pypy/module/select/interp_select.py (original) +++ pypy/dist/pypy/module/select/interp_select.py Thu May 3 23:17:33 2007 @@ -1,7 +1,7 @@ from pypy.interpreter.typedef import TypeDef from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.gateway import W_Root, ObjSpace, interp2app -from pypy.module.select import ctypes_select as _c +from pypy.rlib import _rsocket_ctypes as _c from pypy.rpython.rctypes.aerrno import geterrno from pypy.interpreter.error import OperationError Modified: pypy/dist/pypy/rlib/_rsocket_ctypes.py ============================================================================== --- pypy/dist/pypy/rlib/_rsocket_ctypes.py (original) +++ pypy/dist/pypy/rlib/_rsocket_ctypes.py Thu May 3 23:17:33 2007 @@ -22,6 +22,7 @@ 'sys/socket.h', 'sys/un.h', 'sys/poll.h', + 'sys/select.h', 'netinet/in.h', 'netinet/tcp.h', 'unistd.h', @@ -64,8 +65,6 @@ INVALID_SOCKET = ctypes_platform.DefinedConstantInteger('INVALID_SOCKET') INET_ADDRSTRLEN = ctypes_platform.DefinedConstantInteger('INET_ADDRSTRLEN') INET6_ADDRSTRLEN= ctypes_platform.DefinedConstantInteger('INET6_ADDRSTRLEN') - POLLIN = ctypes_platform.DefinedConstantInteger('POLLIN') - POLLOUT = ctypes_platform.DefinedConstantInteger('POLLOUT') EINPROGRESS = ctypes_platform.DefinedConstantInteger('EINPROGRESS') WSAEINPROGRESS = ctypes_platform.DefinedConstantInteger('WSAEINPROGRESS') EWOULDBLOCK = ctypes_platform.DefinedConstantInteger('EWOULDBLOCK') @@ -134,6 +133,9 @@ TCP_LINGER2 TCP_MAXSEG TCP_NODELAY TCP_QUICKACK TCP_SYNCNT TCP_WINDOW_CLAMP IPX_TYPE + +POLLIN POLLPRI POLLOUT POLLERR POLLHUP POLLNVAL +POLLRDNORM POLLRDBAND POLLWRNORM POLLWEBAND POLLMSG '''.split() for name in constant_names: @@ -313,8 +315,6 @@ FIONBIO = cConfig.FIONBIO INET_ADDRSTRLEN = cConfig.INET_ADDRSTRLEN INET6_ADDRSTRLEN = cConfig.INET6_ADDRSTRLEN -POLLIN = cConfig.POLLIN -POLLOUT = cConfig.POLLOUT EINPROGRESS = cConfig.EINPROGRESS or cConfig.WSAEINPROGRESS EWOULDBLOCK = cConfig.EWOULDBLOCK or cConfig.WSAEWOULDBLOCK EAFNOSUPPORT = cConfig.EAFNOSUPPORT or cConfig.WSAEAFNOSUPPORT From afa at codespeak.net Thu May 3 23:56:27 2007 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 3 May 2007 23:56:27 +0200 (CEST) Subject: [pypy-svn] r42651 - in pypy/dist/pypy: module/select module/select/test rlib Message-ID: <20070503215627.467FF8075@code0.codespeak.net> Author: afa Date: Thu May 3 23:56:26 2007 New Revision: 42651 Modified: pypy/dist/pypy/module/select/interp_select.py pypy/dist/pypy/module/select/test/test_select.py pypy/dist/pypy/rlib/_rsocket_ctypes.py Log: win32 implementation of poll(). Had to change some tests: - select.select() requires at least one socket, even on CPython - different behaviour when closing a socket with unread data. Modified: pypy/dist/pypy/module/select/interp_select.py ============================================================================== --- pypy/dist/pypy/module/select/interp_select.py (original) +++ pypy/dist/pypy/module/select/interp_select.py Thu May 3 23:56:26 2007 @@ -2,6 +2,7 @@ from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.gateway import W_Root, ObjSpace, interp2app from pypy.rlib import _rsocket_ctypes as _c +from ctypes import POINTER, byref from pypy.rpython.rctypes.aerrno import geterrno from pypy.interpreter.error import OperationError @@ -50,43 +51,136 @@ space.wrap(fd)) unregister.unwrap_spec = ['self', ObjSpace, W_Root] - def poll(self, space, w_timeout=None): - if space.is_w(w_timeout, space.w_None): - timeout = -1 - else: - timeout = space.int_w(w_timeout) - - numfd = len(self.fddict) - buf = _c.create_string_buffer(_c.sizeof(_c.pollfd) * numfd) - pollfds = _c.cast(buf, _c.POINTER(_c.pollfd)) - i = 0 - for fd, events in self.fddict.iteritems(): - pollfds[i].fd = fd - pollfds[i].events = events - i += 1 - - # XXX Temporary hack for releasing the GIL - GIL = space.threadlocals.getGIL() - if GIL is not None: GIL.release() - ret = _c.poll(pollfds, numfd, timeout) - if GIL is not None: GIL.acquire(True) - - if ret < 0: - errno = geterrno() - w_module = space.getbuiltinmodule('select') - w_errortype = space.getattr(w_module, space.wrap('error')) - message = _c.strerror(errno) - raise OperationError(w_errortype, - space.newtuple([space.wrap(errno), - space.wrap(message)])) - - retval_w = [] - for i in range(numfd): - pollfd = pollfds[i] - if pollfd.revents: - retval_w.append(space.newtuple([space.wrap(pollfd.fd), - space.wrap(pollfd.revents)])) - return space.newlist(retval_w) + if hasattr(_c, 'poll'): + def poll(self, space, w_timeout=None): + if space.is_w(w_timeout, space.w_None): + timeout = -1 + else: + timeout = space.int_w(w_timeout) + + numfd = len(self.fddict) + buf = _c.create_string_buffer(_c.sizeof(_c.pollfd) * numfd) + pollfds = _c.cast(buf, POINTER(_c.pollfd)) + i = 0 + for fd, events in self.fddict.iteritems(): + pollfds[i].fd = fd + pollfds[i].events = events + i += 1 + + # XXX Temporary hack for releasing the GIL + GIL = space.threadlocals.getGIL() + if GIL is not None: GIL.release() + ret = _c.poll(pollfds, numfd, timeout) + if GIL is not None: GIL.acquire(True) + + if ret < 0: + errno = geterrno() + w_module = space.getbuiltinmodule('select') + w_errortype = space.getattr(w_module, space.wrap('error')) + message = _c.strerror(errno) + raise OperationError(w_errortype, + space.newtuple([space.wrap(errno), + space.wrap(message)])) + + retval_w = [] + for i in range(numfd): + pollfd = pollfds[i] + if pollfd.revents: + retval_w.append(space.newtuple([space.wrap(pollfd.fd), + space.wrap(pollfd.revents)])) + return space.newlist(retval_w) + + elif hasattr(_c, 'WSAEventSelect'): + # win32 implementation + def poll(self, space, w_timeout=None): + numfd = len(self.fddict) + + socketevents = _c.ARRAY(_c.WSAEVENT, numfd)() + + numevents = 0 + eventdict = {} + + for fd, events in self.fddict.iteritems(): + # select desired events + wsaEvents = 0 + if events & _c.POLLIN: + wsaEvents |= _c.FD_READ | _c.FD_ACCEPT | _c.FD_CLOSE + if events & _c.POLLOUT: + wsaEvents |= _c.FD_WRITE | _c.FD_CONNECT | _c.FD_CLOSE + + # if no events then ignore socket + if wsaEvents == 0: + continue + + # select socket for desired events + event = _c.WSACreateEvent() + _c.WSAEventSelect(fd, event, wsaEvents) + + eventdict[fd] = event + socketevents[numevents] = event + numevents += 1 + + # if no sockets then return immediately + if numevents == 0: + return space.newlist([]) + + # prepare timeout + if space.is_w(w_timeout, space.w_None): + timeout = -1 + else: + timeout = space.int_w(w_timeout) + if timeout < 0: + timeout = _c.INFINITE + + # XXX Temporary hack for releasing the GIL + GIL = space.threadlocals.getGIL() + if GIL is not None: GIL.release() + ret = _c.WSAWaitForMultipleEvents(numevents, socketevents, + False, timeout, False) + if GIL is not None: GIL.acquire(True) + + if ret == _c.WSA_WAIT_TIMEOUT: + return space.newlist([]) + + if ret < 0: # WSA_WAIT_FAILED is unsigned... + from pypy.rlib._rsocket_ctypes import socket_strerror, geterrno + errno = geterrno() + w_module = space.getbuiltinmodule('select') + w_errortype = space.getattr(w_module, space.wrap('error')) + message = socket_strerror(errno) + raise OperationError(w_errortype, + space.newtuple([space.wrap(errno), + space.wrap(message)])) + + retval_w = [] + info = _c.WSANETWORKEVENTS() + for fd, event in eventdict.iteritems(): + if _c.WSAEnumNetworkEvents(fd, event, byref(info)) < 0: + continue + revents = 0 + if info.lNetworkEvents & _c.FD_READ: + revents |= _c.POLLIN + if info.lNetworkEvents & _c.FD_ACCEPT: + revents |= _c.POLLIN + if info.lNetworkEvents & _c.FD_WRITE: + revents |= _c.POLLOUT + if info.lNetworkEvents & _c.FD_CONNECT: + if info.iErrorCode[_c.FD_CONNECT_BIT]: + revents |= _c.POLLERR + else: + revents |= _c.POLLOUT + if info.lNetworkEvents & _c.FD_CLOSE: + if info.iErrorCode[_c.FD_CLOSE_BIT]: + revents |= _c.POLLERR + else: + if self.fddict[fd] & _c.POLLIN: + revents |= _c.POLLIN + if self.fddict[fd] & _c.POLLOUT: + revents |= _c.POLLOUT + if revents: + retval_w.append(space.newtuple([space.wrap(fd), + space.wrap(revents)])) + return space.newlist(retval_w) poll.unwrap_spec = ['self', ObjSpace, W_Root] pollmethods = {} Modified: pypy/dist/pypy/module/select/test/test_select.py ============================================================================== --- pypy/dist/pypy/module/select/test/test_select.py (original) +++ pypy/dist/pypy/module/select/test/test_select.py Thu May 3 23:56:26 2007 @@ -4,8 +4,9 @@ class _AppTestSelect: def test_sleep(self): import time, select + readend, writeend = getpair() start = time.time() - iwtd, owtd, ewtd = select.select([], [], [], 0.3) + iwtd, owtd, ewtd = select.select([readend], [], [], 0.3) end = time.time() assert iwtd == owtd == ewtd == [] assert end - start > 0.25 @@ -53,13 +54,16 @@ readend.close() def test_close(self): - import select + import select, sys readend, writeend = getpair() try: try: total_out = writeend.send('x' * 512) finally: - writeend.close() + # win32 sends the 'closed' event immediately, even when + # more data is available + if sys.platform != 'win32': + writeend.close() assert 1 <= total_out <= 512 total_in = 0 while True: @@ -71,6 +75,9 @@ break assert data == 'x' * len(data) total_in += len(data) + # win32: check that closing the socket exits the loop + if sys.platform == 'win32' and total_in == total_out: + writeend.close() assert total_in == total_out finally: readend.close() Modified: pypy/dist/pypy/rlib/_rsocket_ctypes.py ============================================================================== --- pypy/dist/pypy/rlib/_rsocket_ctypes.py (original) +++ pypy/dist/pypy/rlib/_rsocket_ctypes.py Thu May 3 23:56:26 2007 @@ -136,6 +136,10 @@ POLLIN POLLPRI POLLOUT POLLERR POLLHUP POLLNVAL POLLRDNORM POLLRDBAND POLLWRNORM POLLWEBAND POLLMSG + +FD_READ FD_WRITE FD_ACCEPT FD_CONNECT FD_CLOSE +WSA_WAIT_TIMEOUT WSA_WAIT_FAILED INFINITE +FD_CONNECT_BIT FD_CLOSE_BIT '''.split() for name in constant_names: @@ -164,7 +168,13 @@ ('INADDR_NONE', 0xffffffff), ('SHUT_RD', 0), ('SHUT_WR', 1), - ('SHUT_RDWR', 2)] + ('SHUT_RDWR', 2), + ('POLLIN', 1), + ('POLLPRI', 2), + ('POLLOUT', 4), + ('POLLERR', 8), + ('POLLHUP', 16), + ] for name, default in constants_w_defaults: setattr(CConfig, name, ctypes_platform.DefinedConstantInteger(name)) @@ -249,6 +259,13 @@ [('fd', socketfd_type), ('events', c_short), ('revents', c_short)]) +if _MS_WINDOWS: + CConfig.WSAEVENT = ctypes_platform.SimpleType('WSAEVENT', c_void_p) + CConfig.WSANETWORKEVENTS = ctypes_platform.Struct('WSANETWORKEVENTS', + [('lNetworkEvents', c_long), + ('iErrorCode', c_int * 10), #FD_MAX_EVENTS + ]) + CConfig.timeval = ctypes_platform.Struct('struct timeval', [('tv_sec', c_long), @@ -351,6 +368,9 @@ if _POSIX: nfds_t = cConfig.nfds_t pollfd = cConfig.pollfd +if MS_WINDOWS: + WSAEVENT = cConfig.WSAEVENT + WSANETWORKEVENTS = cConfig.WSANETWORKEVENTS timeval = cConfig.timeval if MS_WINDOWS: fd_set = cConfig.fd_set @@ -363,7 +383,7 @@ # functions if MS_WINDOWS: from ctypes import windll - dllname = util.find_library('wsock32') + dllname = util.find_library('ws2_32') assert dllname is not None socketdll = windll.LoadLibrary(dllname) else: @@ -573,6 +593,24 @@ POINTER(timeval)] select.restype = c_int + WSACreateEvent = socketdll.WSACreateEvent + WSACreateEvent.argtypes = [] + WSACreateEvent.restype = WSAEVENT + + WSAEventSelect = socketdll.WSAEventSelect + WSAEventSelect.argtypes = [socketfd_type, WSAEVENT, c_long] + WSAEventSelect.restype = c_int + + WSAWaitForMultipleEvents = socketdll.WSAWaitForMultipleEvents + WSAWaitForMultipleEvents.argtypes = [c_long, POINTER(WSAEVENT), + c_int, c_long, c_int] + WSAWaitForMultipleEvents.restype = c_long + + WSAEnumNetworkEvents = socketdll.WSAEnumNetworkEvents + WSAEnumNetworkEvents.argtypes = [socketfd_type, WSAEVENT, + POINTER(WSANETWORKEVENTS)] + WSAEnumNetworkEvents.restype = c_int + if MS_WINDOWS: WSAData = cConfig.WSAData WSAStartup = socketdll.WSAStartup From afa at codespeak.net Fri May 4 00:01:22 2007 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 4 May 2007 00:01:22 +0200 (CEST) Subject: [pypy-svn] r42652 - in pypy/dist/pypy: module/select rlib Message-ID: <20070503220122.ED2CB8075@code0.codespeak.net> Author: afa Date: Fri May 4 00:01:21 2007 New Revision: 42652 Modified: pypy/dist/pypy/module/select/interp_select.py pypy/dist/pypy/rlib/_rsocket_ctypes.py Log: Close Events handle after use Modified: pypy/dist/pypy/module/select/interp_select.py ============================================================================== --- pypy/dist/pypy/module/select/interp_select.py (original) +++ pypy/dist/pypy/module/select/interp_select.py Fri May 4 00:01:21 2007 @@ -180,6 +180,9 @@ if revents: retval_w.append(space.newtuple([space.wrap(fd), space.wrap(revents)])) + + _c.WSACloseEvent(event) + return space.newlist(retval_w) poll.unwrap_spec = ['self', ObjSpace, W_Root] Modified: pypy/dist/pypy/rlib/_rsocket_ctypes.py ============================================================================== --- pypy/dist/pypy/rlib/_rsocket_ctypes.py (original) +++ pypy/dist/pypy/rlib/_rsocket_ctypes.py Fri May 4 00:01:21 2007 @@ -597,6 +597,10 @@ WSACreateEvent.argtypes = [] WSACreateEvent.restype = WSAEVENT + WSACloseEvent = socketdll.WSACloseEvent + WSACloseEvent.argtypes = [WSAEVENT] + WSACloseEvent.restype = c_int + WSAEventSelect = socketdll.WSAEventSelect WSAEventSelect.argtypes = [socketfd_type, WSAEVENT, c_long] WSAEventSelect.restype = c_int From santagada at codespeak.net Fri May 4 00:37:11 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Fri, 4 May 2007 00:37:11 +0200 (CEST) Subject: [pypy-svn] r42653 - in pypy/dist/pypy/lang/js: . test Message-ID: <20070503223711.76D8A8075@code0.codespeak.net> Author: santagada Date: Fri May 4 00:37:10 2007 New Revision: 42653 Modified: pypy/dist/pypy/lang/js/jsgrammar.txt pypy/dist/pypy/lang/js/newparser.py pypy/dist/pypy/lang/js/test/test_new_parser.py Log: lots of new statement tests and a function declaration tests Modified: pypy/dist/pypy/lang/js/jsgrammar.txt ============================================================================== --- pypy/dist/pypy/lang/js/jsgrammar.txt (original) +++ pypy/dist/pypy/lang/js/jsgrammar.txt Fri May 4 00:37:10 2007 @@ -5,31 +5,31 @@ program : sourceelements EOF ; -sourceelements : sourceelement sourceelements +sourceelements : sourceelement >sourceelements< | sourceelement ; -sourceelement : functiondeclaration - | statement +sourceelement : + | ; -statement : +statement : [";"]? | [";"] | | [";"] - | [";"]? - | [";"] + | + | | [";"] | [";"] | [";"] | [";"] - | [";"] + | | | [";"] | [";"] ; -block : "{" statementlist? "}" +block : ["{"] statementlist? ["}"] ; statementlist : statement >statementlist< @@ -49,7 +49,7 @@ initialiser : ["="] assignmentexpression ; -emptystatement : ";" +emptystatement : [";"] ; expressionstatement : expression @@ -59,7 +59,8 @@ | ["if"] ["("] expression [")"] statement ; -iterationstatement : "do" statement ["while"] ["("] expression [")"] +iterationstatement + : "do" statement ["while"] ["("] expression [")"] | "while" ["("] expression [")"] statement | "for" ["("]expressionnoin? [";"] expression? [";"] expression? [")"] statement | "for" ["("] ["var"] variabledeclarationlistnoin [";"] expression? [";"] expression? [")"] statement @@ -172,8 +173,8 @@ elision : ","+ ; -objectliteral : "{" "}" - | "{" propertynameandvaluelist "}" +objectliteral : ["{"] ["}"] + | ["{"] propertynameandvaluelist ["}"] ; propertynameandvaluelist : hpropertynameandvaluelist* propertyname ":" assignmentexpression @@ -187,17 +188,17 @@ | ; -functiondeclaration : "function" identifier "(" formalparameterlist? ")" "{" functionbody "}" +functiondeclaration : ["function"] identifier ["("] formalparameterlist? [")"] ["{"] functionbody ["}"] ; -functionexpression : "function" identifier? "(" formalparameterlist? ")" "{" functionbody "}" +functionexpression : ["function"] identifier? ["("] formalparameterlist? [")"] ["{"] functionbody ["}"] ; formalparameterlist : identifier [","] >formalparameterlist< | identifier ; -functionbody : sourceelements +functionbody : ; begmemberexpression : @@ -220,11 +221,11 @@ | memberexpression arguments ; -arguments : "(" ")" - | "(" argumentlist ")" +arguments : ["("] [")"] + | "(" ")" ; -argumentlist : assignmentexpression "," >argumentlist< +argumentlist : assignmentexpression [","] >argumentlist< | assignmentexpression; lefthandsideexpression : Modified: pypy/dist/pypy/lang/js/newparser.py ============================================================================== --- pypy/dist/pypy/lang/js/newparser.py (original) +++ pypy/dist/pypy/lang/js/newparser.py Fri May 4 00:37:10 2007 @@ -15,7 +15,7 @@ newrules = [Rule("hacked_first_symbol", [[start, "EOF"]])] + rules return newrules -parse = make_parse_function(regexs, setstartrule(rules, start="statement"), eof=True) +parse = make_parse_function(regexs, rules, eof=True) #parse = make_parse_function(regexs, rules) print rules[2].nonterminal Modified: pypy/dist/pypy/lang/js/test/test_new_parser.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_new_parser.py (original) +++ pypy/dist/pypy/lang/js/test/test_new_parser.py Fri May 4 00:37:10 2007 @@ -191,17 +191,48 @@ assert r['variablestatement'] == 1 def test_empty(self): + self.parse(";") for i in range(1,10): r = self.parse_count('{%s}'%(';'*i)) assert r['emptystatement'] == i def test_if(self): - r = self.parse_count("if(x)return;;") + r = self.parse_count("if(x)return;") assert r['ifstatement'] == 1 assert r.get('emptystatement', 0) == 0 r = self.parse_count("if(x)if(i)return;") assert r['ifstatement'] == 2 r = self.parse_count("if(x)return;else return;") assert r['ifstatement'] == 1 + + def test_iteration(self): + self.parse('for(;;);') + self.parse('for(x;;);') + self.parse('for(;x>0;);') + self.parse('for(;;x++);') + self.parse('for(var x = 1;;);') + self.parse('for(x in z);') + self.parse('for(var x in z);') + self.parse('while(1);') + self.parse('do ; while(1)') + + def test_continue_return_break(self): + self.parse('return;') + self.parse('return x+y;') + self.parse('break;') + self.parse('continue;') + self.parse('break label;') + self.parse('continue label;') + + def test_labeled(self): + self.parse('label: x+y;') + +class TestFunctionDeclaration(BaseGrammarTest): + def setup_class(cls): + cls.parse = parse_func('functiondeclaration') + + def test_simpledecl(self): + self.parse('function x () {;}') + self.parse('function z (a,b,c,d,e) {;}') From santagada at codespeak.net Fri May 4 01:31:25 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Fri, 4 May 2007 01:31:25 +0200 (CEST) Subject: [pypy-svn] r42654 - in pypy/dist/pypy/lang/js: . test Message-ID: <20070503233125.991D88076@code0.codespeak.net> Author: santagada Date: Fri May 4 01:31:24 2007 New Revision: 42654 Modified: pypy/dist/pypy/lang/js/jsgrammar.txt pypy/dist/pypy/lang/js/newparser.py pypy/dist/pypy/lang/js/test/test_new_parser.py Log: testing almost everything of the parser.... should start the trasition to this parser tomorrow Modified: pypy/dist/pypy/lang/js/jsgrammar.txt ============================================================================== --- pypy/dist/pypy/lang/js/jsgrammar.txt (original) +++ pypy/dist/pypy/lang/js/jsgrammar.txt Fri May 4 01:31:24 2007 @@ -22,11 +22,11 @@ | [";"] | [";"] | [";"] - | [";"] + | | - | + | [";"]? | [";"] - | [";"] + | ; block : ["{"] statementlist? ["}"] @@ -96,7 +96,7 @@ # CaseClauses : # CaseClause # CaseClauses CaseClause -caseclauses : caseclause+ +caseclauses : >caseclause<+ ; caseclause : ["case"] expression [":"] statementlist? @@ -174,14 +174,14 @@ ; objectliteral : ["{"] ["}"] - | ["{"] propertynameandvaluelist ["}"] + | ["{"] >propertynameandvaluelist< ["}"] ; -propertynameandvaluelist : hpropertynameandvaluelist* propertyname ":" assignmentexpression +propertynameandvaluelist : propertynameandvalue ([","] propertynameandvalue)* ; -hpropertynameandvaluelist : propertyname ":" assignmentexpression "," - ; +propertynameandvalue : propertyname [":"] assignmentexpression + ; propertyname : | Modified: pypy/dist/pypy/lang/js/newparser.py ============================================================================== --- pypy/dist/pypy/lang/js/newparser.py (original) +++ pypy/dist/pypy/lang/js/newparser.py Fri May 4 01:31:24 2007 @@ -15,8 +15,10 @@ newrules = [Rule("hacked_first_symbol", [[start, "EOF"]])] + rules return newrules -parse = make_parse_function(regexs, rules, eof=True) -#parse = make_parse_function(regexs, rules) +if len(sys.argv) == 1: + parse = make_parse_function(regexs, rules, eof=True) +else: + parse = make_parse_function(regexs, setstartrule(rules,sys.argv[1]), eof=True) print rules[2].nonterminal source = raw_input() Modified: pypy/dist/pypy/lang/js/test/test_new_parser.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_new_parser.py (original) +++ pypy/dist/pypy/lang/js/test/test_new_parser.py Fri May 4 01:31:24 2007 @@ -168,7 +168,23 @@ "2 << 4 << 4", "30 | 3 & 5", ]) - + + def test_primary(self): + self.parse('this') + self.parse('(x)') + self.parse('((((x))))') + self.parse('(x * (x * x)) + x - x') + + def test_array_literal(self): + self.parse('[1,2,3,4]') + self.parse('[1,2,]') + self.parse('[1]') + + def test_object_literal(self): + self.parse('{}') + self.parse('{x:1}') #per spec {x:1,} should not be supported + self.parse('{x:1,y:2}') + class TestStatements(BaseGrammarTest): def setup_class(cls): cls.parse = parse_func('statement') @@ -216,16 +232,41 @@ self.parse('while(1);') self.parse('do ; while(1)') - def test_continue_return_break(self): + def test_continue_return_break_throw(self): self.parse('return;') self.parse('return x+y;') self.parse('break;') self.parse('continue;') self.parse('break label;') self.parse('continue label;') + self.parse('throw (5+5);') + def test_with(self): + self.parse('with(x);') + def test_labeled(self): self.parse('label: x+y;') + + def test_switch(self): + self.parse("""switch(x) { + case 1: break; + case 2: break; + default: ; + } + + """) + self.parse("""switch(x) { + case 1: break; + case 2: break; + default: ; + case 3: break; + } + + """) + def test_try(self): + self.parse('try {x;} catch (e) {x;}') + self.parse('try {x;} catch (e) {x;} finally {x;}') + self.parse('try {x;} finally {x;}') class TestFunctionDeclaration(BaseGrammarTest): def setup_class(cls): From cfbolz at codespeak.net Fri May 4 08:52:13 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 4 May 2007 08:52:13 +0200 (CEST) Subject: [pypy-svn] r42657 - pypy/extradoc/talk/badhonnef2007 Message-ID: <20070504065213.3A5828071@code0.codespeak.net> Author: cfbolz Date: Fri May 4 08:52:10 2007 New Revision: 42657 Modified: pypy/extradoc/talk/badhonnef2007/talk.pdf pypy/extradoc/talk/badhonnef2007/talk.tex Log: some changes Modified: pypy/extradoc/talk/badhonnef2007/talk.pdf ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/badhonnef2007/talk.tex ============================================================================== --- pypy/extradoc/talk/badhonnef2007/talk.tex (original) +++ pypy/extradoc/talk/badhonnef2007/talk.tex Fri May 4 08:52:10 2007 @@ -11,7 +11,7 @@ \usetheme{Warsaw} % or ... - \setbeamercovered{transparent} + %\setbeamercovered{transparent} % or whatever (possibly just delete it) } @@ -78,11 +78,11 @@ \titlepage \end{frame} -\begin{frame} - \frametitle{Outline} - \tableofcontents +%\begin{frame} +% \frametitle{Outline} +% \tableofcontents % You might wish to add the option [pausesections] -\end{frame} +%\end{frame} % Structuring a talk is a difficult task and the following structure @@ -109,14 +109,18 @@ \begin{itemize} \item - Pyrolog is a Prolog interpreter written in RPython + Pyrolog is a Prolog interpreter written in Python + \item + want to compile it to get interesting performance + \item + translation tool-chain part of the PyPy project + \item + Python itself is too dynamic to be translatable to other languages, need a subset \item RPython (``Restricted Python'') is a subset of Python translatable to other languages \item RPython is designed to be significantly faster than regular Python - \item - translation tool-chain part of the PyPy project \end{itemize} \end{frame} @@ -128,9 +132,7 @@ \item started as a Python VM implementation in Python \item - Python itself is too dynamic to be translatable to other languages, need a subset - \item - includes a translation tool-chain for RPython + most important part: translation tool-chain for RPython \item is becoming a general environment for writing interpreters (JavaScript, Prolog started) \item @@ -159,7 +161,7 @@ \begin{frame} - \frametitle{The Python case (i)} + \frametitle{The Python case} CPython (the reference implementation) is a straightforward, portable VM. \begin{itemize} @@ -167,15 +169,15 @@ Pervasive decisions: reference counting, global lock \dots \item No dynamic compilation -% \pause \end{itemize} + \pause \begin{block}{ Extensions:} \begin{itemize} \item - \alert{Stackless} (unlimited recursion, coroutines, serializable continuations) + \alert{Stackless} (unlimited recursion, coroutines, green threads) \item - \alert{Psyco} (run-time specializer) + \alert{Psyco} (run-time specializing compiler) \item \alert{Jython}, \alert{IronPython} \end{itemize} @@ -183,28 +185,6 @@ \end{frame} -\begin{frame} - \frametitle{The Python case (ii)} - \begin{block}{Problems of Extensions:} - \begin{itemize} - \item hard to maintain: need to keep track of CPython - \item Psyco very hard to port to other hardware architectures - \item tedious to write them, Python semantics need to be re-implemented - \end{itemize} - \end{block} - \begin{itemize} - \item - The community wants Python to run everywhere: - Jython (Java), IronPython (.NET). - Lots of effort and duplication. - - \item - At various points various incompatibilities between - the implementations - - \end{itemize} -\end{frame} - \begin{frame} \frametitle{The Prolog case (i)} @@ -232,16 +212,15 @@ changing the language to experiment is hard \item often extensions to core Prolog, incompatible between each other - \item - fixed implementation decisions (GC, how to generate code, etc.) \end{itemize} \end{block} + \pause \begin{block}{implementations on CLR and JVM} \begin{itemize} \item interfacing with libraries of the platform mostly easy \item - no extensions to core Prolog (like tabling, coroutines) + no extensions to core Prolog (like tabling, coroutines, constraints) \item slow, compared to good C implementations \end{itemize} @@ -279,16 +258,19 @@ lower-level targets \begin{itemize} \item C-like - \item Java \item .NET + \item Java \end{itemize} + \pause \item Insert low-level aspects into the code as required by the target \begin{itemize} \item object layout \item memory management - \end{itemize} \item + \end{itemize} + \pause + \item Optionally insert new pervasive features not expressed in the source \begin{itemize} @@ -302,7 +284,7 @@ \begin{frame} \frametitle{Translation Aspects (i)} - Features not present in the source can be added during translation. + Features not present in the source can be added during translation: \begin{itemize} \item \alert{memory management}: use different GC strategies (Boehm collector, custom mark-n-sweep) @@ -340,15 +322,11 @@ \item uses continuation passing style inspired by BinProlog \item - Prolog calls mapped to RPython calls - \item implements large parts of the ISO standard (some builtins missing) \end{itemize} \end{frame} - - \begin{frame} \frametitle{Builtins} \begin{itemize} @@ -411,7 +389,6 @@ \begin{frame} \frametitle{Summary} - % Keep the summary *very short*. \begin{itemize} \item Very simple Prolog interpreter in RPython can compete with interpreters on the JVM, CLR @@ -436,6 +413,16 @@ \end{frame} \begin{frame} + \frametitle{Questions?} + \begin{center} + \includegraphics[scale=0.5]{image/py-web.png} + + \texttt{http://codespeak.net/pypy} + \end{center} +\end{frame} + + +\begin{frame} \frametitle{Backup slides} \dots \end{frame} From afa at codespeak.net Fri May 4 09:43:45 2007 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 4 May 2007 09:43:45 +0200 (CEST) Subject: [pypy-svn] r42661 - pypy/dist/pypy/module/select Message-ID: <20070504074345.3D0BD8075@code0.codespeak.net> Author: afa Date: Fri May 4 09:43:44 2007 New Revision: 42661 Modified: pypy/dist/pypy/module/select/interp_select.py Log: untabify file Modified: pypy/dist/pypy/module/select/interp_select.py ============================================================================== --- pypy/dist/pypy/module/select/interp_select.py (original) +++ pypy/dist/pypy/module/select/interp_select.py Fri May 4 09:43:44 2007 @@ -112,12 +112,12 @@ if wsaEvents == 0: continue - # select socket for desired events + # select socket for desired events event = _c.WSACreateEvent() _c.WSAEventSelect(fd, event, wsaEvents) eventdict[fd] = event - socketevents[numevents] = event + socketevents[numevents] = event numevents += 1 # if no sockets then return immediately From afa at codespeak.net Fri May 4 11:03:17 2007 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 4 May 2007 11:03:17 +0200 (CEST) Subject: [pypy-svn] r42663 - in pypy/dist/pypy: module/select rlib Message-ID: <20070504090317.9E51F8075@code0.codespeak.net> Author: afa Date: Fri May 4 11:03:16 2007 New Revision: 42663 Modified: pypy/dist/pypy/module/select/interp_select.py pypy/dist/pypy/rlib/_rsocket_ctypes.py Log: The select module now correctly translates on win32. It even seem to work. Modified: pypy/dist/pypy/module/select/interp_select.py ============================================================================== --- pypy/dist/pypy/module/select/interp_select.py (original) +++ pypy/dist/pypy/module/select/interp_select.py Fri May 4 11:03:16 2007 @@ -143,11 +143,10 @@ return space.newlist([]) if ret < 0: # WSA_WAIT_FAILED is unsigned... - from pypy.rlib._rsocket_ctypes import socket_strerror, geterrno - errno = geterrno() + errno = _c.geterrno() w_module = space.getbuiltinmodule('select') w_errortype = space.getattr(w_module, space.wrap('error')) - message = socket_strerror(errno) + message = _c.socket_strerror(errno) raise OperationError(w_errortype, space.newtuple([space.wrap(errno), space.wrap(message)])) Modified: pypy/dist/pypy/rlib/_rsocket_ctypes.py ============================================================================== --- pypy/dist/pypy/rlib/_rsocket_ctypes.py (original) +++ pypy/dist/pypy/rlib/_rsocket_ctypes.py Fri May 4 11:03:16 2007 @@ -140,6 +140,8 @@ FD_READ FD_WRITE FD_ACCEPT FD_CONNECT FD_CLOSE WSA_WAIT_TIMEOUT WSA_WAIT_FAILED INFINITE FD_CONNECT_BIT FD_CLOSE_BIT +WSA_IO_PENDING WSA_IO_INCOMPLETE WSA_INVALID_HANDLE +WSA_INVALID_PARAMETER WSA_NOT_ENOUGH_MEMORY WSA_OPERATION_ABORTED '''.split() for name in constant_names: @@ -261,10 +263,11 @@ ('revents', c_short)]) if _MS_WINDOWS: CConfig.WSAEVENT = ctypes_platform.SimpleType('WSAEVENT', c_void_p) - CConfig.WSANETWORKEVENTS = ctypes_platform.Struct('WSANETWORKEVENTS', - [('lNetworkEvents', c_long), - ('iErrorCode', c_int * 10), #FD_MAX_EVENTS - ]) + CConfig.WSANETWORKEVENTS = ctypes_platform.Struct( + 'struct _WSANETWORKEVENTS', + [('lNetworkEvents', c_long), + ('iErrorCode', c_int * 10), #FD_MAX_EVENTS + ]) CConfig.timeval = ctypes_platform.Struct('struct timeval', @@ -685,8 +688,18 @@ # errno.WSANO_RECOVERY: "Unexpected server error encountered", # errno.WSANO_DATA: "Valid name without requested data", # errno.WSANO_ADDRESS: "No address, look for MX record", + + # select() errors + WSA_IO_PENDING: "WSA_IO_PENDING", + WSA_IO_INCOMPLETE: "WSA_IO_INCOMPLETE", + WSA_INVALID_HANDLE: "WSA_INVALID_HANDLE", + WSA_INVALID_PARAMETER: "WSA_INVALID_PARAMETER", + WSA_NOT_ENOUGH_MEMORY: "WSA_NOT_ENOUGH_MEMORY", + WSA_OPERATION_ABORTED: "WSA_OPERATION_ABORTED", } + assert len(WIN32_ERROR_MESSAGES) == 53 # detect duplicates + def socket_strerror(errno): return WIN32_ERROR_MESSAGES.get(errno, "winsock error %d" % errno) else: From afa at codespeak.net Fri May 4 11:13:39 2007 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 4 May 2007 11:13:39 +0200 (CEST) Subject: [pypy-svn] r42664 - in pypy/dist/pypy: config lib/distributed/test module/_socket module/rsocket Message-ID: <20070504091339.3E26E806B@code0.codespeak.net> Author: afa Date: Fri May 4 11:13:38 2007 New Revision: 42664 Added: pypy/dist/pypy/module/_socket/ - copied from r42658, pypy/dist/pypy/module/rsocket/ Removed: pypy/dist/pypy/module/rsocket/ Modified: pypy/dist/pypy/config/pypyoption.py pypy/dist/pypy/lib/distributed/test/test_socklayer.py Log: Move module/rsocket to module/_socket Modified: pypy/dist/pypy/config/pypyoption.py ============================================================================== --- pypy/dist/pypy/config/pypyoption.py (original) +++ pypy/dist/pypy/config/pypyoption.py Fri May 4 11:13:38 2007 @@ -22,13 +22,13 @@ working_modules = default_modules.copy() working_modules.update(dict.fromkeys( - ["rsocket", "unicodedata", "mmap", "fcntl", "rctime", "select", + ["_socket", "unicodedata", "mmap", "fcntl", "rctime", "select", "crypt", "signal", "dyngram", "readline", ] )) if sys.platform == "win32": - del working_modules["rsocket"] + del working_modules["_socket"] del working_modules["fcntl"] del working_modules["select"] del working_modules["readline"] Modified: pypy/dist/pypy/lib/distributed/test/test_socklayer.py ============================================================================== --- pypy/dist/pypy/lib/distributed/test/test_socklayer.py (original) +++ pypy/dist/pypy/lib/distributed/test/test_socklayer.py Fri May 4 11:13:38 2007 @@ -6,7 +6,7 @@ class AppTestSocklayer: def setup_class(cls): cls.space = gettestobjspace(**{"objspace.std.withtproxy": True, - "usemodules":("_stackless","rsocket", "select")}) + "usemodules":("_stackless","_socket", "select")}) def test_socklayer(self): class X: From afa at codespeak.net Fri May 4 11:18:25 2007 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 4 May 2007 11:18:25 +0200 (CEST) Subject: [pypy-svn] r42665 - in pypy/dist/pypy/module/_socket: . test Message-ID: <20070504091825.24E6F8075@code0.codespeak.net> Author: afa Date: Fri May 4 11:18:24 2007 New Revision: 42665 Modified: pypy/dist/pypy/module/_socket/interp_func.py pypy/dist/pypy/module/_socket/test/test_sock_app.py Log: Step 2 of rsocket -> _socket rename: update imports in moved files Modified: pypy/dist/pypy/module/_socket/interp_func.py ============================================================================== --- pypy/dist/pypy/module/_socket/interp_func.py (original) +++ pypy/dist/pypy/module/_socket/interp_func.py Fri May 4 11:18:24 2007 @@ -1,5 +1,5 @@ from pypy.interpreter.gateway import ObjSpace, W_Root, NoneNotWrapped -from pypy.module.rsocket.interp_socket import converted_error, W_RSocket +from pypy.module._socket.interp_socket import converted_error, W_RSocket from pypy.rlib import rsocket from pypy.rlib.rsocket import SocketError from pypy.interpreter.error import OperationError Modified: pypy/dist/pypy/module/_socket/test/test_sock_app.py ============================================================================== --- pypy/dist/pypy/module/_socket/test/test_sock_app.py (original) +++ pypy/dist/pypy/module/_socket/test/test_sock_app.py Fri May 4 11:18:24 2007 @@ -3,7 +3,7 @@ import py def setup_module(mod): - mod.space = gettestobjspace(usemodules=['rsocket']) + mod.space = gettestobjspace(usemodules=['_socket']) global socket import socket mod.w_socket = space.appexec([], "(): import _socket as m; return m") From afa at codespeak.net Fri May 4 11:26:11 2007 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 4 May 2007 11:26:11 +0200 (CEST) Subject: [pypy-svn] r42666 - pypy/dist/pypy/doc/config Message-ID: <20070504092611.505E48075@code0.codespeak.net> Author: afa Date: Fri May 4 11:26:10 2007 New Revision: 42666 Added: pypy/dist/pypy/doc/config/objspace.usemodules._socket.txt Removed: pypy/dist/pypy/doc/config/objspace.usemodules.rsocket.txt Log: Update options documentation Added: pypy/dist/pypy/doc/config/objspace.usemodules._socket.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/doc/config/objspace.usemodules._socket.txt Fri May 4 11:26:10 2007 @@ -0,0 +1,7 @@ +Use the '_socket' module. + +This is our implementation of '_socket', the Python builtin module +exposing socket primitives, which is wrapped and used by the standard +library 'socket.py' module. It is based on `rctypes`_. + +.. _`rctypes`: ../rctypes.html From cfbolz at codespeak.net Fri May 4 13:25:12 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 4 May 2007 13:25:12 +0200 (CEST) Subject: [pypy-svn] r42676 - pypy/dist/pypy/objspace/test Message-ID: <20070504112512.5C182808E@code0.codespeak.net> Author: cfbolz Date: Fri May 4 13:25:11 2007 New Revision: 42676 Modified: pypy/dist/pypy/objspace/test/test_thunkobjspace.py Log: sort of bug in the thunk space Modified: pypy/dist/pypy/objspace/test/test_thunkobjspace.py ============================================================================== --- pypy/dist/pypy/objspace/test/test_thunkobjspace.py (original) +++ pypy/dist/pypy/objspace/test/test_thunkobjspace.py Fri May 4 13:25:11 2007 @@ -98,3 +98,17 @@ assert lst == [3] assert type(y) is int assert lst == [3] + + def test_exception_in_thunk(self): + skip("thunk about what should happen there!") + from __pypy__ import lazy + def f(x): + if x: + return 42 + raise ValueError + f = lazy(f) + y = f(3) + assert y == 42 + y = f(0) + raises(ValueError, "str(y)") + raises(ValueError, "str(y)") # raises "RuntimeError: thunk is already being computed" From cfbolz at codespeak.net Fri May 4 13:29:38 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 4 May 2007 13:29:38 +0200 (CEST) Subject: [pypy-svn] r42677 - in pypy/dist/pypy/objspace: . test Message-ID: <20070504112938.3D1E18095@code0.codespeak.net> Author: cfbolz Date: Fri May 4 13:29:37 2007 New Revision: 42677 Modified: pypy/dist/pypy/objspace/test/test_thunkobjspace.py pypy/dist/pypy/objspace/thunk.py Log: proposed fix for the test Modified: pypy/dist/pypy/objspace/test/test_thunkobjspace.py ============================================================================== --- pypy/dist/pypy/objspace/test/test_thunkobjspace.py (original) +++ pypy/dist/pypy/objspace/test/test_thunkobjspace.py Fri May 4 13:29:37 2007 @@ -100,7 +100,6 @@ assert lst == [3] def test_exception_in_thunk(self): - skip("thunk about what should happen there!") from __pypy__ import lazy def f(x): if x: Modified: pypy/dist/pypy/objspace/thunk.py ============================================================================== --- pypy/dist/pypy/objspace/thunk.py (original) +++ pypy/dist/pypy/objspace/thunk.py Fri May 4 13:29:37 2007 @@ -47,6 +47,7 @@ def __init__(w_self, w_callable, args): w_self.w_callable = w_callable w_self.args = args + w_self.operr = None # special marker to say that w_self has not been computed yet w_NOT_COMPUTED_THUNK = W_Thunk(None, None) @@ -58,6 +59,8 @@ while w_alias is not None: if w_alias is w_NOT_COMPUTED_THUNK: assert isinstance(w_self, W_Thunk) + if w_self.operr is not None: + raise w_self.operr w_callable = w_self.w_callable args = w_self.args if w_callable is None or args is None: @@ -65,7 +68,11 @@ space.wrap("thunk is already being computed")) w_self.w_callable = None w_self.args = None - w_alias = space.call_args(w_callable, args) + try: + w_alias = space.call_args(w_callable, args) + except OperationError, operr: + w_self.operr = operr + raise # XXX detect circular w_alias result w_self.w_thunkalias = w_alias w_self = w_alias From afa at codespeak.net Fri May 4 13:34:48 2007 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 4 May 2007 13:34:48 +0200 (CEST) Subject: [pypy-svn] r42679 - pypy/dist/pypy/config Message-ID: <20070504113448.DAEB68096@code0.codespeak.net> Author: afa Date: Fri May 4 13:34:48 2007 New Revision: 42679 Modified: pypy/dist/pypy/config/pypyoption.py Log: _socket and select modules are supposed to work now Modified: pypy/dist/pypy/config/pypyoption.py ============================================================================== --- pypy/dist/pypy/config/pypyoption.py (original) +++ pypy/dist/pypy/config/pypyoption.py Fri May 4 13:34:48 2007 @@ -28,9 +28,7 @@ )) if sys.platform == "win32": - del working_modules["_socket"] del working_modules["fcntl"] - del working_modules["select"] del working_modules["readline"] del working_modules["crypt"] From arigo at codespeak.net Fri May 4 17:03:29 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 4 May 2007 17:03:29 +0200 (CEST) Subject: [pypy-svn] r42685 - pypy/branch/dist-rctypes2 Message-ID: <20070504150329.E8557806B@code0.codespeak.net> Author: arigo Date: Fri May 4 17:03:29 2007 New Revision: 42685 Added: pypy/branch/dist-rctypes2/ - copied from r42684, pypy/dist/ Log: A branch to work a bit on the 2nd rctypes implementation. Cannot be done in the trunk because of the extregistry conflict with the 1st rctypes implementation. From arigo at codespeak.net Fri May 4 17:04:57 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 4 May 2007 17:04:57 +0200 (CEST) Subject: [pypy-svn] r42686 - in pypy/branch/dist-rctypes2/pypy: rlib/rctypes rpython/rctypes Message-ID: <20070504150457.967DB806B@code0.codespeak.net> Author: arigo Date: Fri May 4 17:04:56 2007 New Revision: 42686 Modified: pypy/branch/dist-rctypes2/pypy/rlib/rctypes/implementation.py pypy/branch/dist-rctypes2/pypy/rpython/rctypes/implementation.py Log: Re-enable the new rctypes, comment out parts of the old one. Most tests fail now, except the ones in the new rctypes, which is the point. Modified: pypy/branch/dist-rctypes2/pypy/rlib/rctypes/implementation.py ============================================================================== --- pypy/branch/dist-rctypes2/pypy/rlib/rctypes/implementation.py (original) +++ pypy/branch/dist-rctypes2/pypy/rlib/rctypes/implementation.py Fri May 4 17:04:56 2007 @@ -1,5 +1,5 @@ import py -py.test.skip("extregistry conflicts with the other rctypes :-(") +##py.test.skip("extregistry conflicts with the other rctypes :-(") from pypy.annotation import model as annmodel from pypy.tool.tls import tlsobject from pypy.rlib.rctypes import rctypesobject Modified: pypy/branch/dist-rctypes2/pypy/rpython/rctypes/implementation.py ============================================================================== --- pypy/branch/dist-rctypes2/pypy/rpython/rctypes/implementation.py (original) +++ pypy/branch/dist-rctypes2/pypy/rpython/rctypes/implementation.py Fri May 4 17:04:56 2007 @@ -120,13 +120,13 @@ # Importing for side effect of registering types with extregistry -from pypy.rpython.rctypes.atype import SomeCTypesType -import pypy.rpython.rctypes.aprimitive -import pypy.rpython.rctypes.apointer -import pypy.rpython.rctypes.aarray -import pypy.rpython.rctypes.afunc -import pypy.rpython.rctypes.achar_p -import pypy.rpython.rctypes.astruct -import pypy.rpython.rctypes.avoid_p -import pypy.rpython.rctypes.astringbuf -import pypy.rpython.rctypes.apyobject +##from pypy.rpython.rctypes.atype import SomeCTypesType +##import pypy.rpython.rctypes.aprimitive +##import pypy.rpython.rctypes.apointer +##import pypy.rpython.rctypes.aarray +##import pypy.rpython.rctypes.afunc +##import pypy.rpython.rctypes.achar_p +##import pypy.rpython.rctypes.astruct +##import pypy.rpython.rctypes.avoid_p +##import pypy.rpython.rctypes.astringbuf +##import pypy.rpython.rctypes.apyobject From cfbolz at codespeak.net Fri May 4 19:12:13 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 4 May 2007 19:12:13 +0200 (CEST) Subject: [pypy-svn] r42688 - pypy/dist/pypy/translator/stackless Message-ID: <20070504171213.D50A38075@code0.codespeak.net> Author: cfbolz Date: Fri May 4 19:12:13 2007 New Revision: 42688 Modified: pypy/dist/pypy/translator/stackless/transform.py Log: typo in the comment Modified: pypy/dist/pypy/translator/stackless/transform.py ============================================================================== --- pypy/dist/pypy/translator/stackless/transform.py (original) +++ pypy/dist/pypy/translator/stackless/transform.py Fri May 4 19:12:13 2007 @@ -140,7 +140,7 @@ # restate subcase at the end of it. # # note that we order types by the index of the erased type in -# STORAGE_TYPES, to increase the chance that we can resuse the types. +# STORAGE_TYPES, to increase the chance that we can reuse the types. # # in simple cases this approach creates graphs that are more # complicated than needed, so we run the graph through a few From cfbolz at codespeak.net Fri May 4 19:13:09 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 4 May 2007 19:13:09 +0200 (CEST) Subject: [pypy-svn] r42689 - in pypy/dist/pypy/objspace: . test Message-ID: <20070504171309.4F2B18075@code0.codespeak.net> Author: cfbolz Date: Fri May 4 19:13:08 2007 New Revision: 42689 Modified: pypy/dist/pypy/objspace/test/test_thunkobjspace.py pypy/dist/pypy/objspace/thunk.py Log: solve various recurivity problems in the thunk space Modified: pypy/dist/pypy/objspace/test/test_thunkobjspace.py ============================================================================== --- pypy/dist/pypy/objspace/test/test_thunkobjspace.py (original) +++ pypy/dist/pypy/objspace/test/test_thunkobjspace.py Fri May 4 19:13:08 2007 @@ -1,4 +1,5 @@ from pypy.conftest import gettestobjspace +from pypy.interpreter import gateway class AppTest_Thunk: @@ -64,6 +65,15 @@ become(y, z) assert x is y is z + def test_double_become2(self): + from __pypy__ import thunk, become + x = [] + y = [] + z = [] + become(x, y) + become(x, z) + assert x is y is z + def test_thunk_forcing_while_forcing(self): from __pypy__ import thunk, become def f(): @@ -71,7 +81,7 @@ x = thunk(f) raises(RuntimeError, 'x+1') - def INPROGRESS_test_thunk_forcing_while_forcing_2(self): + def test_thunk_forcing_while_forcing_2(self): from __pypy__ import thunk, become def f(): return x @@ -85,6 +95,26 @@ assert is_thunk(thunk(f)) assert not is_thunk(42) + def test_is_thunk2(self): + from __pypy__ import thunk, become, is_thunk + def f(): + return 42 + x = thunk(f) + assert is_thunk(x) + assert x == 42 + assert not is_thunk(x) + + def test_is_thunk_become(self): + from __pypy__ import thunk, become, is_thunk + def f(): + return 42 + x = thunk(f) + y = [] + become(y, x) + assert is_thunk(y) + assert y == 42 + assert not is_thunk(y) + def test_lazy(self): from __pypy__ import lazy lst = [] @@ -110,4 +140,10 @@ assert y == 42 y = f(0) raises(ValueError, "str(y)") - raises(ValueError, "str(y)") # raises "RuntimeError: thunk is already being computed" + raises(ValueError, "str(y)") + + def test_become_yourself(self): + from __pypy__ import become + x = [] + become(x, x) + assert str(x) == "[]" Modified: pypy/dist/pypy/objspace/thunk.py ============================================================================== --- pypy/dist/pypy/objspace/thunk.py (original) +++ pypy/dist/pypy/objspace/thunk.py Fri May 4 19:13:08 2007 @@ -73,12 +73,30 @@ except OperationError, operr: w_self.operr = operr raise - # XXX detect circular w_alias result + if _is_circular(w_self, w_alias): + operr = OperationError(space.w_RuntimeError, + space.wrap("circular thunk alias")) + w_self.operr = operr + raise operr w_self.w_thunkalias = w_alias + # XXX do path compression? w_self = w_alias w_alias = w_self.w_thunkalias return w_self +def _is_circular(w_obj, w_alias): + assert (w_obj.w_thunkalias is None or + w_obj.w_thunkalias is w_NOT_COMPUTED_THUNK) + while 1: + if w_obj is w_alias: + return True + w_next = w_alias.w_thunkalias + if w_next is None: + return False + if w_next is w_NOT_COMPUTED_THUNK: + return False + w_alias = w_next + def force(space, w_self): if w_self.w_thunkalias is not None: w_self = _force(space, w_self) @@ -93,13 +111,20 @@ def is_thunk(space, w_obj): """Check if an object is a thunk that has not been computed yet.""" - return space.newbool(w_obj.w_thunkalias is w_NOT_COMPUTED_THUNK) + while 1: + w_alias = w_obj.w_thunkalias + if w_alias is None: + return space.w_False + if w_alias is w_NOT_COMPUTED_THUNK: + return space.w_True + w_obj = w_alias app_is_thunk = gateway.interp2app(is_thunk) def become(space, w_target, w_source): """Globally replace the target object with the source one.""" w_target = force(space, w_target) - w_target.w_thunkalias = w_source + if not _is_circular(w_target, w_source): + w_target.w_thunkalias = w_source return space.w_None app_become = gateway.interp2app(become) From fijal at codespeak.net Sat May 5 19:49:05 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 5 May 2007 19:49:05 +0200 (CEST) Subject: [pypy-svn] r42707 - pypy/dist/pypy/lib/distributed/test Message-ID: <20070505174905.0BC38806D@code0.codespeak.net> Author: fijal Date: Sat May 5 19:49:04 2007 New Revision: 42707 Modified: pypy/dist/pypy/lib/distributed/test/test_socklayer.py Log: Skip in case pylib is not there Modified: pypy/dist/pypy/lib/distributed/test/test_socklayer.py ============================================================================== --- pypy/dist/pypy/lib/distributed/test/test_socklayer.py (original) +++ pypy/dist/pypy/lib/distributed/test/test_socklayer.py Sat May 5 19:49:04 2007 @@ -14,6 +14,10 @@ x = X() + try: + import py + except ImportError: + skip("pylib not importable") from py.__.green.pipe.gsocket import GreenSocket from distributed.socklayer import socket_loop, connect from py.__.green.greensock2 import oneof, allof From santagada at codespeak.net Sun May 6 01:42:08 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Sun, 6 May 2007 01:42:08 +0200 (CEST) Subject: [pypy-svn] r42722 - pypy/dist/pypy/lang/js Message-ID: <20070505234208.597898079@code0.codespeak.net> Author: santagada Date: Sun May 6 01:42:07 2007 New Revision: 42722 Modified: pypy/dist/pypy/lang/js/jsgrammar.txt Log: some more adjusts to the grammar Modified: pypy/dist/pypy/lang/js/jsgrammar.txt ============================================================================== --- pypy/dist/pypy/lang/js/jsgrammar.txt (original) +++ pypy/dist/pypy/lang/js/jsgrammar.txt Sun May 6 01:42:07 2007 @@ -29,7 +29,7 @@ | ; -block : ["{"] statementlist? ["}"] +block : ["{"] >statementliststatementlist< @@ -43,7 +43,7 @@ | variabledeclaration ; -variabledeclaration : identifier initialiser? +variabledeclaration : identifier >initialiser Author: cfbolz Date: Sun May 6 11:08:59 2007 New Revision: 42729 Modified: pypy/dist/pypy/lang/js/newparser.py pypy/dist/pypy/lang/js/operations.py pypy/dist/pypy/lang/js/test/test_new_parser.py Log: first sketch at a transformer to get the evaluation trees the js interpreter works with out of the parse trees the new parser produces. Modified: pypy/dist/pypy/lang/js/newparser.py ============================================================================== --- pypy/dist/pypy/lang/js/newparser.py (original) +++ pypy/dist/pypy/lang/js/newparser.py Sun May 6 11:08:59 2007 @@ -1,29 +1,84 @@ from pypy.rlib.parsing.ebnfparse import parse_ebnf, make_parse_function from pypy.rlib.parsing.parsing import ParseError, Rule +from pypy.rlib.parsing.tree import RPythonVisitor, Symbol +from pypy.lang.js.jsobj import W_Number +from pypy.lang.js import operations import sys -try: - t = open("jsgrammar.txt").read() - regexs, rules, ToAST = parse_ebnf(t) -except ParseError,e: - print e.nice_error_message(filename="jsgrammar.txt",source=t) - sys.exit(1) - -def setstartrule(rules, start): - "takes the rule start and put it on the beginning of the rules" - oldpos = 0 - newrules = [Rule("hacked_first_symbol", [[start, "EOF"]])] + rules - return newrules - -if len(sys.argv) == 1: - parse = make_parse_function(regexs, rules, eof=True) -else: - parse = make_parse_function(regexs, setstartrule(rules,sys.argv[1]), eof=True) - -print rules[2].nonterminal -source = raw_input() -while source != "": - t = parse(source).children[0].visit(ToAST())[0] - print t - t.view() - source = raw_input() +##try: +## t = open("jsgrammar.txt").read() +## regexs, rules, ToAST = parse_ebnf(t) +##except ParseError,e: +## print e.nice_error_message(filename="jsgrammar.txt",source=t) +## sys.exit(1) +## +##def setstartrule(rules, start): +## "takes the rule start and put it on the beginning of the rules" +## oldpos = 0 +## newrules = [Rule("hacked_first_symbol", [[start, "EOF"]])] + rules +## return newrules +## +##if len(sys.argv) == 1: +## parse = make_parse_function(regexs, rules, eof=True) +##else: +## parse = make_parse_function(regexs, setstartrule(rules,sys.argv[1]), eof=True) +## +##print rules[2].nonterminal +##source = raw_input() +##while source != "": +## t = parse(source).children[0].visit(ToAST())[0] +## print t +## t.view() +## source = raw_input() + +class EvalTreeBuilder(RPythonVisitor): + BINOP_TO_CLS = { + '+': operations.Plus, + '-': operations.Minus, + '*': operations.Mult, + '/': operations.Div, + '%': operations.Mod, + } + UNOP_TO_CLS = { + '+': operations.UPlus, + '-': operations.UMinus, + } + def get_instance(self, symbol, cls): + assert isinstance(symbol, Symbol) + source_pos = symbol.token.source_pos + # XXX some of the source positions are not perfect + return cls(None, "no clue what self.type is used for", + symbol.additional_info, + source_pos.lineno, + source_pos.columnno, + source_pos.columnno + len(symbol.additional_info)) + + def visit_DECIMALLITERAL(self, node): + result = self.get_instance(node, operations.Number) + result.num = float(node.additional_info) + return result + + def binaryop(self, node): + left = self.dispatch(node.children[0]) + for i in range((len(node.children) - 1) // 2): + op = node.children[i * 2 + 1] + result = self.get_instance( + op, self.BINOP_TO_CLS[op.additional_info]) + right = self.dispatch(node.children[i * 2 + 2]) + result.left = left + result.right = right + left = result + return left + + visit_additiveexpression = binaryop + visit_multiplicativeexpression = binaryop + + def visit_unaryexpression(self, node): + op = node.children[0] + result = self.get_instance( + op, self.UNOP_TO_CLS[op.additional_info]) + child = self.dispatch(node.children[1]) + result.expr = child + result.postfix = False + return result + Modified: pypy/dist/pypy/lang/js/operations.py ============================================================================== --- pypy/dist/pypy/lang/js/operations.py (original) +++ pypy/dist/pypy/lang/js/operations.py Sun May 6 11:08:59 2007 @@ -4,6 +4,7 @@ Implements the javascript operations nodes for the interpretation tree """ +#XXX * imports are bad from pypy.lang.js.jsobj import * from pypy.lang.js.jsparser import JsSyntaxError from pypy.rlib.parsing.ebnfparse import Symbol, Nonterminal Modified: pypy/dist/pypy/lang/js/test/test_new_parser.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_new_parser.py (original) +++ pypy/dist/pypy/lang/js/test/test_new_parser.py Sun May 6 11:08:59 2007 @@ -4,6 +4,7 @@ from pypy.rlib.parsing.ebnfparse import parse_ebnf, make_parse_function from pypy.rlib.parsing.parsing import ParseError, Rule from pypy.rlib.parsing.tree import RPythonVisitor +from pypy.lang.js.newparser import EvalTreeBuilder from pypy import conftest import sys @@ -277,3 +278,23 @@ self.parse('function z (a,b,c,d,e) {;}') +class TestToEvalTree(BaseGrammarTest): + def setup_class(cls): + cls.parse = parse_func('expression') + + def to_etree(self, s): + return EvalTreeBuilder().dispatch(self.parse(s)) + + def test_primary(self): + etree = self.to_etree('(6)') + w_num = etree.eval(None) + assert w_num.ToNumber() == 6 + etree = self.to_etree('((((6))))') + w_num = etree.eval(None) + assert w_num.ToNumber() == 6 + etree = self.to_etree('1 - 1 - 1') + w_num = etree.eval(None) + assert w_num.ToNumber() == -1 + etree = self.to_etree('-(6 * (6 * 6)) + 6 - 6') + w_num = etree.eval(None) + assert w_num.ToNumber() == -216 From cfbolz at codespeak.net Sun May 6 11:15:33 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 6 May 2007 11:15:33 +0200 (CEST) Subject: [pypy-svn] r42732 - pypy/dist/pypy/lang/js Message-ID: <20070506091533.5010C807C@code0.codespeak.net> Author: cfbolz Date: Sun May 6 11:15:33 2007 New Revision: 42732 Modified: pypy/dist/pypy/lang/js/jsobj.py Log: 2.5 compatibility: self.args is enforced to be an iterable Modified: pypy/dist/pypy/lang/js/jsobj.py ============================================================================== --- pypy/dist/pypy/lang/js/jsobj.py (original) +++ pypy/dist/pypy/lang/js/jsobj.py Sun May 6 11:15:33 2007 @@ -15,7 +15,8 @@ class ThrowException(JsBaseExcept): def __init__(self, exception): - self.exception = self.args = exception + self.exception = exception + self.args = [exception] class JsTypeError(JsBaseExcept): pass From fijal at codespeak.net Sun May 6 11:19:08 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 6 May 2007 11:19:08 +0200 (CEST) Subject: [pypy-svn] r42733 - pypy/dist/pypy/lib/distributed/test Message-ID: <20070506091908.CD0FD807F@code0.codespeak.net> Author: fijal Date: Sun May 6 11:19:08 2007 New Revision: 42733 Modified: pypy/dist/pypy/lib/distributed/test/test_distributed.py Log: Increase recursion limit for these tests Modified: pypy/dist/pypy/lib/distributed/test/test_distributed.py ============================================================================== --- pypy/dist/pypy/lib/distributed/test/test_distributed.py (original) +++ pypy/dist/pypy/lib/distributed/test/test_distributed.py Sun May 6 11:19:08 2007 @@ -3,6 +3,7 @@ """ from pypy.conftest import gettestobjspace +import sys class AppTestNoProxy(object): disabled = True @@ -100,6 +101,11 @@ from distributed import test_env return test_env """) + cls.reclimit = sys.getrecursionlimit() + sys.setrecursionlimit(100000) + + def teardown_class(cls): + sys.setrecursionlimit(cls.reclimit) def test_remote_protocol_call(self): def f(x, y): From fijal at codespeak.net Sun May 6 12:08:52 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 6 May 2007 12:08:52 +0200 (CEST) Subject: [pypy-svn] r42739 - pypy/dist/pypy/translator/js Message-ID: <20070506100852.627B6807F@code0.codespeak.net> Author: fijal Date: Sun May 6 12:08:52 2007 New Revision: 42739 Modified: pypy/dist/pypy/translator/js/opcodes.py Log: Minor cleanup Modified: pypy/dist/pypy/translator/js/opcodes.py ============================================================================== --- pypy/dist/pypy/translator/js/opcodes.py (original) +++ pypy/dist/pypy/translator/js/opcodes.py Sun May 6 12:08:52 2007 @@ -124,8 +124,6 @@ 'oosetfield' : [_SetFieldDispatcher(Builtins, class_map)], 'oogetfield' : [_GetFieldDispatcher(Builtins, class_map)], 'oosend' : [_MethodDispatcher(Builtins, class_map)], - #'ooupcast' : [_NotImplemented("Inheritance not implemented (ooupcast)")], - #'oodowncast' : [_NotImplemented("Inheritance not implemented (oodowncast)")], 'ooupcast' : DoNothing, 'oodowncast' : DoNothing, 'oononnull' : [PushAllArgs,_Prefix('!!')], @@ -133,10 +131,6 @@ 'ooparse_int' : [PushAllArgs,_CastFun("parseInt",2)], 'ooparse_float' : [PushAllArgs,_CastFun("parseFloat",1)], 'oois' : '===', - # when casting from bool we want that every truth value is casted - # to 1: we can't simply DoNothing, because the CLI stack could - # contains a truth value not equal to 1, so we should use the !=0 - # trick. 'cast_bool_to_int': CopyName, 'cast_bool_to_uint': CopyName, 'cast_bool_to_float': CopyName, From fijal at codespeak.net Sun May 6 21:48:04 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 6 May 2007 21:48:04 +0200 (CEST) Subject: [pypy-svn] r42757 - pypy/dist/pypy/translator/oosupport Message-ID: <20070506194804.9AD48806D@code0.codespeak.net> Author: fijal Date: Sun May 6 21:48:04 2007 New Revision: 42757 Modified: pypy/dist/pypy/translator/oosupport/function.py Log: Python 2.5 Modified: pypy/dist/pypy/translator/oosupport/function.py ============================================================================== --- pypy/dist/pypy/translator/oosupport/function.py (original) +++ pypy/dist/pypy/translator/oosupport/function.py Sun May 6 21:48:04 2007 @@ -157,7 +157,7 @@ for link in block.exits: if link.exitcase is None: continue # see above - assert issubclass(link.exitcase, Exception) + assert issubclass(link.exitcase, BaseException) ll_meta_exc = link.llexitcase self.record_ll_meta_exc(ll_meta_exc) self.begin_catch(link.llexitcase) From pedronis at codespeak.net Sun May 6 22:08:55 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sun, 6 May 2007 22:08:55 +0200 (CEST) Subject: [pypy-svn] r42758 - pypy/dist/pypy/translator/oosupport Message-ID: <20070506200855.36C31806D@code0.codespeak.net> Author: pedronis Date: Sun May 6 22:08:53 2007 New Revision: 42758 Modified: pypy/dist/pypy/translator/oosupport/function.py Log: this was what was intended I suppose Modified: pypy/dist/pypy/translator/oosupport/function.py ============================================================================== --- pypy/dist/pypy/translator/oosupport/function.py (original) +++ pypy/dist/pypy/translator/oosupport/function.py Sun May 6 22:08:53 2007 @@ -8,6 +8,8 @@ from pypy.translator.oosupport.treebuilder import SubOperation from pypy.translator.oosupport.metavm import InstructionList, StoreResult + + class Function(object): def __init__(self, db, graph, name = None, is_method = False, is_entrypoint = False): @@ -157,7 +159,7 @@ for link in block.exits: if link.exitcase is None: continue # see above - assert issubclass(link.exitcase, BaseException) + assert issubclass(link.exitcase, py.builtin.BaseException) ll_meta_exc = link.llexitcase self.record_ll_meta_exc(ll_meta_exc) self.begin_catch(link.llexitcase) From afa at codespeak.net Sun May 6 22:55:41 2007 From: afa at codespeak.net (afa at codespeak.net) Date: Sun, 6 May 2007 22:55:41 +0200 (CEST) Subject: [pypy-svn] r42761 - in pypy/dist/pypy/tool: pytest test Message-ID: <20070506205541.862C98067@code0.codespeak.net> Author: afa Date: Sun May 6 22:55:40 2007 New Revision: 42761 Modified: pypy/dist/pypy/tool/pytest/appsupport.py pypy/dist/pypy/tool/test/test_error.py pypy/dist/pypy/tool/test/test_pytestsupport.py Log: Better support in tests, when a applevel function calls an interp-level function which raises a (faked) builtin exception. Just trying to understand why test_select keeps failing on scottdial.com Modified: pypy/dist/pypy/tool/pytest/appsupport.py ============================================================================== --- pypy/dist/pypy/tool/pytest/appsupport.py (original) +++ pypy/dist/pypy/tool/pytest/appsupport.py Sun May 6 22:55:40 2007 @@ -65,7 +65,7 @@ def __init__(self, space, operr): self.space = space self.operr = operr - self.typename = operr.w_type.name + self.typename = operr.w_type.getname(space, "?") self.traceback = AppTraceback(space, self.operr.application_traceback) debug_excs = getattr(operr, 'debug_excs', []) if debug_excs: Modified: pypy/dist/pypy/tool/test/test_error.py ============================================================================== --- pypy/dist/pypy/tool/test/test_error.py (original) +++ pypy/dist/pypy/tool/test/test_error.py Sun May 6 22:55:40 2007 @@ -1,5 +1,5 @@ -""" Tests some error handilng routines +""" Tests some error handling routines """ from pypy.translator.translator import TranslationContext Modified: pypy/dist/pypy/tool/test/test_pytestsupport.py ============================================================================== --- pypy/dist/pypy/tool/test/test_pytestsupport.py (original) +++ pypy/dist/pypy/tool/test/test_pytestsupport.py Sun May 6 22:55:40 2007 @@ -69,6 +69,22 @@ assert not appex.errisinstance(A) +def test_fakedexception(space): + from cPickle import PicklingError + def raise_error(): + raise PicklingError("SomeMessage") + space.setitem(space.builtin.w_dict, space.wrap('raise_error'), + space.wrap(raise_error)) + + try: + space.appexec([], "(): raise_error()") + except OperationError, e: + appex = AppExceptionInfo(space, e) + else: + py.test.fail("did not raise!") + assert "PicklingError" in appex.exconly() + assert "SomeMessage" in appex.exconly() + class AppTestWithWrappedInterplevelAttributes: def setup_class(cls): space = cls.space From santagada at codespeak.net Sun May 6 22:59:06 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Sun, 6 May 2007 22:59:06 +0200 (CEST) Subject: [pypy-svn] r42762 - in pypy/dist/pypy/lang/js: . test Message-ID: <20070506205906.800FE8067@code0.codespeak.net> Author: santagada Date: Sun May 6 22:59:06 2007 New Revision: 42762 Modified: pypy/dist/pypy/lang/js/jsobj.py pypy/dist/pypy/lang/js/newparser.py pypy/dist/pypy/lang/js/test/test_new_parser.py Log: make ++ work on new_parser and everywhere else also Modified: pypy/dist/pypy/lang/js/jsobj.py ============================================================================== --- pypy/dist/pypy/lang/js/jsobj.py (original) +++ pypy/dist/pypy/lang/js/jsobj.py Sun May 6 22:59:06 2007 @@ -77,7 +77,7 @@ raise NotImplementedError def PutValue(self, w, ctx): - raise NotImplementedError + pass def Call(self, ctx, args=[], this=None): raise NotImplementedError @@ -427,7 +427,7 @@ return r_uint(0) return r_uint(self.floatval) - + class W_List(W_Root): def __init__(self, list_w): self.list_w = list_w Modified: pypy/dist/pypy/lang/js/newparser.py ============================================================================== --- pypy/dist/pypy/lang/js/newparser.py (original) +++ pypy/dist/pypy/lang/js/newparser.py Sun May 6 22:59:06 2007 @@ -42,6 +42,7 @@ UNOP_TO_CLS = { '+': operations.UPlus, '-': operations.UMinus, + '++': operations.Increment, } def get_instance(self, symbol, cls): assert isinstance(symbol, Symbol) @@ -78,6 +79,7 @@ result = self.get_instance( op, self.UNOP_TO_CLS[op.additional_info]) child = self.dispatch(node.children[1]) + print child result.expr = child result.postfix = False return result Modified: pypy/dist/pypy/lang/js/test/test_new_parser.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_new_parser.py (original) +++ pypy/dist/pypy/lang/js/test/test_new_parser.py Sun May 6 22:59:06 2007 @@ -298,3 +298,7 @@ etree = self.to_etree('-(6 * (6 * 6)) + 6 - 6') w_num = etree.eval(None) assert w_num.ToNumber() == -216 + etree = self.to_etree('++5') + w_num = etree.eval(None) + assert w_num.ToNumber() == 6 + \ No newline at end of file From santagada at codespeak.net Sun May 6 23:16:46 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Sun, 6 May 2007 23:16:46 +0200 (CEST) Subject: [pypy-svn] r42763 - in pypy/dist/pypy/lang/js: . test Message-ID: <20070506211646.7305B8067@code0.codespeak.net> Author: santagada Date: Sun May 6 23:16:46 2007 New Revision: 42763 Modified: pypy/dist/pypy/lang/js/newparser.py pypy/dist/pypy/lang/js/test/test_new_parser.py Log: support for strings on newparser... now I think I fully understand how it works and I am planing to move it to another file, maybe concrete_tree_builder or something. Modified: pypy/dist/pypy/lang/js/newparser.py ============================================================================== --- pypy/dist/pypy/lang/js/newparser.py (original) +++ pypy/dist/pypy/lang/js/newparser.py Sun May 6 23:16:46 2007 @@ -43,6 +43,7 @@ '+': operations.UPlus, '-': operations.UMinus, '++': operations.Increment, + '--': operations.Decrement, } def get_instance(self, symbol, cls): assert isinstance(symbol, Symbol) @@ -59,6 +60,15 @@ result.num = float(node.additional_info) return result + def string(self,node): + print node.additional_info + result = self.get_instance(node, operations.String) + result.strval = node.additional_info[1:-1] #XXX should do unquoting + return result + + visit_DOUBLESTRING = string + visit_SINGLESTRING = string + def binaryop(self, node): left = self.dispatch(node.children[0]) for i in range((len(node.children) - 1) // 2): @@ -79,7 +89,6 @@ result = self.get_instance( op, self.UNOP_TO_CLS[op.additional_info]) child = self.dispatch(node.children[1]) - print child result.expr = child result.postfix = False return result Modified: pypy/dist/pypy/lang/js/test/test_new_parser.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_new_parser.py (original) +++ pypy/dist/pypy/lang/js/test/test_new_parser.py Sun May 6 23:16:46 2007 @@ -284,7 +284,11 @@ def to_etree(self, s): return EvalTreeBuilder().dispatch(self.parse(s)) - + + def eval_expr(self, s): + etree = self.to_etree(s) + return etree.eval(None) + def test_primary(self): etree = self.to_etree('(6)') w_num = etree.eval(None) @@ -301,4 +305,8 @@ etree = self.to_etree('++5') w_num = etree.eval(None) assert w_num.ToNumber() == 6 + w_num = self.eval_expr('--5') + assert w_num.ToNumber() == 4 + w_str = self.eval_expr('"hello "+\'world\'') + assert w_str.ToString() == 'hello world' \ No newline at end of file From antocuni at codespeak.net Mon May 7 11:55:00 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 7 May 2007 11:55:00 +0200 (CEST) Subject: [pypy-svn] r42780 - pypy/dist/pypy/translator/cli Message-ID: <20070507095500.06CC48065@code0.codespeak.net> Author: antocuni Date: Mon May 7 11:55:00 2007 New Revision: 42780 Modified: pypy/dist/pypy/translator/cli/cts.py Log: a much saner way to escape keywords Modified: pypy/dist/pypy/translator/cli/cts.py ============================================================================== --- pypy/dist/pypy/translator/cli/cts.py (original) +++ pypy/dist/pypy/translator/cli/cts.py Mon May 7 11:55:00 2007 @@ -136,7 +136,7 @@ def escape_name(self, name): """Mangle then name if it's a ilasm reserved word""" if name in self.ILASM_KEYWORDS: - return name + '__MANGLED' # XXX: it could not be unique + return "'%s'" % name else: return name From cfbolz at codespeak.net Mon May 7 12:29:26 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 7 May 2007 12:29:26 +0200 (CEST) Subject: [pypy-svn] r42784 - in pypy/dist/demo: . sqlinj Message-ID: <20070507102926.382398065@code0.codespeak.net> Author: cfbolz Date: Mon May 7 12:29:25 2007 New Revision: 42784 Modified: pypy/dist/demo/fibonacci.py pypy/dist/demo/fibonacci2.py pypy/dist/demo/sharedref.py pypy/dist/demo/sqlinj/tserver.py Log: oops, leftover occurences of pypymagic Modified: pypy/dist/demo/fibonacci.py ============================================================================== --- pypy/dist/demo/fibonacci.py (original) +++ pypy/dist/demo/fibonacci.py Mon May 7 12:29:25 2007 @@ -9,7 +9,7 @@ """ try: - from pypymagic import thunk # only available in 'py.py -o thunk' + from __pypy__ import thunk # only available in 'py.py -o thunk' except ImportError: print __doc__ raise SystemExit(2) Modified: pypy/dist/demo/fibonacci2.py ============================================================================== --- pypy/dist/demo/fibonacci2.py (original) +++ pypy/dist/demo/fibonacci2.py Mon May 7 12:29:25 2007 @@ -11,7 +11,7 @@ import pprint try: - from pypymagic import lazy + from __pypy__ import lazy except ImportError: print __doc__ raise SystemExit(2) Modified: pypy/dist/demo/sharedref.py ============================================================================== --- pypy/dist/demo/sharedref.py (original) +++ pypy/dist/demo/sharedref.py Mon May 7 12:29:25 2007 @@ -35,7 +35,7 @@ """ import sys, marshal -from pypymagic import thunk, become +from __pypy__ import thunk, become from socket import * from select import select Modified: pypy/dist/demo/sqlinj/tserver.py ============================================================================== --- pypy/dist/demo/sqlinj/tserver.py (original) +++ pypy/dist/demo/sqlinj/tserver.py Mon May 7 12:29:25 2007 @@ -14,7 +14,7 @@ """ import sys -import pypymagic +import __pypy__ import gadfly From fijal at codespeak.net Mon May 7 13:15:13 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 7 May 2007 13:15:13 +0200 (CEST) Subject: [pypy-svn] r42788 - pypy/dist/pypy/translator/oosupport Message-ID: <20070507111513.EC6358065@code0.codespeak.net> Author: fijal Date: Mon May 7 13:15:13 2007 New Revision: 42788 Modified: pypy/dist/pypy/translator/oosupport/metavm.py Log: Add possibility for slicing parameters, just in case Modified: pypy/dist/pypy/translator/oosupport/metavm.py ============================================================================== --- pypy/dist/pypy/translator/oosupport/metavm.py (original) +++ pypy/dist/pypy/translator/oosupport/metavm.py Mon May 7 13:15:13 2007 @@ -239,8 +239,17 @@ class _PushAllArgs(MicroInstruction): """ Pushes all arguments of the instruction onto the stack in order. """ + def __init__(self, slice=None): + """ Eventually slice args + """ + self.slice = slice + def render(self, generator, op): - for arg in op.args: + if self.slice is not None: + args = op.args[self.slice] + else: + args = op.args + for arg in args: generator.load(arg) class PushPrimitive(MicroInstruction): From fijal at codespeak.net Mon May 7 13:15:43 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 7 May 2007 13:15:43 +0200 (CEST) Subject: [pypy-svn] r42789 - pypy/dist/pypy/translator/js/test Message-ID: <20070507111543.42E7C8065@code0.codespeak.net> Author: fijal Date: Mon May 7 13:15:42 2007 New Revision: 42789 Modified: pypy/dist/pypy/translator/js/test/test_rdict.py pypy/dist/pypy/translator/js/test/test_rlist.py Log: Add references to original tests, for py.test -k completion Modified: pypy/dist/pypy/translator/js/test/test_rdict.py ============================================================================== --- pypy/dist/pypy/translator/js/test/test_rdict.py (original) +++ pypy/dist/pypy/translator/js/test/test_rdict.py Mon May 7 13:15:42 2007 @@ -3,6 +3,8 @@ from pypy.rpython.test.test_rdict import BaseTestRdict from pypy.translator.js.test.runtest import JsTest +# ====> ../../../rpython/test/test_rdict.py + class TestJsDict(JsTest, BaseTestRdict): def test_tuple_dict(self): py.test.skip("rdict not implemented") Modified: pypy/dist/pypy/translator/js/test/test_rlist.py ============================================================================== --- pypy/dist/pypy/translator/js/test/test_rlist.py (original) +++ pypy/dist/pypy/translator/js/test/test_rlist.py Mon May 7 13:15:42 2007 @@ -9,6 +9,8 @@ class Bar(Foo): pass +# ====> ../../../rpython/test/test_rlist.py + class TestJsList(JsTest, BaseTestRlist): def test_append(self): def dummyfn(): From fijal at codespeak.net Mon May 7 13:17:59 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 7 May 2007 13:17:59 +0200 (CEST) Subject: [pypy-svn] r42790 - pypy/dist/pypy/translator/js Message-ID: <20070507111759.B25818065@code0.codespeak.net> Author: fijal Date: Mon May 7 13:17:59 2007 New Revision: 42790 Modified: pypy/dist/pypy/translator/js/asmgen.py pypy/dist/pypy/translator/js/function.py pypy/dist/pypy/translator/js/js.py pypy/dist/pypy/translator/js/jsbuiltin.py pypy/dist/pypy/translator/js/metavm.py pypy/dist/pypy/translator/js/opcodes.py Log: * builtin refactorings, got rid of some ugly hacks * subst_table is working again => better resulting code * be insanly delicate about stack. No unnecessary values should reside on stack any more * this refactoring should fix misterious test failure apeearing from time to time in the nightly test run Modified: pypy/dist/pypy/translator/js/asmgen.py ============================================================================== --- pypy/dist/pypy/translator/js/asmgen.py (original) +++ pypy/dist/pypy/translator/js/asmgen.py Mon May 7 13:17:59 2007 @@ -53,6 +53,15 @@ def __len__(self): return len(self.l) + def __nonzero__(self): + return len(self.l) > 0 + + def empty(self): + self.l = [] + + def __repr__(self): + return "" % (repr(self.l),) + class AsmGen(object): """ JS 'assembler' generator routines """ @@ -166,9 +175,8 @@ self.codegenerator.openblock() def change_name(self, from_name, to_name): - # if isinstance(from_name,Variable) and isinstance(to_name,Variable): - # self.subst_table[from_name.name] = to_name.name - pass + if isinstance(from_name,Variable) and isinstance(to_name,Variable): + self.subst_table[from_name.name] = to_name.name def cast_function(self, name, num): # FIXME: redundancy with call @@ -193,7 +201,7 @@ def set_field(self, useless_parameter, name): v = self.right_hand.pop() self.codegenerator.writeline("%s.%s = %s;"%(self.right_hand.pop(), name, v)) - self.right_hand.append(None) + #self.right_hand.append(None) def call_method(self, obj, name, signature): l = [self.right_hand.pop() for i in signature] @@ -215,7 +223,7 @@ self.right_hand.append("this") def store_void(self): - if len(self.right_hand) == 0: + if not len(self.right_hand): return v = self.right_hand.pop() if v is not None and v.find('('): @@ -286,3 +294,6 @@ def throw_real(self, s): self.codegenerator.writeline("throw(%s);"%s) + + def clean_stack(self): + self.right_hand.empty() Modified: pypy/dist/pypy/translator/js/function.py ============================================================================== --- pypy/dist/pypy/translator/js/function.py (original) +++ pypy/dist/pypy/translator/js/function.py Mon May 7 13:17:59 2007 @@ -34,13 +34,11 @@ assert False def store(self, v): - if isinstance(v, flowmodel.Variable): - if v.concretetype is not Void: - self.ilasm.store_local(v) - else: - self.ilasm.store_void() + assert isinstance(v, flowmodel.Variable) + if v.concretetype is not Void: + self.ilasm.store_local(v) else: - assert False + self.ilasm.store_void() def change_name(self, name, to_name): self.ilasm.change_name(name, to_name) @@ -230,3 +228,6 @@ def begin_try(self): self.ilasm.begin_try() + + def clean_stack(self): + self.ilasm.clean_stack() Modified: pypy/dist/pypy/translator/js/js.py ============================================================================== --- pypy/dist/pypy/translator/js/js.py (original) +++ pypy/dist/pypy/translator/js/js.py Mon May 7 13:17:59 2007 @@ -101,6 +101,7 @@ self.gen_pendings() self.db.gen_constants(self.ilasm, self.db._pending_nodes) self.ilasm.close() + assert len(self.ilasm.right_hand) == 0 return self.tmpfile.strpath def write_source(self): Modified: pypy/dist/pypy/translator/js/jsbuiltin.py ============================================================================== --- pypy/dist/pypy/translator/js/jsbuiltin.py (original) +++ pypy/dist/pypy/translator/js/jsbuiltin.py Mon May 7 13:17:59 2007 @@ -1,55 +1,52 @@ """ JavaScript builtin mappings """ -from pypy.translator.oosupport.metavm import InstructionList, PushAllArgs +from pypy.translator.oosupport.metavm import InstructionList, PushAllArgs,\ + _PushAllArgs from pypy.translator.js.metavm import SetBuiltinField, ListGetitem, ListSetitem, \ GetBuiltinField, CallBuiltin, Call, SetTimeout, ListContains,\ - NewBuiltin, SetOnEvent + NewBuiltin, SetOnEvent, ListRemove, CallBuiltinMethod, _GetPredefinedField,\ + _SetPredefinedField from pypy.rpython.ootypesystem import ootype class _Builtins(object): def __init__(self): - list_resize = lambda g,op: SetBuiltinField.run_it(g, op.args[1], 'length', op.args[2]) + list_resize = _SetPredefinedField('length') self.builtin_map = { 'll_js_jseval' : CallBuiltin('eval'), 'set_on_keydown' : SetOnEvent('onkeydown'), 'set_on_keyup' : SetOnEvent('onkeyup'), 'setTimeout' : SetTimeout, - 'll_int_str' : lambda g,op: Call._render_builtin_method(g, 'toString' , [op.args[2]]), - 'll_strconcat' : InstructionList([PushAllArgs, '+']), + 'll_int_str' : CallBuiltinMethod('toString', [2]), + 'll_strconcat' : InstructionList([_PushAllArgs(slice(1, None)), '+']), 'll_int' : CallBuiltin('parseInt'), #'alert' : CallBuiltin('alert'), 'seval' : CallBuiltin('seval'), 'date': NewBuiltin('Date'), - 'll_math.ll_math_fmod' : InstructionList([PushAllArgs, '%']), + 'll_math.ll_math_fmod' : InstructionList([_PushAllArgs(slice(1, None)), '%']), 'll_time_time' : CallBuiltin('time'), 'll_time_clock' : CallBuiltin('clock'), 'll_os_write' : CallBuiltin('print'), } self.builtin_obj_map = { ootype.String.__class__: { - 'll_strconcat' : InstructionList([PushAllArgs, '+']), - 'll_strlen' : lambda g,op: GetBuiltinField.run_it(g, op.args[1], 'length'), - 'll_stritem_nonneg' : lambda g, op: Call._render_builtin_method(g, 'charAt', [op.args[1], op.args[2]]), - 'll_streq' : InstructionList([PushAllArgs, '==']), + 'll_strconcat' : InstructionList([_PushAllArgs(slice(1, None)), '+']), + 'll_strlen' : _GetPredefinedField('length'), + 'll_stritem_nonneg' : CallBuiltinMethod('charAt', slice(1,None)), + 'll_streq' : InstructionList([_PushAllArgs(slice(1, None)), '==']), 'll_strcmp' : CallBuiltin('strcmp'), 'll_startswith' : CallBuiltin('startswith'), 'll_endswith' : CallBuiltin('endswith'), 'll_split_chr' : CallBuiltin('splitchr'), - #'ll_substring' : lambda g,op: Call._render_builtin_method(g, 'substring', [op.args[1], op.args[2], op.args[3]]), 'll_substring' : CallBuiltin('substring'), - 'll_lower' : lambda g, op: Call._render_builtin_method(g, 'toLowerCase', [op.args[1]]), - 'll_upper' : lambda g, op: Call._render_builtin_method(g, 'toUpperCase', [op.args[1]]), + 'll_lower' : CallBuiltinMethod('toLowerCase', slice(1, None)), + 'll_upper' : CallBuiltinMethod('toUpperCase', slice(1, None)), 'll_find' : CallBuiltin('findIndexOf'), 'll_find_char' : CallBuiltin('findIndexOf'), - #'ll_find' : lambda g, op: Call._render_builtin_method(g, 'indexOf', [op.args[1], op.args[2], op.args[3]]), - #'ll_find_char' : lambda g, op: Call._render_builtin_method(g, 'indexOf', [op.args[1], op.args[2], op.args[3]]), 'll_contains' : CallBuiltin('findIndexOfTrue'), - 'll_replace_chr_chr' : lambda g, op: - Call._render_builtin_method(g, 'replace', - [op.args[1], op.args[2], op.args[3], 'g']), + 'll_replace_chr_chr' : CallBuiltinMethod('replace', slice(1, None), ['g']), 'll_count_char' : CallBuiltin('countCharOf'), 'll_count' : CallBuiltin('countOf'), }, @@ -59,7 +56,7 @@ '_ll_resize' : list_resize, '_ll_resize_ge' : list_resize, '_ll_resize_le' : list_resize, - 'll_length' : lambda g,op: GetBuiltinField.run_it(g, op.args[1], 'length'), + 'll_length' : _GetPredefinedField('length'), }, ootype.Dict: { 'll_get' : ListGetitem, @@ -67,7 +64,7 @@ 'll_contains' : ListContains, 'll_get_items_iterator' : CallBuiltin('dict_items_iterator'), 'll_length' : CallBuiltin('get_dict_len'), - 'll_remove' : lambda g, op: CallBuiltin('delete')._render_builtin_prepared_args(g, 'delete', ['%s[%s]' % (op.args[1], op.args[2])]), + 'll_remove' : ListRemove, 'll_clear': CallBuiltin('clear_dict'), }, ootype.Record: { @@ -76,5 +73,12 @@ 'll_contains' : ListContains, } } + self.fix_opcodes() + + def fix_opcodes(self): + from pypy.translator.js.metavm import fix_opcodes + #fix_opcodes(self.builtin_map) + #for value in self.builtin_obj_map.values(): + # fix_opcodes(value) Builtins = _Builtins() Modified: pypy/dist/pypy/translator/js/metavm.py ============================================================================== --- pypy/dist/pypy/translator/js/metavm.py (original) +++ pypy/dist/pypy/translator/js/metavm.py Mon May 7 13:17:59 2007 @@ -3,7 +3,7 @@ #from pypy.translator.js.jsbuiltin import Builtins from pypy.translator.oosupport.metavm import PushArg, PushAllArgs, StoreResult,\ - InstructionList, New, SetField, GetField, MicroInstruction + InstructionList, New, GetField, MicroInstruction from pypy.translator.js.log import log from pypy.rpython.ootypesystem import ootype @@ -82,6 +82,20 @@ def render(self, generator, op): self._render_builtin(generator, self.builtin, op.args) +class CallBuiltinMethod(_Call): + def __init__(self, builtin, slice=None, additional_args=[]): + self.builtin = builtin + self.slice = slice + self.additional_args = additional_args + + def render(self, generator, op): + if self.slice is not None: + args = op.args[self.slice] + else: + args = op.args + args += self.additional_args + self._render_builtin_method(generator, self.builtin, args) + class _SameAs(MicroInstruction): def render(self, generator, op): generator.change_name(op.result, op.args[0]) @@ -128,11 +142,17 @@ def render(self, generator, op): this = op.args[0] field = op.args[1].value[1:] - self.run_it(generator, this, field) - - def run_it(self, generator, this, field_name): generator.load(this) - generator.get_field(None, field_name) + generator.get_field(None, field) + +class _GetPredefinedField(MicroInstruction): + def __init__(self, field): + self.field = field + + def render(self, generator, op): + this = op.args[1] + generator.load(this) + generator.get_field(None, self.field) GetBuiltinField = _GetBuiltinField() @@ -151,6 +171,15 @@ generator.load(this) generator.load_special(value) generator.set_field(None, field_name) + +class _SetPredefinedField(_SetBuiltinField): + def __init__(self, field): + self.field = field + + def render(self, generator, op): + value = op.args[2] + this = op.args[1] + self.run_it(generator, this, self.field, value) class _SetExternalField(_SetBuiltinField): def render(self, generator, op): @@ -177,7 +206,7 @@ this = op.args[1].concretetype method = op.args[0] method_name = method.value - generator.load(op.args[1]) + #generator.load(op.args[1]) self._render_builtin_method(generator, method_name, op.args[1:]) CallBuiltinObject = _CallBuiltinObject() @@ -213,6 +242,10 @@ generator.load(op.args[2]) generator.call_external('setTimeout',[0]*2) +class _DiscardStack(MicroInstruction): + def render(self, generator, op): + generator.clean_stack() + class SetOnEvent(MicroInstruction): def __init__(self, field): self.field = field @@ -222,29 +255,23 @@ val = op.args[1].value val = val.concretize().value assert(isinstance(val, ootype._static_meth)) - #if isinstance(val, ootype.StaticMethod): real_name = val._name generator.db.pending_function(val.graph) - #generator.db.pending_function(val.graph) - #else: - # concrete = val.concretize() - # real_name = concrete.value._name - # generator.db.pending_function(concrete.value.graph) - #generator.load_str("'%s()'" % real_name) - #generator.load(op.args[2]) generator.load_str("document") generator.load_str(real_name) generator.set_field(None, self.field) - #generator.call_external('setTimeout',[0]*2) -##class _XmlSetCallback(MicroInstruction): -## # FIXME: Another dirty hack. To remove soon -## def render(self, generator, op): -## generator.load(op.args[2]) -## generator.load(op.args[1]) -## generator.set_field(None, 'onreadystatechange') -## -##XmlSetCallback = _XmlSetCallback() +class _CheckLength(MicroInstruction): + def render(self, generator, op): + assert not generator.ilasm.right_hand + +class _ListRemove(MicroInstruction): + def render(self, generator, op): + generator.list_getitem(op.args[1], op.args[2]) + generator.call_external('delete', [0]) + +ListRemove = _ListRemove() +CheckLength = _CheckLength() SetTimeout = _SetTimeout() IndirectCall = _IndirectCall() IsInstance = _IsInstance() @@ -252,3 +279,19 @@ CopyName = [PushAllArgs, _SameAs ()] CastString = _CastFun("convertToString", 1) SameAs = CopyName +DiscardStack = _DiscardStack() + +def fix_opcodes(opcodes): + for key, value in opcodes.iteritems(): + if type(value) is str: + value = InstructionList([PushAllArgs, value, StoreResult, CheckLength]) + elif value == []: + value = InstructionList([CheckLength]) + elif value is not None: + if StoreResult not in value: + value.append(StoreResult) + if CheckLength not in value: + value.append(CheckLength) + value = InstructionList(value) + + opcodes[key] = value Modified: pypy/dist/pypy/translator/js/opcodes.py ============================================================================== --- pypy/dist/pypy/translator/js/opcodes.py (original) +++ pypy/dist/pypy/translator/js/opcodes.py Mon May 7 13:17:59 2007 @@ -2,19 +2,21 @@ """ from pypy.translator.oosupport.metavm import PushArg, PushAllArgs, StoreResult,\ - InstructionList, New, SetField, GetField, MicroInstruction, RuntimeNew, PushPrimitive + InstructionList, New, GetField, MicroInstruction, RuntimeNew, PushPrimitive from pypy.translator.oosupport.metavm import _GetFieldDispatcher, _SetFieldDispatcher, \ - _CallDispatcher, _MethodDispatcher + _CallDispatcher, _MethodDispatcher, SetField -from pypy.translator.js.metavm import SameAs, IsInstance, Call, CallMethod, CopyName, CastString,\ - _Prefix, _CastFun, _NotImplemented, CallBuiltin, CallBuiltinObject, GetBuiltinField, SetBuiltinField,\ - IndirectCall, CallExternalObject, SetExternalField, _CastMethod, _LoadConst +from pypy.translator.js.metavm import IsInstance, Call, CallMethod,\ + CopyName, CastString, _Prefix, _CastFun, _NotImplemented, CallBuiltin,\ + CallBuiltinObject, GetBuiltinField, SetBuiltinField, IndirectCall,\ + CallExternalObject, SetExternalField, _CastMethod, _LoadConst,\ + DiscardStack, CheckLength, fix_opcodes from pypy.translator.js.jsbuiltin import Builtins from pypy.rpython.ootypesystem import ootype -DoNothing = [PushAllArgs] +DoNothing = [] from pypy.translator.js.log import log @@ -113,7 +115,7 @@ 'direct_call' : [_CallDispatcher(Builtins, class_map)], 'indirect_call' : [IndirectCall], - 'same_as' : SameAs, + 'same_as' : CopyName, 'new' : [New], 'runtimenew' : [RuntimeNew], 'instanceof' : [IsInstance], @@ -124,8 +126,8 @@ 'oosetfield' : [_SetFieldDispatcher(Builtins, class_map)], 'oogetfield' : [_GetFieldDispatcher(Builtins, class_map)], 'oosend' : [_MethodDispatcher(Builtins, class_map)], - 'ooupcast' : DoNothing, - 'oodowncast' : DoNothing, + 'ooupcast' : CopyName, + 'oodowncast' : CopyName, 'oononnull' : [PushAllArgs,_Prefix('!!')], 'oostring' : [PushArg(0),CastString], 'ooparse_int' : [PushAllArgs,_CastFun("parseInt",2)], @@ -153,12 +155,4 @@ 'is_early_constant': [PushPrimitive(ootype.Bool, False)], } -for key, value in opcodes.iteritems(): - if type(value) is str: - value = InstructionList([PushAllArgs, value, StoreResult]) - elif value is not None: - if StoreResult not in value: - value.append(StoreResult) - value = InstructionList(value) - - opcodes[key] = value +fix_opcodes(opcodes) From afa at codespeak.net Mon May 7 16:02:03 2007 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 7 May 2007 16:02:03 +0200 (CEST) Subject: [pypy-svn] r42801 - in pypy/dist/pypy: . module/posix/test Message-ID: <20070507140203.ECE098065@code0.codespeak.net> Author: afa Date: Mon May 7 16:02:03 2007 New Revision: 42801 Modified: pypy/dist/pypy/conftest.py pypy/dist/pypy/module/posix/test/test_posix2.py Log: Correct some tests in posix module: - updates for win32 - test_environ was empty. Re-add code deleted in rev31440, and correctly call space.startup() in all applevel tests. There were many other tests deleted by rev 31440. Will try to re-add them later. The missing startup() might be the reason why some socket/select tests fail on the win32 testbot: Error messages seem to imply that WSAStartup() was not called. It doesnt fail on my machines because "import socket" at the CPython level has the same effect. Does the testbot run applevel tests in a separate process? Modified: pypy/dist/pypy/conftest.py ============================================================================== --- pypy/dist/pypy/conftest.py (original) +++ pypy/dist/pypy/conftest.py Mon May 7 16:02:03 2007 @@ -59,6 +59,7 @@ traceback.print_exc() py.test.fail("fatal: cannot initialize objspace: %r" % (config.objspace.name,)) + space.startup() # Initialize all builtin modules space.setitem(space.builtin.w_dict, space.wrap('AssertionError'), appsupport.build_pytest_assertion(space)) space.setitem(space.builtin.w_dict, space.wrap('raises'), Modified: pypy/dist/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/dist/pypy/module/posix/test/test_posix2.py (original) +++ pypy/dist/pypy/module/posix/test/test_posix2.py Mon May 7 16:02:03 2007 @@ -105,7 +105,9 @@ assert posix.access(pdir, posix.R_OK) assert posix.access(pdir, posix.W_OK) - assert not posix.access(pdir, posix.X_OK) + import sys + if sys.platform != "win32": + assert not posix.access(pdir, posix.X_OK) def test_strerror(self): @@ -123,9 +125,11 @@ # XXX check status1 pass # <- please, inspect.getsource(), don't crash - if hasattr(__import__(os.name), "execv"): # and fork + if hasattr(__import__(os.name), "execv"): def test_execv(self): os = self.posix + if not hasattr(os, "fork"): + skip("Need fork() to test execv()") pid = os.fork() if pid == 0: os.execv("/usr/bin/env", ["env", "python", "-c", "open('onefile', 'w').write('1')"]) @@ -138,7 +142,7 @@ raises(OSError, 'os.execv("saddsadsadsadsa", ["saddsadsasaddsa"])') def test_execve(self): - skip("not implemented") + skip("Not implemented") os = self.posix pid = os.fork() if pid == 0: @@ -161,9 +165,14 @@ cls.w_posix = space.appexec([], "(): import %s as m ; return m" % os.name) cls.w_os = space.appexec([], "(): import os; return os") cls.w_path = space.wrap(str(path)) + def test_environ(self): posix = self.posix os = self.os + assert posix.environ['PATH'] + del posix.environ['PATH'] + def fn(): posix.environ['PATH'] + raises(KeyError, fn) if hasattr(__import__(os.name), "unsetenv"): def test_unsetenv_nonexisting(self): From fijal at codespeak.net Mon May 7 17:19:52 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 7 May 2007 17:19:52 +0200 (CEST) Subject: [pypy-svn] r42813 - pypy/dist/pypy/translator/js/examples Message-ID: <20070507151952.72EBE8065@code0.codespeak.net> Author: fijal Date: Mon May 7 17:19:52 2007 New Revision: 42813 Modified: pypy/dist/pypy/translator/js/examples/overmind.py Log: Ugh. missing import in error recovery Modified: pypy/dist/pypy/translator/js/examples/overmind.py ============================================================================== --- pypy/dist/pypy/translator/js/examples/overmind.py (original) +++ pypy/dist/pypy/translator/js/examples/overmind.py Mon May 7 17:19:52 2007 @@ -56,11 +56,13 @@ bnb.exposed = True def handle_error(self, exc, e_value, tb): + import traceback tb_formatted = '\n'.join(traceback.format_tb(tb)) + \ "%s: %s" % (exc, e_value) log_file = open("/tmp/play1_error_log", "a") log_file.write(tb_formatted) log_file.close() + print tb_formatted class Handler(server.NewHandler): application = Root() From arigo at codespeak.net Tue May 8 14:31:03 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 8 May 2007 14:31:03 +0200 (CEST) Subject: [pypy-svn] r42852 - pypy/extradoc/talk/dyla2007 Message-ID: <20070508123103.2D1348069@code0.codespeak.net> Author: arigo Date: Tue May 8 14:31:01 2007 New Revision: 42852 Added: pypy/extradoc/talk/dyla2007/ (props changed) pypy/extradoc/talk/dyla2007/draft.txt (contents, props changed) Log: A possible outline for a dyla2007 position paper. Added: pypy/extradoc/talk/dyla2007/draft.txt ============================================================================== --- (empty file) +++ pypy/extradoc/talk/dyla2007/draft.txt Tue May 8 14:31:01 2007 @@ -0,0 +1,112 @@ +~~~~~ +Draft +~~~~~ + + +Introduction +============ + +- Some kind of intro about dynamic language implementations + +- In the context of academic or Open Source communities: + limited resources + +- What is the best way to implement dynamic languages? + Current situation: + * interpreters only + * for C (Posix, Windows), as custom VMs + * for Java (common in academia) + * for .NET (recent) + Trend: + * more support from OO VMs, better integration + * standardizing on one OO VM to increase interoperability + and reduce redundancy and out-of-sync implementations of the dyn language + +- Our view on the issue: + * high-level VMs come with limitations + * a single OO VM for everybody is not a good idea + * this is really a metaprogramming issue, not a standardization issue + * implementations should be generated - don't write VMs by hand any more + + +The OO VM Arguments +=================== + +- Writing dynamic language implementations in an OO VM gives advantages + over writing custom VMs in C. Supposedly we get: + * more interoperability than the C level + * better tools + * better implementation of low-level issues like GC + * better performance + * ease of implementation + * a single base, which makes alternate implementations + unnecessary + (expand each topic with concrete supporting data...) + +- Some of these are only partially true: + * there is no single best base VM and unclear if there will ever be: + so far, all major dynamic languages have multiple implementations + (Perl has only a C/Posix impl) and it's probably a good thing + * "better performance": so far it's the other way around, and the + VM's JIT compilers don't help much the very dynamic languages (expand...) + * "better GCs" is unclear - obvious in theory, but OO VMs tend to + have quite large memory overheads to start with (ref...) + * "ease of implementation" disputable: it's really a trade-off: + it gives a higher-level language but also an imposed model in + which concepts must be mapped, which may be anywhere from easy + to mostly impossible depending on the language we want to implement + (ref... e.g. bad functional languages support for .NET?) + +- Not all arguments are bogus: we get better tools and GCs in theory, + and interoperability with the rest of the VM + + +Metaprogramming Is Good +======================= + +- Introduction to the generation of VMs from high-level interpreters + * PyPy proof of concept: can target many environments + * single source => multiple VMs + +- (Describe some of the PyPy architecture, ref...) + +- Makes interpreters easy to write, update, and generally experiment with + * expand... + * the requirement "write an interpreter for your language" is much + less strong than "fit it into the OO VM's model" [e.g. Pyrolog on .NET] + * transformations [Stackless] + +- We can get similarly good GCs and tools + * no easier or harder than what needs to be put in an OO VM + * existing GCs can also be reused + * a metaprogramming translation toolchain requires a lot of work, + but it is highly reusable + +- We can get better performance + * good baseline speed (e.g. pypy-c quite faster than Jython) + * JIT generation framework + * expand... + +- No need for VM standardization + * we reduce the need for alternative implementations by making + one source code enough for many environments + * even the translation framework need not be standardized, as + we could cross-translate high-level source code written for one + with another [e.g. steal the Jikes GC] + * might open the door to better solutions for interoperability + (high-level bridges instead of low-level ones, cross-translation...) + + +Conclusion +========== + +- Don't write dynamic language implementations "by hand" + * writing them more abstractly at a high level has mostly only advantages + +- Don't write VMs by hand any more + * certainly not language-specific VMs + * but even general-purpose OO VMs have trade-offs + +- Let's write more metaprogramming translation toolchains + * diversity is good + * ultimately a better investment of efforts than writing general-purpose VMs From cfbolz at codespeak.net Wed May 9 10:47:18 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 9 May 2007 10:47:18 +0200 (CEST) Subject: [pypy-svn] r42894 - pypy/extradoc/eu-report Message-ID: <20070509084718.E08918077@code0.codespeak.net> Author: cfbolz Date: Wed May 9 10:47:18 2007 New Revision: 42894 Modified: pypy/extradoc/eu-report/D07.1_Massive_Parallelism_and_Translation_Aspects-2007-02-28.pdf Log: just update report here (not perfectly nice, different date really, but not too much changed and I recently gave this link to someone) Modified: pypy/extradoc/eu-report/D07.1_Massive_Parallelism_and_Translation_Aspects-2007-02-28.pdf ============================================================================== Files pypy/extradoc/eu-report/D07.1_Massive_Parallelism_and_Translation_Aspects-2007-02-28.pdf (original) and pypy/extradoc/eu-report/D07.1_Massive_Parallelism_and_Translation_Aspects-2007-02-28.pdf Wed May 9 10:47:18 2007 differ From cfbolz at codespeak.net Wed May 9 10:57:06 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 9 May 2007 10:57:06 +0200 (CEST) Subject: [pypy-svn] r42898 - pypy/extradoc/eu-report Message-ID: <20070509085706.4501B8077@code0.codespeak.net> Author: cfbolz Date: Wed May 9 10:57:05 2007 New Revision: 42898 Modified: pypy/extradoc/eu-report/D13.1_Integration_and_Configuration-2007-03-30.pdf Log: update D13.1 Modified: pypy/extradoc/eu-report/D13.1_Integration_and_Configuration-2007-03-30.pdf ============================================================================== Binary files. No diff available. From cfbolz at codespeak.net Wed May 9 11:13:23 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 9 May 2007 11:13:23 +0200 (CEST) Subject: [pypy-svn] r42901 - pypy/extradoc/eu-report Message-ID: <20070509091323.581F58078@code0.codespeak.net> Author: cfbolz Date: Wed May 9 11:13:22 2007 New Revision: 42901 Modified: pypy/extradoc/eu-report/D14.4_Report_About_Milestone_Phase_3-2007-05-01.pdf Log: update 14.4 too Modified: pypy/extradoc/eu-report/D14.4_Report_About_Milestone_Phase_3-2007-05-01.pdf ============================================================================== Binary files. No diff available. From fijal at codespeak.net Wed May 9 14:51:11 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 9 May 2007 14:51:11 +0200 (CEST) Subject: [pypy-svn] r42943 - in pypy/dist/pypy/lib/distributed: . test Message-ID: <20070509125111.37062807B@code0.codespeak.net> Author: fijal Date: Wed May 9 14:51:10 2007 New Revision: 42943 Modified: pypy/dist/pypy/lib/distributed/protocol.py pypy/dist/pypy/lib/distributed/test/test_distributed.py Log: Possibility of listing keys Modified: pypy/dist/pypy/lib/distributed/protocol.py ============================================================================== --- pypy/dist/pypy/lib/distributed/protocol.py (original) +++ pypy/dist/pypy/lib/distributed/protocol.py Wed May 9 14:51:10 2007 @@ -335,6 +335,9 @@ value = protocol.unwrap(w_value) getattr(type(obj), name).__set__(obj, value) send(('finished', protocol.wrap(None))) + elif command == 'remote_keys': + keys = protocol.keeper.exported_names.keys() + send(('finished', protocol.wrap(keys))) else: raise NotImplementedError("command %s" % command) @@ -389,12 +392,15 @@ def get(self, name, obj, type): self.send(("desc_get", (name, self.wrap(obj), self.wrap(type)))) - retval = remote_loop(self) - return retval + return remote_loop(self) def set(self, obj, value): self.send(("desc_set", (name, self.wrap(obj), self.wrap(value)))) + def remote_keys(self): + self.send(("remote_keys",None)) + return remote_loop(self) + class RemoteObject(object): def __init__(self, protocol, id): self.id = id Modified: pypy/dist/pypy/lib/distributed/test/test_distributed.py ============================================================================== --- pypy/dist/pypy/lib/distributed/test/test_distributed.py (original) +++ pypy/dist/pypy/lib/distributed/test/test_distributed.py Wed May 9 14:51:10 2007 @@ -298,3 +298,7 @@ protocol = self.test_env({}) raises(ObjectNotFound, "protocol.get_remote('x')") + def test_list_items(self): + protocol = self.test_env({'x':3, 'y':8}) + assert sorted(protocol.remote_keys()) == ['x', 'y'] + From fijal at codespeak.net Wed May 9 20:37:02 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 9 May 2007 20:37:02 +0200 (CEST) Subject: [pypy-svn] r42966 - in pypy/dist/pypy/lib/distributed: . test Message-ID: <20070509183702.ED355807A@code0.codespeak.net> Author: fijal Date: Wed May 9 20:37:01 2007 New Revision: 42966 Added: pypy/dist/pypy/lib/distributed/support.py Modified: pypy/dist/pypy/lib/distributed/socklayer.py pypy/dist/pypy/lib/distributed/test/test_socklayer.py Log: Make slighlty better interface for lib/distributed Modified: pypy/dist/pypy/lib/distributed/socklayer.py ============================================================================== --- pypy/dist/pypy/lib/distributed/socklayer.py (original) +++ pypy/dist/pypy/lib/distributed/socklayer.py Wed May 9 20:37:01 2007 @@ -60,12 +60,13 @@ s = socket(AF_INET, SOCK_STREAM) print "Connecting %s" % (address,) s.connect(address) - + return SenderWrapper(s).send, ReceiverWrapper(s).receive def connect(address, socket=socket): + from distributed.support import get_remote_view from distributed import RemoteProtocol - return RemoteProtocol(*socket_connecter(address, socket)) + return get_remote_view(RemoteProtocol(*socket_connecter(address, socket))) def spawn_remote_side(code, gw): """ A very simple wrapper around greenexecnet to allow Added: pypy/dist/pypy/lib/distributed/support.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lib/distributed/support.py Wed May 9 20:37:01 2007 @@ -0,0 +1,14 @@ + +""" Some random support functions +""" + +def get_remote_view(protocol): + # this is dynamic to provide needed level of laziness + class RemoteView(object): + pass + + for key in protocol.remote_keys(): + getter = lambda self: protocol.get_remote(key) + setattr(RemoteView, key, property(getter)) + + return RemoteView() Modified: pypy/dist/pypy/lib/distributed/test/test_socklayer.py ============================================================================== --- pypy/dist/pypy/lib/distributed/test/test_socklayer.py (original) +++ pypy/dist/pypy/lib/distributed/test/test_socklayer.py Wed May 9 20:37:01 2007 @@ -27,6 +27,7 @@ def two(): rp = connect(('127.0.0.1', 21211), GreenSocket) - assert rp.get_remote('x').z == 3 + assert rp.x.z == 3 + assert [i for i in dir(rp) if not i.startswith('__')] == ['x'] oneof(one, two) From fijal at codespeak.net Wed May 9 20:56:27 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 9 May 2007 20:56:27 +0200 (CEST) Subject: [pypy-svn] r42967 - pypy/branch/lib-distributed-rewrite Message-ID: <20070509185627.B5EAF807A@code0.codespeak.net> Author: fijal Date: Wed May 9 20:56:26 2007 New Revision: 42967 Removed: pypy/branch/lib-distributed-rewrite/ Log: Delete merged branch From fijal at codespeak.net Wed May 9 21:17:20 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 9 May 2007 21:17:20 +0200 (CEST) Subject: [pypy-svn] r42968 - pypy/dist/pypy/translator/js/test Message-ID: <20070509191720.5F408807A@code0.codespeak.net> Author: fijal Date: Wed May 9 21:17:18 2007 New Revision: 42968 Removed: pypy/dist/pypy/translator/js/test/test_jseval.py Log: Entirely stupid idea for now (and this way!). From fijal at codespeak.net Wed May 9 21:18:09 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 9 May 2007 21:18:09 +0200 (CEST) Subject: [pypy-svn] r42969 - pypy/dist/pypy/rlib Message-ID: <20070509191809.60B11807B@code0.codespeak.net> Author: fijal Date: Wed May 9 21:18:08 2007 New Revision: 42969 Removed: pypy/dist/pypy/rlib/rjs.py Log: Some reminescence of old js backend From fijal at codespeak.net Wed May 9 21:19:41 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 9 May 2007 21:19:41 +0200 (CEST) Subject: [pypy-svn] r42971 - in pypy/dist/pypy/rpython: . module Message-ID: <20070509191941.53D63807A@code0.codespeak.net> Author: fijal Date: Wed May 9 21:19:41 2007 New Revision: 42971 Removed: pypy/dist/pypy/rpython/module/ll_js.py Modified: pypy/dist/pypy/rpython/extfunctable.py Log: Remove references to rjs Modified: pypy/dist/pypy/rpython/extfunctable.py ============================================================================== --- pypy/dist/pypy/rpython/extfunctable.py (original) +++ pypy/dist/pypy/rpython/extfunctable.py Wed May 9 21:19:41 2007 @@ -271,11 +271,6 @@ 'll_stackless/switch')) # ___________________________________________________________ -# javascript -from pypy.rlib import rjs -declare(rjs.jseval, str, 'll_js/jseval') - -# ___________________________________________________________ # the exceptions that can be implicitely raised by some operations standardexceptions = { TypeError : True, From fijal at codespeak.net Wed May 9 22:51:58 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 9 May 2007 22:51:58 +0200 (CEST) Subject: [pypy-svn] r42977 - in pypy/branch/pypy-string-formatting/rlib: . test Message-ID: <20070509205158.8DCBF807A@code0.codespeak.net> Author: fijal Date: Wed May 9 22:51:57 2007 New Revision: 42977 Added: pypy/branch/pypy-string-formatting/rlib/rcharbuilder.py pypy/branch/pypy-string-formatting/rlib/test/test_rcharbuilder.py Log: A char builder, which allows for keeping list of chars or repeated chars Added: pypy/branch/pypy-string-formatting/rlib/rcharbuilder.py ============================================================================== --- (empty file) +++ pypy/branch/pypy-string-formatting/rlib/rcharbuilder.py Wed May 9 22:51:57 2007 @@ -0,0 +1,55 @@ + +""" RPython-level implementation of string builder, +which can keep joined list of strings with additional +optimizations (like 'x' * 123 or 'xxx' * 444) +""" + +class AbstractElement: + pass + +class SingleChar(AbstractElement): + def __init__(self, c): + self.c = c + + def flatten(self): + return self.c + + def chars(self): + return [self.c] + +class MultipleChar(AbstractElement): + def __init__(self, c, count): + self.c = c + self.count = count + + def flatten(self): + return self.c * self.count + + def chars(self): + return [self.c] * self.count + +class rcharbuilder: + """ An implementation of special char-like builder, which + keeps list of substrings with additional informations. + + Need to be char-like to keep eventual unicode characters :-( + """ + def __init__(self): + self.l = [] + + def append_char(self, c): + self.l.append(SingleChar(c)) + + def append_multiple_char(self, c, count): + self.l.append(MultipleChar(c, count)) + + def flatten(self): + return "".join([i.flatten() for i in self.l]) + + def chars(self): + # XXX unicode literals in RPython anyone? + l = [] + for i in self.l: + l += i.chars() + return l + Added: pypy/branch/pypy-string-formatting/rlib/test/test_rcharbuilder.py ============================================================================== --- (empty file) +++ pypy/branch/pypy-string-formatting/rlib/test/test_rcharbuilder.py Wed May 9 22:51:57 2007 @@ -0,0 +1,18 @@ + +from pypy.rlib.rcharbuilder import rcharbuilder + +class TestRStringBuilder: + def test_simple(self): + sb = rcharbuilder() + sb.append_char('x') + sb.append_char('y') + assert sb.flatten() == 'xy' + assert sb.chars() == ['x', 'y'] + + def test_multiple(self): + sb = rcharbuilder() + sb.append_char('x') + sb.append_multiple_char('z', 3) + sb.append_multiple_char('x', 4) + assert sb.flatten() == 'xzzzxxxx' + assert sb.chars() == ['x', 'z', 'z', 'z', 'x', 'x', 'x', 'x'] From fijal at codespeak.net Wed May 9 22:57:58 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 9 May 2007 22:57:58 +0200 (CEST) Subject: [pypy-svn] r42979 - in pypy/branch/pypy-string-formatting/rlib: . test Message-ID: <20070509205758.A173D807A@code0.codespeak.net> Author: fijal Date: Wed May 9 22:57:58 2007 New Revision: 42979 Removed: pypy/branch/pypy-string-formatting/rlib/rcharbuilder.py pypy/branch/pypy-string-formatting/rlib/test/test_rcharbuilder.py Log: Let them go From jlg at codespeak.net Thu May 10 15:26:02 2007 From: jlg at codespeak.net (jlg at codespeak.net) Date: Thu, 10 May 2007 15:26:02 +0200 (CEST) Subject: [pypy-svn] r43031 - in pypy/dist/pypy/lang/scheme: . test Message-ID: <20070510132602.3D8D08078@code0.codespeak.net> Author: jlg Date: Thu May 10 15:26:01 2007 New Revision: 43031 Added: pypy/dist/pypy/lang/scheme/ (props changed) pypy/dist/pypy/lang/scheme/__init__.py (contents, props changed) pypy/dist/pypy/lang/scheme/autopath.py (contents, props changed) pypy/dist/pypy/lang/scheme/test/ (props changed) pypy/dist/pypy/lang/scheme/test/__init__.py (contents, props changed) Log: Added pypy/lang/scheme tree Added: pypy/dist/pypy/lang/scheme/__init__.py ============================================================================== Added: pypy/dist/pypy/lang/scheme/autopath.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/scheme/autopath.py Thu May 10 15:26:01 2007 @@ -0,0 +1,114 @@ +""" +self cloning, automatic path configuration + +copy this into any subdirectory of pypy from which scripts need +to be run, typically all of the test subdirs. +The idea is that any such script simply issues + + import autopath + +and this will make sure that the parent directory containing "pypy" +is in sys.path. + +If you modify the master "autopath.py" version (in pypy/tool/autopath.py) +you can directly run it which will copy itself on all autopath.py files +it finds under the pypy root directory. + +This module always provides these attributes: + + pypydir pypy root directory path + this_dir directory where this autopath.py resides + +""" + + +def __dirinfo(part): + """ return (partdir, this_dir) and insert parent of partdir + into sys.path. If the parent directories don't have the part + an EnvironmentError is raised.""" + + import sys, os + try: + head = this_dir = os.path.realpath(os.path.dirname(__file__)) + except NameError: + head = this_dir = os.path.realpath(os.path.dirname(sys.argv[0])) + + while head: + partdir = head + head, tail = os.path.split(head) + if tail == part: + break + else: + raise EnvironmentError, "'%s' missing in '%r'" % (partdir, this_dir) + + pypy_root = os.path.join(head, '') + try: + sys.path.remove(head) + except ValueError: + pass + sys.path.insert(0, head) + + munged = {} + for name, mod in sys.modules.items(): + if '.' in name: + continue + fn = getattr(mod, '__file__', None) + if not isinstance(fn, str): + continue + newname = os.path.splitext(os.path.basename(fn))[0] + if not newname.startswith(part + '.'): + continue + path = os.path.join(os.path.dirname(os.path.realpath(fn)), '') + if path.startswith(pypy_root) and newname != part: + modpaths = os.path.normpath(path[len(pypy_root):]).split(os.sep) + if newname != '__init__': + modpaths.append(newname) + modpath = '.'.join(modpaths) + if modpath not in sys.modules: + munged[modpath] = mod + + for name, mod in munged.iteritems(): + if name not in sys.modules: + sys.modules[name] = mod + if '.' in name: + prename = name[:name.rfind('.')] + postname = name[len(prename)+1:] + if prename not in sys.modules: + __import__(prename) + if not hasattr(sys.modules[prename], postname): + setattr(sys.modules[prename], postname, mod) + + return partdir, this_dir + +def __clone(): + """ clone master version of autopath.py into all subdirs """ + from os.path import join, walk + if not this_dir.endswith(join('pypy','tool')): + raise EnvironmentError("can only clone master version " + "'%s'" % join(pypydir, 'tool',_myname)) + + + def sync_walker(arg, dirname, fnames): + if _myname in fnames: + fn = join(dirname, _myname) + f = open(fn, 'rwb+') + try: + if f.read() == arg: + print "checkok", fn + else: + print "syncing", fn + f = open(fn, 'w') + f.write(arg) + finally: + f.close() + s = open(join(pypydir, 'tool', _myname), 'rb').read() + walk(pypydir, sync_walker, s) + +_myname = 'autopath.py' + +# set guaranteed attributes + +pypydir, this_dir = __dirinfo('pypy') + +if __name__ == '__main__': + __clone() Added: pypy/dist/pypy/lang/scheme/test/__init__.py ============================================================================== From busemann at codespeak.net Thu May 10 17:51:44 2007 From: busemann at codespeak.net (busemann at codespeak.net) Date: Thu, 10 May 2007 17:51:44 +0200 (CEST) Subject: [pypy-svn] r43065 - pypy/dist/pypy/lib/pyontology/test Message-ID: <20070510155144.793B78080@code0.codespeak.net> Author: busemann Date: Thu May 10 17:51:43 2007 New Revision: 43065 Modified: pypy/dist/pypy/lib/pyontology/test/test_sparql.py Log: Working SPARQL queries Modified: pypy/dist/pypy/lib/pyontology/test/test_sparql.py ============================================================================== --- pypy/dist/pypy/lib/pyontology/test/test_sparql.py (original) +++ pypy/dist/pypy/lib/pyontology/test/test_sparql.py Thu May 10 17:51:43 2007 @@ -69,7 +69,8 @@ assert len(res[0]) == len(res[1]) == 4 assert res[0][1] in ['http://example.org/ns#p', 'http://example.org/ns#q'] assert res[1][1] in ['http://example.org/ns#p', 'http://example.org/ns#q'] - assert result[3].formula == 'query_x<2' + # assert result[3].formula == 'query_x<2' + assert (['query_x<2'], {}).formula == 'query_x<2' # There are 8 ways of having the triples in the query, if predicate is not a builtin owl predicate # @@ -220,7 +221,7 @@ O.attach_fd() res = O.sparql(query_x) - assert len(res) == 1 + assert len(res) == 8 assert res[0]['x'] == u'http://www.lt-world.org/ltw.owl#obj_61128' query_y = """ @@ -274,7 +275,7 @@ O.attach_fd() res = O.sparql(query1_a) - assert len(res) == 1 + assert len(res) == 81 def test_query1_b(): O = Ontology() @@ -284,15 +285,45 @@ res = O.sparql(query1_b) assert len(res) == 1 + +#List of people +query0a = """ + PREFIX ltw : + PREFIX rdf : + PREFIX owl : + SELECT ?person + WHERE { + ?person_obj rdf:type ltw:Active_Person . + ?person_obj ltw:personName ?person . + }""" + +def test_query0a(): + O = Ontology() + O.add_file(datapath("testont2.rdf"), 'xml') + O.attach_fd() + + query_vars = ['person'] + res = O.sparql(query0a) + for result in res: + + for v in query_vars: + print "%s:" % v, result[v], + print + + assert len(res) == 2 + + + query1 = """ PREFIX ltw : PREFIX owl : PREFIX rdf : SELECT ?person ?activity WHERE { - ?activity rdf:type ltw:Active_Project . + ?activity_obj rdf:type ltw:Active_Project . ?person_obj rdf:type ltw:Active_Person . - ?activity ltw:hasParticipant ?person_obj . + ?activity_obj ltw:hasParticipant ?person_obj . + ?activity_obj ltw:projectName ?activity . ?person_obj ltw:personName ?person . } ORDER BY ?person""" @@ -368,19 +399,25 @@ O.add_file(datapath("testont2.rdf"), 'xml') O.attach_fd() + query_vars = ['person','activity'] res = O.sparql(query1) + for result in res: + for v in query_vars: + print "%s:" % v, result[v], + print + assert len(res) == 1 - assert res[0]['activity'] == u'http://www.lt-world.org/ltw.owl#obj_59754' - assert res[0]['person'] == u'\nKlara Vicsi' + assert res[0]['person'] == u'\nKlara Vicsi' + def test_query_2(): - py.test.skip("Needs special care for dateTime type") + # py.test.skip("Needs special care for dateTime type") O = Ontology() O.add_file(datapath("testont2.rdf"), 'xml') O.attach_fd() res = O.sparql(query2) - assert len(res) == 1 + assert len(res) == 3 assert res[0]['project'] == u'http://www.lt-world.org/ltw.owl#obj_59754' assert res[0]['date_begin'] == datetime.date(1998,9,1) From fijal at codespeak.net Thu May 10 23:19:54 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 10 May 2007 23:19:54 +0200 (CEST) Subject: [pypy-svn] r43108 - pypy/dist/pypy/lib/distributed/demo Message-ID: <20070510211954.03AAA808E@code0.codespeak.net> Author: fijal Date: Thu May 10 23:19:54 2007 New Revision: 43108 Modified: pypy/dist/pypy/lib/distributed/demo/fileclient.py Log: Adapt demo to new API Modified: pypy/dist/pypy/lib/distributed/demo/fileclient.py ============================================================================== --- pypy/dist/pypy/lib/distributed/demo/fileclient.py (original) +++ pypy/dist/pypy/lib/distributed/demo/fileclient.py Thu May 10 23:19:54 2007 @@ -8,7 +8,7 @@ PORT = 12221 from distributed.socklayer import connect -file_opener = connect((HOST, PORT)).get_remote('open') +file_opener = connect((HOST, PORT)).open # now you can do for example file_opener('/etc/passwd').read() to # read remote /etc/passwd From fijal at codespeak.net Fri May 11 15:46:57 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 11 May 2007 15:46:57 +0200 (CEST) Subject: [pypy-svn] r43199 - in pypy/dist/pypy/lang/js/test/ecma: Boolean/CVS ExecutionContexts/CVS Expressions/CVS FunctionObjects/CVS GlobalObject/CVS LexicalConventions/CVS Math/CVS NativeObjects/CVS Number/CVS ObjectObjects/CVS SourceText/CVS Statements/CVS String/CVS TypeConversion/CVS Types/CVS Message-ID: <20070511134657.5EB938096@code0.codespeak.net> Author: fijal Date: Fri May 11 15:46:57 2007 New Revision: 43199 Removed: pypy/dist/pypy/lang/js/test/ecma/Boolean/CVS/ pypy/dist/pypy/lang/js/test/ecma/ExecutionContexts/CVS/ pypy/dist/pypy/lang/js/test/ecma/Expressions/CVS/ pypy/dist/pypy/lang/js/test/ecma/FunctionObjects/CVS/ pypy/dist/pypy/lang/js/test/ecma/GlobalObject/CVS/ pypy/dist/pypy/lang/js/test/ecma/LexicalConventions/CVS/ pypy/dist/pypy/lang/js/test/ecma/Math/CVS/ pypy/dist/pypy/lang/js/test/ecma/NativeObjects/CVS/ pypy/dist/pypy/lang/js/test/ecma/Number/CVS/ pypy/dist/pypy/lang/js/test/ecma/ObjectObjects/CVS/ pypy/dist/pypy/lang/js/test/ecma/SourceText/CVS/ pypy/dist/pypy/lang/js/test/ecma/Statements/CVS/ pypy/dist/pypy/lang/js/test/ecma/String/CVS/ pypy/dist/pypy/lang/js/test/ecma/TypeConversion/CVS/ pypy/dist/pypy/lang/js/test/ecma/Types/CVS/ Log: Remove CVS directories from ecma tests From fijal at codespeak.net Fri May 11 16:03:32 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 11 May 2007 16:03:32 +0200 (CEST) Subject: [pypy-svn] r43203 - pypy/dist/pypy/doc/discussion Message-ID: <20070511140332.376848096@code0.codespeak.net> Author: fijal Date: Fri May 11 16:03:31 2007 New Revision: 43203 Added: pypy/dist/pypy/doc/discussion/compiled-swamp.txt Log: Add a document not to forget swamp of compiled pypy-cs Added: pypy/dist/pypy/doc/discussion/compiled-swamp.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/doc/discussion/compiled-swamp.txt Fri May 11 16:03:31 2007 @@ -0,0 +1,14 @@ + +We've got huge swamp of compiled pypy-c's used for: + +* benchmarks +* tests +* compliancy tests +* play1 +* downloads +* ... + +We've got build tool, which we don't use, etc. etc. + +Idea is to formalize it more or less, so we'll have single script +to make all of this work, upload builds to the web page etc. From cfbolz at codespeak.net Fri May 11 16:05:52 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 11 May 2007 16:05:52 +0200 (CEST) Subject: [pypy-svn] r43204 - pypy/extradoc/eu-report Message-ID: <20070511140552.5C98E8096@code0.codespeak.net> Author: cfbolz Date: Fri May 11 16:05:51 2007 New Revision: 43204 Added: pypy/extradoc/eu-report/D01.1-Create_QA_plan_for_the_project.pdf (contents, props changed) Log: add D01.1 too Added: pypy/extradoc/eu-report/D01.1-Create_QA_plan_for_the_project.pdf ============================================================================== Binary file. No diff available. From fijal at codespeak.net Fri May 11 18:31:30 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 11 May 2007 18:31:30 +0200 (CEST) Subject: [pypy-svn] r43222 - in pypy/dist/lib-python/2.4.1: CVS bsddb/CVS bsddb/test/CVS compiler/CVS curses/CVS distutils/CVS distutils/command/CVS distutils/tests/CVS email/CVS email/test/CVS email/test/data/CVS encodings/CVS hotshot/CVS idlelib/CVS idlelib/Icons/CVS lib-old/CVS lib-tk/CVS logging/CVS plat-aix3/CVS plat-aix4/CVS plat-atheos/CVS plat-beos5/CVS plat-darwin/CVS plat-freebsd2/CVS plat-freebsd3/CVS plat-freebsd4/CVS plat-freebsd5/CVS plat-freebsd6/CVS plat-generic/CVS plat-irix5/CVS plat-irix6/CVS plat-linux2/CVS plat-mac/CVS plat-mac/Carbon/CVS plat-mac/lib-scriptpackages/CVS plat-mac/lib-scriptpackages/CodeWarrior/CVS plat-mac/lib-scriptpackages/Explorer/CVS plat-mac/lib-scriptpackages/Finder/CVS plat-mac/lib-scriptpackages/Netscape/CVS plat-mac/lib-scriptpackages/StdSuites/CVS plat-mac/lib-scriptpackages/SystemEvents/CVS plat-mac/lib-scriptpackages/Terminal/CVS plat-mac/lib-scriptpackages/_builtinSuites/CVS plat-netbsd1/CVS plat-next3/CVS plat-os2emx/CVS plat-riscos/CVS plat-sunos5/CVS plat-unixware7/CVS site-packages/CVS test test/CVS test/decimaltestdata/CVS test/output/CVS xml/CVS xml/dom/CVS xml/parsers/CVS xml/sax/CVS Message-ID: <20070511163130.DB2778095@code0.codespeak.net> Author: fijal Date: Fri May 11 18:31:30 2007 New Revision: 43222 Removed: pypy/dist/lib-python/2.4.1/CVS/ pypy/dist/lib-python/2.4.1/bsddb/CVS/ pypy/dist/lib-python/2.4.1/bsddb/test/CVS/ pypy/dist/lib-python/2.4.1/compiler/CVS/ pypy/dist/lib-python/2.4.1/curses/CVS/ pypy/dist/lib-python/2.4.1/distutils/CVS/ pypy/dist/lib-python/2.4.1/distutils/command/CVS/ pypy/dist/lib-python/2.4.1/distutils/tests/CVS/ pypy/dist/lib-python/2.4.1/email/CVS/ pypy/dist/lib-python/2.4.1/email/test/CVS/ pypy/dist/lib-python/2.4.1/email/test/data/CVS/ pypy/dist/lib-python/2.4.1/encodings/CVS/ pypy/dist/lib-python/2.4.1/hotshot/CVS/ pypy/dist/lib-python/2.4.1/idlelib/CVS/ pypy/dist/lib-python/2.4.1/idlelib/Icons/CVS/ pypy/dist/lib-python/2.4.1/lib-old/CVS/ pypy/dist/lib-python/2.4.1/lib-tk/CVS/ pypy/dist/lib-python/2.4.1/logging/CVS/ pypy/dist/lib-python/2.4.1/plat-aix3/CVS/ pypy/dist/lib-python/2.4.1/plat-aix4/CVS/ pypy/dist/lib-python/2.4.1/plat-atheos/CVS/ pypy/dist/lib-python/2.4.1/plat-beos5/CVS/ pypy/dist/lib-python/2.4.1/plat-darwin/CVS/ pypy/dist/lib-python/2.4.1/plat-freebsd2/CVS/ pypy/dist/lib-python/2.4.1/plat-freebsd3/CVS/ pypy/dist/lib-python/2.4.1/plat-freebsd4/CVS/ pypy/dist/lib-python/2.4.1/plat-freebsd5/CVS/ pypy/dist/lib-python/2.4.1/plat-freebsd6/CVS/ pypy/dist/lib-python/2.4.1/plat-generic/CVS/ pypy/dist/lib-python/2.4.1/plat-irix5/CVS/ pypy/dist/lib-python/2.4.1/plat-irix6/CVS/ pypy/dist/lib-python/2.4.1/plat-linux2/CVS/ pypy/dist/lib-python/2.4.1/plat-mac/CVS/ pypy/dist/lib-python/2.4.1/plat-mac/Carbon/CVS/ pypy/dist/lib-python/2.4.1/plat-mac/lib-scriptpackages/CVS/ pypy/dist/lib-python/2.4.1/plat-mac/lib-scriptpackages/CodeWarrior/CVS/ pypy/dist/lib-python/2.4.1/plat-mac/lib-scriptpackages/Explorer/CVS/ pypy/dist/lib-python/2.4.1/plat-mac/lib-scriptpackages/Finder/CVS/ pypy/dist/lib-python/2.4.1/plat-mac/lib-scriptpackages/Netscape/CVS/ pypy/dist/lib-python/2.4.1/plat-mac/lib-scriptpackages/StdSuites/CVS/ pypy/dist/lib-python/2.4.1/plat-mac/lib-scriptpackages/SystemEvents/CVS/ pypy/dist/lib-python/2.4.1/plat-mac/lib-scriptpackages/Terminal/CVS/ pypy/dist/lib-python/2.4.1/plat-mac/lib-scriptpackages/_builtinSuites/CVS/ pypy/dist/lib-python/2.4.1/plat-netbsd1/CVS/ pypy/dist/lib-python/2.4.1/plat-next3/CVS/ pypy/dist/lib-python/2.4.1/plat-os2emx/CVS/ pypy/dist/lib-python/2.4.1/plat-riscos/CVS/ pypy/dist/lib-python/2.4.1/plat-sunos5/CVS/ pypy/dist/lib-python/2.4.1/plat-unixware7/CVS/ pypy/dist/lib-python/2.4.1/site-packages/CVS/ pypy/dist/lib-python/2.4.1/test/CVS/ pypy/dist/lib-python/2.4.1/test/decimaltestdata/CVS/ pypy/dist/lib-python/2.4.1/test/output/CVS/ pypy/dist/lib-python/2.4.1/xml/CVS/ pypy/dist/lib-python/2.4.1/xml/dom/CVS/ pypy/dist/lib-python/2.4.1/xml/parsers/CVS/ pypy/dist/lib-python/2.4.1/xml/sax/CVS/ Modified: pypy/dist/lib-python/2.4.1/test/test_format.py Log: Remove CVS directories Modified: pypy/dist/lib-python/2.4.1/test/test_format.py ============================================================================== --- pypy/dist/lib-python/2.4.1/test/test_format.py (original) +++ pypy/dist/lib-python/2.4.1/test/test_format.py Fri May 11 18:31:30 2007 @@ -37,7 +37,6 @@ if have_unicode: testformat(unicode(formatstr), *args) - testboth("%.1d", (1,), "1") testboth("%.*d", (sys.maxint,1)) # expect overflow testboth("%.100d", (1,), '0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001') From fijal at codespeak.net Fri May 11 18:38:00 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 11 May 2007 18:38:00 +0200 (CEST) Subject: [pypy-svn] r43223 - pypy/dist/lib-python/2.4.1/test Message-ID: <20070511163800.D4062809B@code0.codespeak.net> Author: fijal Date: Fri May 11 18:38:00 2007 New Revision: 43223 Added: pypy/dist/lib-python/2.4.1/test/test_format.py - copied unchanged from r43211, pypy/dist/lib-python/2.4.1/test/test_format.py Log: Revert last checkin From arigo at codespeak.net Fri May 11 18:57:25 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 11 May 2007 18:57:25 +0200 (CEST) Subject: [pypy-svn] r43228 - pypy/dist/lib-python/modified-2.4.1/encodings/CVS Message-ID: <20070511165725.104428092@code0.codespeak.net> Author: arigo Date: Fri May 11 18:57:24 2007 New Revision: 43228 Removed: pypy/dist/lib-python/modified-2.4.1/encodings/CVS/ Log: Remove this CVS directory too. From cfbolz at codespeak.net Fri May 11 21:24:39 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 11 May 2007 21:24:39 +0200 (CEST) Subject: [pypy-svn] r43258 - pypy/extradoc/eu-report Message-ID: <20070511192439.86A9D8065@code0.codespeak.net> Author: cfbolz Date: Fri May 11 21:24:39 2007 New Revision: 43258 Added: pypy/extradoc/eu-report/D09.1_Constraint_Solving_and_Semantic_Web-2007-05-11.pdf (contents, props changed) Log: final version of D09.1 Added: pypy/extradoc/eu-report/D09.1_Constraint_Solving_and_Semantic_Web-2007-05-11.pdf ============================================================================== Binary file. No diff available. From cfbolz at codespeak.net Fri May 11 21:31:46 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 11 May 2007 21:31:46 +0200 (CEST) Subject: [pypy-svn] r43259 - pypy/dist/pypy/doc Message-ID: <20070511193146.3DE668069@code0.codespeak.net> Author: cfbolz Date: Fri May 11 21:31:46 2007 New Revision: 43259 Modified: pypy/dist/pypy/doc/index-report.txt Log: update the link Modified: pypy/dist/pypy/doc/index-report.txt ============================================================================== --- pypy/dist/pypy/doc/index-report.txt (original) +++ pypy/dist/pypy/doc/index-report.txt Fri May 11 21:31:46 2007 @@ -12,6 +12,11 @@ Reports of 2007 =============== +`D09.1 Constraint Solving and Semantic Web`_ is a report about PyPy's logic +programming and constraint solving features, as well as the work going on to +tie semantic web technologies and PyPy together. *(2007-05-11)* + + `D14.4 PyPy-1.0 Milestone report`_ (for language developers and researchers) summarizes research & technical results of the PyPy-1.0 release and discusses related development process and community aspects. *(2007-05-01)* @@ -78,11 +83,6 @@ (stackless) features. This report refers to the paper `PyPy's approach to virtual machine construction`_. *(2007-02-28)* -`D09.1 Constraint Solving and Semantic Web`_ is an interim version of a -report about PyPy's logic programming and constraint solving features, as well -as the work going on to tie semantic web technologies and PyPy together. All -feedback for it is welcome. *(2007-02-28)* - .. _`py-lib`: http://codespeak.net/py/current/doc/ @@ -146,7 +146,7 @@ .. _`D07.1 Massive Parallelism and Translation Aspects`: http://codespeak.net/pypy/extradoc/eu-report/D07.1_Massive_Parallelism_and_Translation_Aspects-2007-02-28.pdf .. _`D08.2 JIT Compiler Architecture`: http://codespeak.net/pypy/extradoc/eu-report/D08.2_JIT_Compiler_Architecture-2007-05-01.pdf .. _`D08.1 JIT Compiler Release`: http://codespeak.net/pypy/extradoc/eu-report/D08.1_JIT_Compiler_Release-2007-04-30.pdf -.. _`D09.1 Constraint Solving and Semantic Web`: http://codespeak.net/pypy/extradoc/eu-report/D09.1_Constraint_Solving_and_Semantic_Web-interim-2007-02-28.pdf +.. _`D09.1 Constraint Solving and Semantic Web`: http://codespeak.net/pypy/extradoc/eu-report/D09.1_Constraint_Solving_and_Semantic_Web-2007-05-11.pdf .. _`D10.1 Aspect-Oriented, Design-by-Contract Programming and RPython static checking`: http://codespeak.net/pypy/extradoc/eu-report/D10.1_Aspect_Oriented_Programming_in_PyPy-2007-03-22.pdf .. _`D11.1 PyPy for Embedded Devices`: http://codespeak.net/pypy/extradoc/eu-report/D11.1_PyPy_for_Embedded_Devices-2007-03-26.pdf .. _`D12.1 High-Level-Backends and Feature Prototypes`: http://codespeak.net/pypy/extradoc/eu-report/D12.1_H-L-Backends_and_Feature_Prototypes-2007-03-22.pdf From hpk at codespeak.net Sat May 12 10:51:55 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Sat, 12 May 2007 10:51:55 +0200 (CEST) Subject: [pypy-svn] r43266 - pypy/extradoc/talk/dyla2007 Message-ID: <20070512085155.A64928078@code0.codespeak.net> Author: hpk Date: Sat May 12 10:51:55 2007 New Revision: 43266 Modified: pypy/extradoc/talk/dyla2007/draft.txt Log: some expansion ideas Modified: pypy/extradoc/talk/dyla2007/draft.txt ============================================================================== --- pypy/extradoc/talk/dyla2007/draft.txt (original) +++ pypy/extradoc/talk/dyla2007/draft.txt Sat May 12 10:51:55 2007 @@ -23,7 +23,7 @@ and reduce redundancy and out-of-sync implementations of the dyn language - Our view on the issue: - * high-level VMs come with limitations + * high-level VMs come with limitations * a single OO VM for everybody is not a good idea * this is really a metaprogramming issue, not a standardization issue * implementations should be generated - don't write VMs by hand any more @@ -68,11 +68,17 @@ * PyPy proof of concept: can target many environments * single source => multiple VMs -- (Describe some of the PyPy architecture, ref...) +- PyPy architecture in metaprogramming terms: + * very expressive object language (RPython) for language VMs and semantics + * very expressive metalanguage (Python) for analysis and susccessive transformation + * SSI for representing VM spec on meta level + * transformations add aspects and specialize to fit + virtual or hardware runtime environment - Makes interpreters easy to write, update, and generally experiment with - * expand... - * the requirement "write an interpreter for your language" is much + * more expressiveness helps on all levels (use d12's + security references and texts) + * the requirement "specify an interpreter for your language" is much less strong than "fit it into the OO VM's model" [e.g. Pyrolog on .NET] * transformations [Stackless] @@ -108,5 +114,15 @@ * but even general-purpose OO VMs have trade-offs - Let's write more metaprogramming translation toolchains - * diversity is good + * diversity is good (high level SSI good as starting base) * ultimately a better investment of efforts than writing general-purpose VMs + + +References +------------- + +DLS paper + +expressiveness: couldn't find a classification scheme for computer languages, but see: + http://en.wikipedia.org/wiki/Comparison_of_programming_languages#Expressiveness + From arigo at codespeak.net Sat May 12 12:29:35 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 12 May 2007 12:29:35 +0200 (CEST) Subject: [pypy-svn] r43276 - in pypy/branch/pypy-string-formatting/rpython: . test Message-ID: <20070512102935.72BB6807A@code0.codespeak.net> Author: arigo Date: Sat May 12 12:29:35 2007 New Revision: 43276 Modified: pypy/branch/pypy-string-formatting/rpython/rlist.py pypy/branch/pypy-string-formatting/rpython/test/test_rlist.py Log: Support for "lst += string" in RPython. The idea is to special-case and support efficiently a few combined operations too: "lst += string[x:y]" and "lst += char * length". Modified: pypy/branch/pypy-string-formatting/rpython/rlist.py ============================================================================== --- pypy/branch/pypy-string-formatting/rpython/rlist.py (original) +++ pypy/branch/pypy-string-formatting/rpython/rlist.py Sat May 12 12:29:35 2007 @@ -4,6 +4,7 @@ from pypy.rpython.error import TyperError from pypy.rpython.rmodel import Repr, IteratorRepr, IntegerRepr, inputconst from pypy.rpython.rslice import AbstractSliceRepr +from pypy.rpython.rstr import AbstractStringRepr from pypy.rpython.lltypesystem.lltype import typeOf, Ptr, Void, Signed, Bool from pypy.rpython.lltypesystem.lltype import nullptr, Char, UniChar from pypy.rpython import robject @@ -350,6 +351,17 @@ hop.gendirectcall(ll_extend, v_lst1, v_lst2) return v_lst1 +class __extend__(pairtype(AbstractListRepr, AbstractStringRepr)): + + def rtype_inplace_add((r_lst1, r_str2), hop): + string_repr = r_lst1.rtyper.type_system.rstr.string_repr + v_lst1, v_str2 = hop.inputargs(r_lst1, string_repr) + c_strlen = hop.inputconst(Void, string_repr.ll.ll_strlen) + c_stritem = hop.inputconst(Void, string_repr.ll.ll_stritem_nonneg) + hop.gendirectcall(ll_extend_with_str, v_lst1, v_str2, + c_strlen, c_stritem) + return v_lst1 + class __extend__(pairtype(AbstractBaseListRepr, AbstractSliceRepr)): @@ -692,6 +704,18 @@ i += 1 j += 1 +def ll_extend_with_str(lst, s, getstrlen, getstritem): + len1 = lst.ll_length() + len2 = getstrlen(s) + newlength = len1 + len2 + lst._ll_resize_ge(newlength) + i = 0 + j = len1 + while i < len2: + lst.ll_setitem_fast(j, getstritem(s, i)) + i += 1 + j += 1 + def ll_listslice_startonly(RESLIST, l1, start): len1 = l1.ll_length() debug_assert(start >= 0, "unexpectedly negative list slice start") Modified: pypy/branch/pypy-string-formatting/rpython/test/test_rlist.py ============================================================================== --- pypy/branch/pypy-string-formatting/rpython/test/test_rlist.py (original) +++ pypy/branch/pypy-string-formatting/rpython/test/test_rlist.py Sat May 12 12:29:35 2007 @@ -1171,6 +1171,15 @@ else: assert False + def test_charlist_extensions(self): + def f(n): + s = 'hello%d' % n + l = ['a', 'b'] + l += s + return ''.join(l) + res = self.interpret(f, [58]) + assert self.ll_to_string(res) == 'abhello58' + class TestLLtype(BaseTestRlist, LLRtypeMixin): rlist = ll_rlist From arigo at codespeak.net Sat May 12 13:02:30 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 12 May 2007 13:02:30 +0200 (CEST) Subject: [pypy-svn] r43277 - in pypy/branch/pypy-string-formatting: rpython rpython/test translator Message-ID: <20070512110230.AF376807A@code0.codespeak.net> Author: arigo Date: Sat May 12 13:02:29 2007 New Revision: 43277 Modified: pypy/branch/pypy-string-formatting/rpython/rlist.py pypy/branch/pypy-string-formatting/rpython/rtyper.py pypy/branch/pypy-string-formatting/rpython/test/test_rlist.py pypy/branch/pypy-string-formatting/translator/transform.py Log: Direct support for the following combined operations: lst += string[start:] lst += string[start:stop] lst += string[:-1] Lengthy code, so I wouldn't be surprized if there is a bug somewhere despite the tests. Modified: pypy/branch/pypy-string-formatting/rpython/rlist.py ============================================================================== --- pypy/branch/pypy-string-formatting/rpython/rlist.py (original) +++ pypy/branch/pypy-string-formatting/rpython/rlist.py Sat May 12 13:02:29 2007 @@ -362,6 +362,27 @@ c_strlen, c_stritem) return v_lst1 + def rtype_extend_with_str_slice((r_lst1, r_str2), hop): + rs = r_lst1.rtyper.type_system.rslice + string_repr = r_lst1.rtyper.type_system.rstr.string_repr + c_strlen = hop.inputconst(Void, string_repr.ll.ll_strlen) + c_stritem = hop.inputconst(Void, string_repr.ll.ll_stritem_nonneg) + r_slic = hop.args_r[2] + v_lst1, v_str2, v_slice = hop.inputargs(r_lst1, string_repr, r_slic) + if r_slic == rs.startonly_slice_repr: + hop.gendirectcall(ll_extend_with_str_slice_startonly, + v_lst1, v_str2, c_strlen, c_stritem, v_slice) + elif r_slic == rs.startstop_slice_repr: + hop.gendirectcall(ll_extend_with_str_slice, + v_lst1, v_str2, c_strlen, c_stritem, v_slice) + elif r_slic == rs.minusone_slice_repr: + hop.gendirectcall(ll_extend_with_str_slice_minusone, + v_lst1, v_str2, c_strlen, c_stritem) + else: + raise TyperError('lst += str[:] does not support slices with %r' % + (r_slic,)) + return v_lst1 + class __extend__(pairtype(AbstractBaseListRepr, AbstractSliceRepr)): @@ -705,17 +726,54 @@ j += 1 def ll_extend_with_str(lst, s, getstrlen, getstritem): + return ll_extend_with_str_slice_startonly(lst, s, getstrlen, getstritem, 0) + +def ll_extend_with_str_slice_startonly(lst, s, getstrlen, getstritem, start): len1 = lst.ll_length() len2 = getstrlen(s) - newlength = len1 + len2 + debug_assert(start >= 0, "unexpectedly negative str slice start") + debug_assert(start <= len2, "str slice start larger than str length") + newlength = len1 + len2 - start lst._ll_resize_ge(newlength) - i = 0 + i = start j = len1 while i < len2: lst.ll_setitem_fast(j, getstritem(s, i)) i += 1 j += 1 +def ll_extend_with_str_slice(lst, s, getstrlen, getstritem, slice): + start = slice.start + stop = slice.stop + len1 = lst.ll_length() + len2 = getstrlen(s) + debug_assert(start >= 0, "unexpectedly negative str slice start") + debug_assert(start <= len2, "str slice start larger than str length") + debug_assert(stop >= start, "str slice stop smaller than start") + if stop > len2: + stop = len2 + newlength = len1 + stop - start + lst._ll_resize_ge(newlength) + i = start + j = len1 + while i < stop: + lst.ll_setitem_fast(j, getstritem(s, i)) + i += 1 + j += 1 + +def ll_extend_with_str_slice_minusone(lst, s, getstrlen, getstritem): + len1 = lst.ll_length() + len2m1 = getstrlen(s) - 1 + debug_assert(len2m1 >= 0, "empty string is sliced with [:-1]") + newlength = len1 + len2m1 + lst._ll_resize_ge(newlength) + i = 0 + j = len1 + while i < len2m1: + lst.ll_setitem_fast(j, getstritem(s, i)) + i += 1 + j += 1 + def ll_listslice_startonly(RESLIST, l1, start): len1 = l1.ll_length() debug_assert(start >= 0, "unexpectedly negative list slice start") Modified: pypy/branch/pypy-string-formatting/rpython/rtyper.py ============================================================================== --- pypy/branch/pypy-string-formatting/rpython/rtyper.py (original) +++ pypy/branch/pypy-string-formatting/rpython/rtyper.py Sat May 12 13:02:29 2007 @@ -576,6 +576,11 @@ def translate_op_alloc_and_set(self, hop): return rlist.rtype_alloc_and_set(hop) + def translate_op_extend_with_str_slice(self, hop): + r_arg1 = hop.args_r[0] + r_arg2 = hop.args_r[1] + return pair(r_arg1, r_arg2).rtype_extend_with_str_slice(hop) + def translate_op_newtuple(self, hop): return self.type_system.rtuple.rtype_newtuple(hop) Modified: pypy/branch/pypy-string-formatting/rpython/test/test_rlist.py ============================================================================== --- pypy/branch/pypy-string-formatting/rpython/test/test_rlist.py (original) +++ pypy/branch/pypy-string-formatting/rpython/test/test_rlist.py Sat May 12 13:02:29 2007 @@ -1180,6 +1180,32 @@ res = self.interpret(f, [58]) assert self.ll_to_string(res) == 'abhello58' + def f(n, i): + s = 'hello%d' % n + assert 0 <= i <= len(s) + l = ['a', 'b'] + l += s[i:] + return ''.join(l) + res = self.interpret(f, [9381701, 3]) + assert self.ll_to_string(res) == 'ablo9381701' + + def f(n, i, j): + s = 'hello%d' % n + assert 0 <= i <= j <= len(s) + l = ['a', 'b'] + l += s[i:j] + return ''.join(l) + res = self.interpret(f, [9381701, 3, 7]) + assert self.ll_to_string(res) == 'ablo93' + + def f(n): + s = 'hello%d' % n + l = ['a', 'b'] + l += s[:-1] + return ''.join(l) + res = self.interpret(f, [9381701]) + assert self.ll_to_string(res) == 'abhello938170' + class TestLLtype(BaseTestRlist, LLRtypeMixin): rlist = ll_rlist Modified: pypy/branch/pypy-string-formatting/translator/transform.py ============================================================================== --- pypy/branch/pypy-string-formatting/translator/transform.py (original) +++ pypy/branch/pypy-string-formatting/translator/transform.py Sat May 12 13:02:29 2007 @@ -55,6 +55,32 @@ op.result) block.operations[i] = new_op +# lst += string[x:y] +# --> +# b = getitem(string, slice) +# c = inplace_add(lst, b) +# --> +# c = extend_with_str_slice(lst, string, slice) + +def transform_extend_with_str_slice(self, block_subset): + """Transforms lst += string[x:y] to extend_with_str_slice""" + for block in block_subset: + slice_sources = {} # maps b to [string, slice] in the above notation + for i in range(len(block.operations)): + op = block.operations[i] + if (op.opname == 'getitem' and + self.gettype(op.args[0]) is str and + self.gettype(op.args[1]) is slice): + slice_sources[op.result] = op.args + elif (op.opname == 'inplace_add' and + op.args[1] in slice_sources and + self.gettype(op.args[0]) is list): + v_string, v_slice = slice_sources[op.args[1]] + new_op = SpaceOperation('extend_with_str_slice', + [op.args[0], v_string, v_slice], + op.result) + block.operations[i] = new_op + # a[b:c] # --> # d = newslice(b, c, None) @@ -62,22 +88,23 @@ # --> # e = getslice(a, b, c) -def transform_slice(self, block_subset): - """Transforms a[b:c] to getslice(a, b, c).""" - for block in block_subset: - operations = block.operations[:] - n_op = len(operations) - for i in range(0, n_op-1): - op1 = operations[i] - op2 = operations[i+1] - if (op1.opname == 'newslice' and - self.gettype(op1.args[2]) is types.NoneType and - op2.opname == 'getitem' and - op1.result is op2.args[1]): - new_op = SpaceOperation('getslice', - (op2.args[0], op1.args[0], op1.args[1]), - op2.result) - block.operations[i+1:i+2] = [new_op] +##def transform_slice(self, block_subset): -- not used any more -- +## """Transforms a[b:c] to getslice(a, b, c).""" +## for block in block_subset: +## operations = block.operations[:] +## n_op = len(operations) +## for i in range(0, n_op-1): +## op1 = operations[i] +## op2 = operations[i+1] +## if (op1.opname == 'newslice' and +## self.gettype(op1.args[2]) is types.NoneType and +## op2.opname == 'getitem' and +## op1.result is op2.args[1]): +## new_op = SpaceOperation('getslice', +## (op2.args[0], op1.args[0], op1.args[1]), +## op2.result) +## block.operations[i+1:i+2] = [new_op] + def transform_dead_op_vars(self, block_subset): # we redo the same simplification from simplify.py, @@ -215,6 +242,7 @@ default_extra_passes = [ transform_allocate, + transform_extend_with_str_slice, ] def transform_graph(ann, extra_passes=None, block_subset=None): From hpk at codespeak.net Sat May 12 13:31:52 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Sat, 12 May 2007 13:31:52 +0200 (CEST) Subject: [pypy-svn] r43281 - pypy/extradoc/eu-report Message-ID: <20070512113152.0CCBA8078@code0.codespeak.net> Author: hpk Date: Sat May 12 13:31:52 2007 New Revision: 43281 Added: pypy/extradoc/eu-report/PYPY-EU-Final-Activity-Report.pdf - copied unchanged from r43280, pypy/eu-tracking/deliverable/activity_reports/final/activity_report.pdf Log: put final activity report to publish location From hpk at codespeak.net Sat May 12 13:41:59 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Sat, 12 May 2007 13:41:59 +0200 (CEST) Subject: [pypy-svn] r43282 - pypy/dist/pypy/doc/tool Message-ID: <20070512114159.ABFDC807A@code0.codespeak.net> Author: hpk Date: Sat May 12 13:41:59 2007 New Revision: 43282 Modified: pypy/dist/pypy/doc/tool/makecontributor.py Log: be more tolerant about unknown realnames Modified: pypy/dist/pypy/doc/tool/makecontributor.py ============================================================================== --- pypy/dist/pypy/doc/tool/makecontributor.py (original) +++ pypy/dist/pypy/doc/tool/makecontributor.py Sat May 12 13:41:59 2007 @@ -31,7 +31,10 @@ mark = False for author, count in items: user = uconf.system.User(author) - realname = user.realname.strip() + try: + realname = user.realname.strip() + except KeyError: + realname = author if not mark and count < cutoff: mark = True print '-'*60 From arigo at codespeak.net Sat May 12 13:43:42 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 12 May 2007 13:43:42 +0200 (CEST) Subject: [pypy-svn] r43283 - in pypy/branch/pypy-string-formatting: rpython rpython/test translator Message-ID: <20070512114342.2EF80807A@code0.codespeak.net> Author: arigo Date: Sat May 12 13:43:41 2007 New Revision: 43283 Modified: pypy/branch/pypy-string-formatting/rpython/rlist.py pypy/branch/pypy-string-formatting/rpython/rtyper.py pypy/branch/pypy-string-formatting/rpython/test/test_rlist.py pypy/branch/pypy-string-formatting/translator/transform.py Log: Direct support for lst += char * count lst += count * char Also check that 'lst' is really a list of chars, and complain clearly if it something else like a list of strings. Modified: pypy/branch/pypy-string-formatting/rpython/rlist.py ============================================================================== --- pypy/branch/pypy-string-formatting/rpython/rlist.py (original) +++ pypy/branch/pypy-string-formatting/rpython/rlist.py Sat May 12 13:43:41 2007 @@ -4,7 +4,7 @@ from pypy.rpython.error import TyperError from pypy.rpython.rmodel import Repr, IteratorRepr, IntegerRepr, inputconst from pypy.rpython.rslice import AbstractSliceRepr -from pypy.rpython.rstr import AbstractStringRepr +from pypy.rpython.rstr import AbstractStringRepr, AbstractCharRepr from pypy.rpython.lltypesystem.lltype import typeOf, Ptr, Void, Signed, Bool from pypy.rpython.lltypesystem.lltype import nullptr, Char, UniChar from pypy.rpython import robject @@ -354,6 +354,9 @@ class __extend__(pairtype(AbstractListRepr, AbstractStringRepr)): def rtype_inplace_add((r_lst1, r_str2), hop): + if r_lst1.item_repr.lowleveltype != Char: + raise TyperError('"lst += string" only supported with a list ' + 'of chars') string_repr = r_lst1.rtyper.type_system.rstr.string_repr v_lst1, v_str2 = hop.inputargs(r_lst1, string_repr) c_strlen = hop.inputconst(Void, string_repr.ll.ll_strlen) @@ -363,6 +366,9 @@ return v_lst1 def rtype_extend_with_str_slice((r_lst1, r_str2), hop): + if r_lst1.item_repr.lowleveltype != Char: + raise TyperError('"lst += string" only supported with a list ' + 'of chars') rs = r_lst1.rtyper.type_system.rslice string_repr = r_lst1.rtyper.type_system.rstr.string_repr c_strlen = hop.inputconst(Void, string_repr.ll.ll_strlen) @@ -383,6 +389,17 @@ (r_slic,)) return v_lst1 +class __extend__(pairtype(AbstractListRepr, AbstractCharRepr)): + + def rtype_extend_with_char_count((r_lst1, r_chr2), hop): + if r_lst1.item_repr.lowleveltype != Char: + raise TyperError('"lst += string" only supported with a list ' + 'of chars') + char_repr = r_lst1.rtyper.type_system.rstr.char_repr + v_lst1, v_chr, v_count = hop.inputargs(r_lst1, char_repr, Signed) + hop.gendirectcall(ll_extend_with_char_count, v_lst1, v_chr, v_count) + return v_lst1 + class __extend__(pairtype(AbstractBaseListRepr, AbstractSliceRepr)): @@ -774,6 +791,17 @@ i += 1 j += 1 +def ll_extend_with_char_count(lst, char, count): + if count <= 0: + return + len1 = lst.ll_length() + newlength = len1 + count + lst._ll_resize_ge(newlength) + j = len1 + while j < newlength: + lst.ll_setitem_fast(j, char) + j += 1 + def ll_listslice_startonly(RESLIST, l1, start): len1 = l1.ll_length() debug_assert(start >= 0, "unexpectedly negative list slice start") Modified: pypy/branch/pypy-string-formatting/rpython/rtyper.py ============================================================================== --- pypy/branch/pypy-string-formatting/rpython/rtyper.py (original) +++ pypy/branch/pypy-string-formatting/rpython/rtyper.py Sat May 12 13:43:41 2007 @@ -581,6 +581,11 @@ r_arg2 = hop.args_r[1] return pair(r_arg1, r_arg2).rtype_extend_with_str_slice(hop) + def translate_op_extend_with_char_count(self, hop): + r_arg1 = hop.args_r[0] + r_arg2 = hop.args_r[1] + return pair(r_arg1, r_arg2).rtype_extend_with_char_count(hop) + def translate_op_newtuple(self, hop): return self.type_system.rtuple.rtype_newtuple(hop) Modified: pypy/branch/pypy-string-formatting/rpython/test/test_rlist.py ============================================================================== --- pypy/branch/pypy-string-formatting/rpython/test/test_rlist.py (original) +++ pypy/branch/pypy-string-formatting/rpython/test/test_rlist.py Sat May 12 13:43:41 2007 @@ -1,5 +1,6 @@ import sys from pypy.translator.translator import TranslationContext +from pypy.rpython.error import TyperError from pypy.rpython.lltypesystem.lltype import * from pypy.rpython.ootypesystem import ootype from pypy.rpython.rlist import * @@ -1171,7 +1172,7 @@ else: assert False - def test_charlist_extensions(self): + def test_charlist_extension_1(self): def f(n): s = 'hello%d' % n l = ['a', 'b'] @@ -1180,6 +1181,15 @@ res = self.interpret(f, [58]) assert self.ll_to_string(res) == 'abhello58' + def test_extend_a_non_char_list_1(self): + def f(n): + s = 'hello%d' % n + l = ['foo', 'bar'] + l += s # NOT SUPPORTED for now if l is not a list of chars + return ''.join(l) + py.test.raises(TyperError, self.interpret, f, [58]) + + def test_charlist_extension_2(self): def f(n, i): s = 'hello%d' % n assert 0 <= i <= len(s) @@ -1189,6 +1199,16 @@ res = self.interpret(f, [9381701, 3]) assert self.ll_to_string(res) == 'ablo9381701' + def test_extend_a_non_char_list_2(self): + def f(n, i): + s = 'hello%d' % n + assert 0 <= i <= len(s) + l = ['foo', 'bar'] + l += s[i:] # NOT SUPPORTED for now if l is not a list of chars + return ''.join(l) + py.test.raises(TyperError, self.interpret, f, [9381701, 3]) + + def test_charlist_extension_3(self): def f(n, i, j): s = 'hello%d' % n assert 0 <= i <= j <= len(s) @@ -1198,6 +1218,7 @@ res = self.interpret(f, [9381701, 3, 7]) assert self.ll_to_string(res) == 'ablo93' + def test_charlist_extension_4(self): def f(n): s = 'hello%d' % n l = ['a', 'b'] @@ -1206,6 +1227,34 @@ res = self.interpret(f, [9381701]) assert self.ll_to_string(res) == 'abhello938170' + def test_charlist_extension_5(self): + def f(count): + l = ['a', 'b'] + l += '.' * count # char * count + return ''.join(l) + res = self.interpret(f, [7]) + assert self.ll_to_string(res) == 'ab.......' + res = self.interpret(f, [0]) + assert self.ll_to_string(res) == 'ab' + + def test_charlist_extension_6(self): + def f(count): + l = ['a', 'b'] + l += count * '.' # count * char + return ''.join(l) + res = self.interpret(f, [7]) + assert self.ll_to_string(res) == 'ab.......' + res = self.interpret(f, [0]) + assert self.ll_to_string(res) == 'ab' + + def test_extend_a_non_char_list_6(self): + def f(count): + l = ['foo', 'bar'] + # NOT SUPPORTED for now if l is not a list of chars + l += count * '.' + return ''.join(l) + py.test.raises(TyperError, self.interpret, f, [5]) + class TestLLtype(BaseTestRlist, LLRtypeMixin): rlist = ll_rlist Modified: pypy/branch/pypy-string-formatting/translator/transform.py ============================================================================== --- pypy/branch/pypy-string-formatting/translator/transform.py (original) +++ pypy/branch/pypy-string-formatting/translator/transform.py Sat May 12 13:43:41 2007 @@ -81,6 +81,37 @@ op.result) block.operations[i] = new_op +# lst += char*count [or count*char] +# --> +# b = mul(char, count) [or count, char] +# c = inplace_add(lst, b) +# --> +# c = extend_with_char_count(lst, char, count) + +def transform_extend_with_char_count(self, block_subset): + """Transforms lst += char*count to extend_with_char_count""" + for block in block_subset: + mul_sources = {} # maps b to (char, count) in the above notation + for i in range(len(block.operations)): + op = block.operations[i] + if op.opname == 'mul': + s0 = self.binding(op.args[0], None) + s1 = self.binding(op.args[1], None) + if (isinstance(s0, annmodel.SomeChar) and + isinstance(s1, annmodel.SomeInteger)): + mul_sources[op.result] = op.args[0], op.args[1] + elif (isinstance(s1, annmodel.SomeChar) and + isinstance(s0, annmodel.SomeInteger)): + mul_sources[op.result] = op.args[1], op.args[0] + elif (op.opname == 'inplace_add' and + op.args[1] in mul_sources and + self.gettype(op.args[0]) is list): + v_char, v_count = mul_sources[op.args[1]] + new_op = SpaceOperation('extend_with_char_count', + [op.args[0], v_char, v_count], + op.result) + block.operations[i] = new_op + # a[b:c] # --> # d = newslice(b, c, None) @@ -243,6 +274,7 @@ default_extra_passes = [ transform_allocate, transform_extend_with_str_slice, + transform_extend_with_char_count, ] def transform_graph(ann, extra_passes=None, block_subset=None): From fijal at codespeak.net Sat May 12 13:48:17 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 12 May 2007 13:48:17 +0200 (CEST) Subject: [pypy-svn] r43284 - pypy/dist/pypy/doc Message-ID: <20070512114817.61882807A@code0.codespeak.net> Author: fijal Date: Sat May 12 13:48:17 2007 New Revision: 43284 Added: pypy/dist/pypy/doc/download.txt Modified: pypy/dist/pypy/doc/confrest.py Log: Add a download page. First approach Modified: pypy/dist/pypy/doc/confrest.py ============================================================================== --- pypy/dist/pypy/doc/confrest.py (original) +++ pypy/dist/pypy/doc/confrest.py Sat May 12 13:48:17 2007 @@ -12,6 +12,8 @@ class_="menu"), " ", html.a("documentation", href=self.get_doclink("index.html"), class_="menu"), " ", + html.a("downloads", href=self.get_doclink("download.html"), + class_="menu"), " ", html.a("svn", href="https://codespeak.net/viewvc/pypy/dist/", class_="menu"), " ", html.a("issues", Added: pypy/dist/pypy/doc/download.txt ============================================================================== --- (empty file) +++ pypy/dist/pypy/doc/download.txt Sat May 12 13:48:17 2007 @@ -0,0 +1,30 @@ + +Download one of the following release files: +============================================= + +PyPy 1.0 Sources: +----------------- + +* `pypy-1.0.0.tar.bz2`_ (sources, unix line endings) or +* `pypy-1.0.0.tar.gz`_ (sources, unix line endings) or +* `pypy-1.0.0.zip`_ (sources, windows line-endings) or + +PyPy 1.0 prebuilt binaries: +---------------------------- +* `pypy-1.0.0-win32.zip`_ (precompiled executables for windows) +* `pypy-1.0.0-dynamic.tar.bz2` (precombiled binaries for linux, dynamic + executable, GLIBC 2.5) +* `pypy-1.0.0-static.tar.bz2` (precombiled binaries for linux, static + executable, GLIBC 2.5) +* `pypy-1.0.0-minimal-dynamic.tar.bz2`_ (precompiled minimal binary for + linux, dynamic executable, GLIBC 2.5) +* `pypy-1.0.0-minimal-static.tar.bz2`_ (precompiled minimal binary for + linux, statix executable, GLIBC 2.5) + +.. _`pypy-1.0.0.tar.bz2`: http://codespeak.net/download/pypy/pypy-1.0.0.tar.bz2 +.. _`pypy-1.0.0.zip`: http://codespeak.net/download/pypy/pypy-1.0.0.zip +.. _`pypy-1.0.0.tar.gz`: http://codespeak.net/download/pypy/pypy-1.0.0.tar.gz +.. _`pypy-1.0.0-win32.zip`: http://codespeak.net/download/pypy/pypy-1.0.0-win32.zip +.. _`pypy-1.0.0-minimal-dynamic.tar.bz2`: http://codespeak.net/download/pypy/pypy-1.0.0-minimal.tar.bz2 +.. _`pypy-1.0.0-minimal-static.tar.bz2`: http://codespeak.net/download/pypy/pypy-1.0.0-minimal-static.tar.bz2 + From arigo at codespeak.net Sat May 12 13:54:21 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 12 May 2007 13:54:21 +0200 (CEST) Subject: [pypy-svn] r43285 - pypy/branch/pypy-string-formatting/objspace/std Message-ID: <20070512115421.DD8FE807A@code0.codespeak.net> Author: arigo Date: Sat May 12 13:54:20 2007 New Revision: 43285 Modified: pypy/branch/pypy-string-formatting/objspace/std/formatting.py Log: Use the newly supported efficient "lst += string" constructions. Modified: pypy/branch/pypy-string-formatting/objspace/std/formatting.py ============================================================================== --- pypy/branch/pypy-string-formatting/objspace/std/formatting.py (original) +++ pypy/branch/pypy-string-formatting/objspace/std/formatting.py Sat May 12 13:54:20 2007 @@ -39,6 +39,7 @@ def std_wp_int(self, r, prefix=''): # use self.prec to add some '0' on the left of the number + import pdb; pdb.set_trace() if self.prec >= 0: sign = r[0] == '-' padding = self.prec - (len(r)-int(sign)) @@ -248,14 +249,15 @@ while True: # fast path: consume as many characters as possible fmt = self.fmt - i = self.fmtpos + i = i0 = self.fmtpos while i < len(fmt): if fmt[i] == '%': break - result.append(fmt[i]) i += 1 else: + result += fmt[i0:] break # end of 'fmt' string + result += fmt[i0:i] self.fmtpos = i + 1 # interpret the next formatter @@ -308,13 +310,10 @@ result = self.result padding = self.width - length if not self.f_ljust: - for i in range(padding): # add any padding at the left of 'r' - result.append(' ') + result += ' ' * padding # add any padding at the left of 'r' padding = 0 - for i in range(length): # add 'r' itself - result.append(r[i]) - for i in range(padding): # add any remaining padding at the right - result.append(' ') + result += r[:length] # add 'r' itself + result += ' ' * padding # add any remaining padding at the right std_wp._annspecialcase_ = 'specialize:argtype(1)' def std_wp_number(self, r, prefix=''): @@ -341,20 +340,15 @@ padnumber = '>' if padnumber == '>': - for i in range(padding): - result.append(' ') # pad with spaces on the left + result += ' ' * padding # pad with spaces on the left if sign: result.append(r[0]) # the sign - for c in prefix: - result.append(c) # the prefix + result += prefix # the prefix if padnumber == '0': - for i in range(padding): # pad with zeroes - result.append('0') - for j in range(int(sign), len(r)): # the rest of the number - result.append(r[j]) + result += '0' * padding # pad with zeroes + result += r[int(sign):] # the rest of the number if padnumber == '<': # spaces on the right - for i in range(padding): - result.append(' ') + result += ' ' * padding def fmt_s(self, w_value): space = self.space From hpk at codespeak.net Sat May 12 13:56:56 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Sat, 12 May 2007 13:56:56 +0200 (CEST) Subject: [pypy-svn] r43286 - pypy/dist/pypy/doc Message-ID: <20070512115656.3E215807A@code0.codespeak.net> Author: hpk Date: Sat May 12 13:56:55 2007 New Revision: 43286 Modified: pypy/dist/pypy/doc/index-report.txt pypy/dist/pypy/doc/news.txt Log: publish Final-A report, add news item about finish up of EU project period. Modified: pypy/dist/pypy/doc/index-report.txt ============================================================================== --- pypy/dist/pypy/doc/index-report.txt (original) +++ pypy/dist/pypy/doc/index-report.txt Sat May 12 13:56:55 2007 @@ -12,11 +12,17 @@ Reports of 2007 =============== +The `PyPy EU Final Activity Report`_ summarizes the 28 month EU project +period (Dec 2004-March 2007) on technical, scientific and community levels. +You do not need prior knowledge about PyPy but some technical knowledge about +computer language implementations is helpful. The report contains reflections +and recommendations which might be interesting for other project aiming +at funded Open Source research. *(2007-05-11)* + `D09.1 Constraint Solving and Semantic Web`_ is a report about PyPy's logic programming and constraint solving features, as well as the work going on to tie semantic web technologies and PyPy together. *(2007-05-11)* - `D14.4 PyPy-1.0 Milestone report`_ (for language developers and researchers) summarizes research & technical results of the PyPy-1.0 release and discusses related development process and community aspects. *(2007-05-01)* @@ -129,6 +135,7 @@ `D14.1 Report about Milestone/Phase 1`_ describes what happened in the PyPy project during the first year of EU funding (December 2004 - December 2005) +.. _`PyPy EU Final Activity Report`: http://codespeak.net/pypy/extradoc/eu-report/PYPY-EU-Final-Activity-Report.pdf .. _`D01.2-4 Project Organization`: http://codespeak.net/pypy/extradoc/eu-report/D01.2-4_Project_Organization-2007-03-28.pdf .. _`D02.1 Development Tools and Website`: http://codespeak.net/pypy/extradoc/eu-report/D02.1_Development_Tools_and_Website-2007-03-21.pdf .. _`D02.2 Release Scheme`: http://codespeak.net/svn/pypy/extradoc/eu-report/D02.2_Release_Scheme-2007-03-30.pdf Modified: pypy/dist/pypy/doc/news.txt ============================================================================== --- pypy/dist/pypy/doc/news.txt (original) +++ pypy/dist/pypy/doc/news.txt Sat May 12 13:56:55 2007 @@ -14,6 +14,19 @@ .. _`iCalendar format`: webcal://pypycal.sabi.net///calendars/PyPy.ics .. _eventhistory: eventhistory.html +PyPy EU funding period over, Review ahead +=========================================================== + +The 28 month EU project period of PyPy is over and new things are to come! +On 11th May we `submitted last documents`_ to the European Union and are now +heading towards a 31st May Review Meeting in Bruxelles. The `PyPy EU Final +Activity Report`_ summarizes what we did and what we have in mind +on technical, scientific and community levels. It also contains reflections +and recommendations possibly interesting to other projects aiming at +EU funded Open Source research.*(2007-05-12)* + +.. _`submitted last documents`: http://codespeak.net/pypy/dist/pypy/doc/index-report.html +.. _`PyPy EU Final Activity Report`: http://codespeak.net/pypy/extradoc/eu-report/PYPY-EU-Final-Activity-Report.pdf PyPy 1.0: JIT compiler generator, optimizations and more ================================================================== From arigo at codespeak.net Sat May 12 13:57:17 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 12 May 2007 13:57:17 +0200 (CEST) Subject: [pypy-svn] r43287 - pypy/branch/pypy-string-formatting/objspace/std Message-ID: <20070512115717.2F9DC807B@code0.codespeak.net> Author: arigo Date: Sat May 12 13:57:16 2007 New Revision: 43287 Modified: pypy/branch/pypy-string-formatting/objspace/std/formatting.py Log: Oups Modified: pypy/branch/pypy-string-formatting/objspace/std/formatting.py ============================================================================== --- pypy/branch/pypy-string-formatting/objspace/std/formatting.py (original) +++ pypy/branch/pypy-string-formatting/objspace/std/formatting.py Sat May 12 13:57:16 2007 @@ -39,7 +39,6 @@ def std_wp_int(self, r, prefix=''): # use self.prec to add some '0' on the left of the number - import pdb; pdb.set_trace() if self.prec >= 0: sign = r[0] == '-' padding = self.prec - (len(r)-int(sign)) From hpk at codespeak.net Sat May 12 13:59:09 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Sat, 12 May 2007 13:59:09 +0200 (CEST) Subject: [pypy-svn] r43289 - pypy/extradoc/pypy.org Message-ID: <20070512115909.94CEE807A@code0.codespeak.net> Author: hpk Date: Sat May 12 13:59:09 2007 New Revision: 43289 Modified: pypy/extradoc/pypy.org/news.txt Log: redundantly add news item Modified: pypy/extradoc/pypy.org/news.txt ============================================================================== --- pypy/extradoc/pypy.org/news.txt (original) +++ pypy/extradoc/pypy.org/news.txt Sat May 12 13:59:09 2007 @@ -1,3 +1,17 @@ +PyPy EU funding period over, Review ahead +----------------------------------------------------- + +The 28 month EU project period of PyPy is over and new things are to come! +On 11th May we `submitted last documents`_ to the European Union and are now +heading towards a 31st May Review Meeting in Bruxelles. The `PyPy EU Final +Activity Report`_ summarizes what we did and what we have in mind +on technical, scientific and community levels. It also contains reflections +and recommendations possibly interesting to other projects aiming at +EU funded Open Source research.*(2007-05-12)* + +.. _`submitted last documents`: http://codespeak.net/pypy/dist/pypy/doc/index-report.html +.. _`PyPy EU Final Activity Report`: http://codespeak.net/pypy/extradoc/eu-report/PYPY-EU-Final-Activity-Report.pdf + PyPy 1.0: JIT compiler generator, optimizations and more ------------------------------------------------------------------ From arigo at codespeak.net Sat May 12 13:59:57 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 12 May 2007 13:59:57 +0200 (CEST) Subject: [pypy-svn] r43290 - pypy/branch/pypy-string-formatting/objspace/std/test Message-ID: <20070512115957.8B18C807A@code0.codespeak.net> Author: arigo Date: Sat May 12 13:59:56 2007 New Revision: 43290 Modified: pypy/branch/pypy-string-formatting/objspace/std/test/test_stringformat.py Log: This annoying test passes now on top of CPython, although it instantly consumes and releases 2GB of RAM. Modified: pypy/branch/pypy-string-formatting/objspace/std/test/test_stringformat.py ============================================================================== --- pypy/branch/pypy-string-formatting/objspace/std/test/test_stringformat.py (original) +++ pypy/branch/pypy-string-formatting/objspace/std/test/test_stringformat.py Sat May 12 13:59:56 2007 @@ -201,9 +201,9 @@ assert u"%.1d" % 3 == '3' def test_unicode_overflow(self): - skip("do something about it or just ignore it") + skip("nicely passes on top of CPython but requires > 2GB of RAM") import sys - raises(OverflowError, 'u"%.*d" % (sys.maxint, 1)') + raises((OverflowError, MemoryError), 'u"%.*d" % (sys.maxint, 1)') def test_unicode_format_a(self): assert u'%x' % 10L == 'a' From fijal at codespeak.net Sat May 12 14:04:37 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 12 May 2007 14:04:37 +0200 (CEST) Subject: [pypy-svn] r43291 - pypy/dist/pypy/doc Message-ID: <20070512120437.5A24B807A@code0.codespeak.net> Author: fijal Date: Sat May 12 14:04:36 2007 New Revision: 43291 Modified: pypy/dist/pypy/doc/download.txt Log: Dynamic executables does not depend on exact glibc Modified: pypy/dist/pypy/doc/download.txt ============================================================================== --- pypy/dist/pypy/doc/download.txt (original) +++ pypy/dist/pypy/doc/download.txt Sat May 12 14:04:36 2007 @@ -13,11 +13,11 @@ ---------------------------- * `pypy-1.0.0-win32.zip`_ (precompiled executables for windows) * `pypy-1.0.0-dynamic.tar.bz2` (precombiled binaries for linux, dynamic - executable, GLIBC 2.5) + executable) * `pypy-1.0.0-static.tar.bz2` (precombiled binaries for linux, static executable, GLIBC 2.5) * `pypy-1.0.0-minimal-dynamic.tar.bz2`_ (precompiled minimal binary for - linux, dynamic executable, GLIBC 2.5) + linux, dynamic executable) * `pypy-1.0.0-minimal-static.tar.bz2`_ (precompiled minimal binary for linux, statix executable, GLIBC 2.5) From arigo at codespeak.net Sat May 12 14:35:32 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 12 May 2007 14:35:32 +0200 (CEST) Subject: [pypy-svn] r43292 - in pypy/branch/pypy-string-formatting/rpython: . test Message-ID: <20070512123532.13CC1807C@code0.codespeak.net> Author: arigo Date: Sat May 12 14:35:31 2007 New Revision: 43292 Modified: pypy/branch/pypy-string-formatting/rpython/rlist.py pypy/branch/pypy-string-formatting/rpython/test/test_rlist.py Log: Support for "lst += string" where lst is a list of unicode characters. Modified: pypy/branch/pypy-string-formatting/rpython/rlist.py ============================================================================== --- pypy/branch/pypy-string-formatting/rpython/rlist.py (original) +++ pypy/branch/pypy-string-formatting/rpython/rlist.py Sat May 12 14:35:31 2007 @@ -354,9 +354,9 @@ class __extend__(pairtype(AbstractListRepr, AbstractStringRepr)): def rtype_inplace_add((r_lst1, r_str2), hop): - if r_lst1.item_repr.lowleveltype != Char: + if r_lst1.item_repr.lowleveltype not in (Char, UniChar): raise TyperError('"lst += string" only supported with a list ' - 'of chars') + 'of chars or unichars') string_repr = r_lst1.rtyper.type_system.rstr.string_repr v_lst1, v_str2 = hop.inputargs(r_lst1, string_repr) c_strlen = hop.inputconst(Void, string_repr.ll.ll_strlen) @@ -366,9 +366,9 @@ return v_lst1 def rtype_extend_with_str_slice((r_lst1, r_str2), hop): - if r_lst1.item_repr.lowleveltype != Char: + if r_lst1.item_repr.lowleveltype not in (Char, UniChar): raise TyperError('"lst += string" only supported with a list ' - 'of chars') + 'of chars or unichars') rs = r_lst1.rtyper.type_system.rslice string_repr = r_lst1.rtyper.type_system.rstr.string_repr c_strlen = hop.inputconst(Void, string_repr.ll.ll_strlen) @@ -392,9 +392,9 @@ class __extend__(pairtype(AbstractListRepr, AbstractCharRepr)): def rtype_extend_with_char_count((r_lst1, r_chr2), hop): - if r_lst1.item_repr.lowleveltype != Char: + if r_lst1.item_repr.lowleveltype not in (Char, UniChar): raise TyperError('"lst += string" only supported with a list ' - 'of chars') + 'of chars or unichars') char_repr = r_lst1.rtyper.type_system.rstr.char_repr v_lst1, v_chr, v_count = hop.inputargs(r_lst1, char_repr, Signed) hop.gendirectcall(ll_extend_with_char_count, v_lst1, v_chr, v_count) @@ -500,6 +500,12 @@ return nullptr(ITEM.TO) return None +def listItemType(lst): + LIST = typeOf(lst) + if isinstance(LIST, Ptr): # lltype + LIST = LIST.TO + return LIST.ITEM + def ll_copy(RESLIST, l): length = l.ll_length() @@ -755,7 +761,10 @@ i = start j = len1 while i < len2: - lst.ll_setitem_fast(j, getstritem(s, i)) + c = getstritem(s, i) + if listItemType(lst) is UniChar: + c = unichr(ord(c)) + lst.ll_setitem_fast(j, c) i += 1 j += 1 @@ -774,7 +783,10 @@ i = start j = len1 while i < stop: - lst.ll_setitem_fast(j, getstritem(s, i)) + c = getstritem(s, i) + if listItemType(lst) is UniChar: + c = unichr(ord(c)) + lst.ll_setitem_fast(j, c) i += 1 j += 1 @@ -787,7 +799,10 @@ i = 0 j = len1 while i < len2m1: - lst.ll_setitem_fast(j, getstritem(s, i)) + c = getstritem(s, i) + if listItemType(lst) is UniChar: + c = unichr(ord(c)) + lst.ll_setitem_fast(j, c) i += 1 j += 1 @@ -798,6 +813,8 @@ newlength = len1 + count lst._ll_resize_ge(newlength) j = len1 + if listItemType(lst) is UniChar: + char = unichr(ord(char)) while j < newlength: lst.ll_setitem_fast(j, char) j += 1 Modified: pypy/branch/pypy-string-formatting/rpython/test/test_rlist.py ============================================================================== --- pypy/branch/pypy-string-formatting/rpython/test/test_rlist.py (original) +++ pypy/branch/pypy-string-formatting/rpython/test/test_rlist.py Sat May 12 14:35:31 2007 @@ -1181,6 +1181,15 @@ res = self.interpret(f, [58]) assert self.ll_to_string(res) == 'abhello58' + def test_unicharlist_extension_1(self): + def f(n): + s = 'hello%d' % n + l = [u'a', u'b'] + l += s + return ''.join([chr(ord(c)) for c in l]) + res = self.interpret(f, [58]) + assert self.ll_to_string(res) == 'abhello58' + def test_extend_a_non_char_list_1(self): def f(n): s = 'hello%d' % n @@ -1199,6 +1208,16 @@ res = self.interpret(f, [9381701, 3]) assert self.ll_to_string(res) == 'ablo9381701' + def test_unicharlist_extension_2(self): + def f(n, i): + s = 'hello%d' % n + assert 0 <= i <= len(s) + l = [u'a', u'b'] + l += s[i:] + return ''.join([chr(ord(c)) for c in l]) + res = self.interpret(f, [9381701, 3]) + assert self.ll_to_string(res) == 'ablo9381701' + def test_extend_a_non_char_list_2(self): def f(n, i): s = 'hello%d' % n @@ -1218,6 +1237,16 @@ res = self.interpret(f, [9381701, 3, 7]) assert self.ll_to_string(res) == 'ablo93' + def test_unicharlist_extension_3(self): + def f(n, i, j): + s = 'hello%d' % n + assert 0 <= i <= j <= len(s) + l = [u'a', u'b'] + l += s[i:j] + return ''.join([chr(ord(c)) for c in l]) + res = self.interpret(f, [9381701, 3, 7]) + assert self.ll_to_string(res) == 'ablo93' + def test_charlist_extension_4(self): def f(n): s = 'hello%d' % n @@ -1227,6 +1256,15 @@ res = self.interpret(f, [9381701]) assert self.ll_to_string(res) == 'abhello938170' + def test_unicharlist_extension_4(self): + def f(n): + s = 'hello%d' % n + l = [u'a', u'b'] + l += s[:-1] + return ''.join([chr(ord(c)) for c in l]) + res = self.interpret(f, [9381701]) + assert self.ll_to_string(res) == 'abhello938170' + def test_charlist_extension_5(self): def f(count): l = ['a', 'b'] @@ -1237,6 +1275,16 @@ res = self.interpret(f, [0]) assert self.ll_to_string(res) == 'ab' + def test_unicharlist_extension_5(self): + def f(count): + l = [u'a', u'b'] + l += '.' * count # NON-UNICODE-char * count + return ''.join([chr(ord(c)) for c in l]) + res = self.interpret(f, [7]) + assert self.ll_to_string(res) == 'ab.......' + res = self.interpret(f, [0]) + assert self.ll_to_string(res) == 'ab' + def test_charlist_extension_6(self): def f(count): l = ['a', 'b'] From fijal at codespeak.net Sat May 12 14:48:51 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 12 May 2007 14:48:51 +0200 (CEST) Subject: [pypy-svn] r43293 - pypy/dist/pypy/doc Message-ID: <20070512124851.B6B91807C@code0.codespeak.net> Author: fijal Date: Sat May 12 14:48:51 2007 New Revision: 43293 Modified: pypy/dist/pypy/doc/download.txt Log: Added missing binaries Modified: pypy/dist/pypy/doc/download.txt ============================================================================== --- pypy/dist/pypy/doc/download.txt (original) +++ pypy/dist/pypy/doc/download.txt Sat May 12 14:48:51 2007 @@ -12,9 +12,9 @@ PyPy 1.0 prebuilt binaries: ---------------------------- * `pypy-1.0.0-win32.zip`_ (precompiled executables for windows) -* `pypy-1.0.0-dynamic.tar.bz2` (precombiled binaries for linux, dynamic +* `pypy-1.0.0-dynamic.tar.bz2`_ (precombiled binaries for linux, dynamic executable) -* `pypy-1.0.0-static.tar.bz2` (precombiled binaries for linux, static +* `pypy-1.0.0-static.tar.bz2`_ (precombiled binaries for linux, static executable, GLIBC 2.5) * `pypy-1.0.0-minimal-dynamic.tar.bz2`_ (precompiled minimal binary for linux, dynamic executable) @@ -27,4 +27,6 @@ .. _`pypy-1.0.0-win32.zip`: http://codespeak.net/download/pypy/pypy-1.0.0-win32.zip .. _`pypy-1.0.0-minimal-dynamic.tar.bz2`: http://codespeak.net/download/pypy/pypy-1.0.0-minimal.tar.bz2 .. _`pypy-1.0.0-minimal-static.tar.bz2`: http://codespeak.net/download/pypy/pypy-1.0.0-minimal-static.tar.bz2 +.. _`pypy-1.0.0-dynamic.tar.bz2`: http://codespeak.net/download/pypy/pypy-1.0.0-dynamic.tar.bz2 +.. _`pypy-1.0.0-static.tar.bz2`: http://codespeak.net/download/pypy/pypy-1.0.0-static.tar.bz2 From arigo at codespeak.net Sat May 12 14:55:27 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 12 May 2007 14:55:27 +0200 (CEST) Subject: [pypy-svn] r43294 - pypy/branch/pypy-string-formatting/objspace/std Message-ID: <20070512125527.790D2807C@code0.codespeak.net> Author: arigo Date: Sat May 12 14:55:26 2007 New Revision: 43294 Modified: pypy/branch/pypy-string-formatting/objspace/std/marshal_impl.py Log: Support marshal directly from the built-in W_ComplexObject type. This probably fixes an annotation problem. Modified: pypy/branch/pypy-string-formatting/objspace/std/marshal_impl.py ============================================================================== --- pypy/branch/pypy-string-formatting/objspace/std/marshal_impl.py (original) +++ pypy/branch/pypy-string-formatting/objspace/std/marshal_impl.py Sat May 12 14:55:26 2007 @@ -20,6 +20,7 @@ from pypy.interpreter import gateway from pypy.objspace.std.boolobject import W_BoolObject +from pypy.objspace.std.complexobject import W_ComplexObject from pypy.objspace.std.intobject import W_IntObject from pypy.objspace.std.floatobject import W_FloatObject from pypy.objspace.std.tupleobject import W_TupleObject @@ -207,12 +208,10 @@ space.wrap(u.get_pascal())) register(TYPE_FLOAT + TYPE_BINARY_FLOAT, unmarshal_Float) -# this is not a native type, yet, so we have to -# dispatch on it in ANY - -def marshal_w_Complex(space, w_complex, m): - w_real = space.getattr(w_complex, space.wrap('real')) - w_imag = space.getattr(w_complex, space.wrap('imag')) +def marshal_w__Complex(space, w_complex, m): + # XXX a bit too wrap-happy + w_real = space.wrap(w_complex.realval) + w_imag = space.wrap(w_complex.imagval) if m.version > 1: m.start(TYPE_BINARY_COMPLEX) m.put(space.str_w(float_to_str(space, w_real))) @@ -222,8 +221,6 @@ m.put_pascal(space.str_w(repr_float(space, w_real))) m.put_pascal(space.str_w(repr_float(space, w_imag))) -handled_by_any.append( ('complex', marshal_w_Complex) ) - def unmarshal_Complex(space, u, tc): if tc == TYPE_BINARY_COMPLEX: w_real = str_to_float(space, space.wrap(u.get(8))) From pedronis at codespeak.net Sat May 12 15:05:37 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sat, 12 May 2007 15:05:37 +0200 (CEST) Subject: [pypy-svn] r43295 - pypy/dist/pypy/doc Message-ID: <20070512130537.40F3D807C@code0.codespeak.net> Author: pedronis Date: Sat May 12 15:05:36 2007 New Revision: 43295 Modified: pypy/dist/pypy/doc/download.txt Log: oops, comment out out the removed binaries need to sort out license issues first Modified: pypy/dist/pypy/doc/download.txt ============================================================================== --- pypy/dist/pypy/doc/download.txt (original) +++ pypy/dist/pypy/doc/download.txt Sat May 12 15:05:36 2007 @@ -12,14 +12,16 @@ PyPy 1.0 prebuilt binaries: ---------------------------- * `pypy-1.0.0-win32.zip`_ (precompiled executables for windows) -* `pypy-1.0.0-dynamic.tar.bz2`_ (precombiled binaries for linux, dynamic - executable) -* `pypy-1.0.0-static.tar.bz2`_ (precombiled binaries for linux, static - executable, GLIBC 2.5) -* `pypy-1.0.0-minimal-dynamic.tar.bz2`_ (precompiled minimal binary for - linux, dynamic executable) -* `pypy-1.0.0-minimal-static.tar.bz2`_ (precompiled minimal binary for - linux, statix executable, GLIBC 2.5) + +.. disable + * `pypy-1.0.0-dynamic.tar.bz2`_ (precombiled binaries for linux, dynamic + executable) + * `pypy-1.0.0-static.tar.bz2`_ (precombiled binaries for linux, static + executable, GLIBC 2.5) + * `pypy-1.0.0-minimal-dynamic.tar.bz2`_ (precompiled minimal binary for + linux, dynamic executable) + * `pypy-1.0.0-minimal-static.tar.bz2`_ (precompiled minimal binary for + linux, statix executable, GLIBC 2.5) .. _`pypy-1.0.0.tar.bz2`: http://codespeak.net/download/pypy/pypy-1.0.0.tar.bz2 .. _`pypy-1.0.0.zip`: http://codespeak.net/download/pypy/pypy-1.0.0.zip From arigo at codespeak.net Sat May 12 15:24:27 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 12 May 2007 15:24:27 +0200 (CEST) Subject: [pypy-svn] r43296 - in pypy/branch/pypy-string-formatting/objspace/std: . test Message-ID: <20070512132427.02D53807C@code0.codespeak.net> Author: arigo Date: Sat May 12 15:24:27 2007 New Revision: 43296 Modified: pypy/branch/pypy-string-formatting/objspace/std/formatting.py pypy/branch/pypy-string-formatting/objspace/std/test/test_stringformat.py Log: Test for typos. Modified: pypy/branch/pypy-string-formatting/objspace/std/formatting.py ============================================================================== --- pypy/branch/pypy-string-formatting/objspace/std/formatting.py (original) +++ pypy/branch/pypy-string-formatting/objspace/std/formatting.py Sat May 12 15:24:27 2007 @@ -285,6 +285,7 @@ return result def unknown_fmtchar(self): + space = self.space self.fmtpos -= 1 c = self.peekchr() if do_unicode: @@ -292,14 +293,13 @@ space.sys.get('getdefaultencoding')) w_s = space.call_method(space.newunicode([c]), "encode", - [w_defaultencoding, - space.wrap('replace')]) + w_defaultencoding, + space.wrap('replace')) s = space.str_w(w_s) else: s = c msg = "unsupported format character '%s' (0x%x) at index %d" % ( s, ord(c), self.fmtpos) - space = self.space raise OperationError(space.w_ValueError, space.wrap(msg)) def std_wp(self, r): Modified: pypy/branch/pypy-string-formatting/objspace/std/test/test_stringformat.py ============================================================================== --- pypy/branch/pypy-string-formatting/objspace/std/test/test_stringformat.py (original) +++ pypy/branch/pypy-string-formatting/objspace/std/test/test_stringformat.py Sat May 12 15:24:27 2007 @@ -214,3 +214,6 @@ def test_missing_cases(self): print '%032d' % -123456789012345678901234567890L assert '%032d' % -123456789012345678901234567890L == '-0123456789012345678901234567890' + + def test_invalid_char(self): + raises(ValueError, 'u"%\u1234" % (4,)') From arigo at codespeak.net Sat May 12 15:38:09 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 12 May 2007 15:38:09 +0200 (CEST) Subject: [pypy-svn] r43297 - in pypy/branch/pypy-string-formatting/rpython: . test Message-ID: <20070512133809.E3383807C@code0.codespeak.net> Author: arigo Date: Sat May 12 15:38:09 2007 New Revision: 43297 Modified: pypy/branch/pypy-string-formatting/rpython/rstr.py pypy/branch/pypy-string-formatting/rpython/test/test_rstr.py Log: Bug, test, fix. Modified: pypy/branch/pypy-string-formatting/rpython/rstr.py ============================================================================== --- pypy/branch/pypy-string-formatting/rpython/rstr.py (original) +++ pypy/branch/pypy-string-formatting/rpython/rstr.py Sat May 12 15:38:09 2007 @@ -437,6 +437,8 @@ class __extend__(AbstractUniCharRepr): def convert_const(self, value): + if isinstance(value, str): + value = unicode(value) if not isinstance(value, unicode) or len(value) != 1: raise TyperError("not a unicode character: %r" % (value,)) return value Modified: pypy/branch/pypy-string-formatting/rpython/test/test_rstr.py ============================================================================== --- pypy/branch/pypy-string-formatting/rpython/test/test_rstr.py (original) +++ pypy/branch/pypy-string-formatting/rpython/test/test_rstr.py Sat May 12 15:38:09 2007 @@ -168,6 +168,13 @@ assert self.interpret(fn, ['(', u'(']) == True assert self.interpret(fn, ['(', u'\u1028']) == False + def test_char_unichar_eq_2(self): + def fn(c1): + return c1 == 'X' + assert self.interpret(fn, [u'(']) == False + assert self.interpret(fn, [u'\u1058']) == False + assert self.interpret(fn, [u'X']) == True + def test_is_none(self): def fn(i): s1 = ['foo', None][i] From arigo at codespeak.net Sat May 12 16:20:58 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 12 May 2007 16:20:58 +0200 (CEST) Subject: [pypy-svn] r43298 - in pypy/branch/pypy-string-formatting/annotation: . test Message-ID: <20070512142058.1D406807F@code0.codespeak.net> Author: arigo Date: Sat May 12 16:20:56 2007 New Revision: 43298 Modified: pypy/branch/pypy-string-formatting/annotation/builtin.py pypy/branch/pypy-string-formatting/annotation/test/test_annrpython.py Log: Annotation: int(mybool) and int(mynonnegint) should return a *nonneg* integer. Modified: pypy/branch/pypy-string-formatting/annotation/builtin.py ============================================================================== --- pypy/branch/pypy-string-formatting/annotation/builtin.py (original) +++ pypy/branch/pypy-string-formatting/annotation/builtin.py Sat May 12 16:20:56 2007 @@ -91,7 +91,8 @@ args_s = [s_obj, s_base] else: args_s = [s_obj] - return constpropagate(int, args_s, SomeInteger()) + nonneg = isinstance(s_obj, SomeInteger) and s_obj.nonneg + return constpropagate(int, args_s, SomeInteger(nonneg=nonneg)) def builtin_float(s_obj): return constpropagate(float, [s_obj], SomeFloat()) Modified: pypy/branch/pypy-string-formatting/annotation/test/test_annrpython.py ============================================================================== --- pypy/branch/pypy-string-formatting/annotation/test/test_annrpython.py (original) +++ pypy/branch/pypy-string-formatting/annotation/test/test_annrpython.py Sat May 12 16:20:56 2007 @@ -1589,6 +1589,15 @@ s = a.build_types(f, [int, str]) assert s.knowntype == int + def test_int_nonneg(self): + def f(x, y): + assert x >= 0 + return int(x) + int(y == 3) + a = self.RPythonAnnotator() + s = a.build_types(f, [int, int]) + assert isinstance(s, annmodel.SomeInteger) + assert s.nonneg + def test_listitem_merge_asymmetry_bug(self): class K: pass From arigo at codespeak.net Sat May 12 18:12:32 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 12 May 2007 18:12:32 +0200 (CEST) Subject: [pypy-svn] r43302 - pypy/branch/pypy-string-formatting/objspace/std Message-ID: <20070512161232.33334807F@code0.codespeak.net> Author: arigo Date: Sat May 12 18:12:31 2007 New Revision: 43302 Modified: pypy/branch/pypy-string-formatting/objspace/std/formatting.py Log: Translation fixes. Modified: pypy/branch/pypy-string-formatting/objspace/std/formatting.py ============================================================================== --- pypy/branch/pypy-string-formatting/objspace/std/formatting.py (original) +++ pypy/branch/pypy-string-formatting/objspace/std/formatting.py Sat May 12 18:12:31 2007 @@ -196,7 +196,8 @@ else: self.prec = -1 - if self.peekchr() in 'hlL': + c = self.peekchr() + if c == 'h' or c == 'l' or c == 'L': self.forward() return w_value @@ -231,8 +232,10 @@ w_value = self.nextinputvalue() return space.int_w(maybe_int(space, w_value)) result = 0 - while '0' <= c <= '9': + while True: n = ord(c) - ord('0') + if not (0 <= n < 10): + break try: result = ovfcheck(ovfcheck(result * 10) + n) except OverflowError: @@ -286,8 +289,7 @@ def unknown_fmtchar(self): space = self.space - self.fmtpos -= 1 - c = self.peekchr() + c = self.fmt[self.fmtpos - 1] if do_unicode: w_defaultencoding = space.call_function( space.sys.get('getdefaultencoding')) @@ -304,8 +306,9 @@ def std_wp(self, r): length = len(r) - if self.prec >= 0 and self.prec < length: - length = self.prec # ignore the end of the string if too long + prec = self.prec + if prec >= 0 and prec < length: + length = prec # ignore the end of the string if too long result = self.result padding = self.width - length if not self.f_ljust: From fijal at codespeak.net Sat May 12 19:13:53 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 12 May 2007 19:13:53 +0200 (CEST) Subject: [pypy-svn] r43303 - pypy/dist/pypy/doc Message-ID: <20070512171353.CFE5A807F@code0.codespeak.net> Author: fijal Date: Sat May 12 19:13:52 2007 New Revision: 43303 Modified: pypy/dist/pypy/doc/download.txt Log: Added binaries without readline, updated. Modified: pypy/dist/pypy/doc/download.txt ============================================================================== --- pypy/dist/pypy/doc/download.txt (original) +++ pypy/dist/pypy/doc/download.txt Sat May 12 19:13:52 2007 @@ -12,23 +12,21 @@ PyPy 1.0 prebuilt binaries: ---------------------------- * `pypy-1.0.0-win32.zip`_ (precompiled executables for windows) - -.. disable - * `pypy-1.0.0-dynamic.tar.bz2`_ (precombiled binaries for linux, dynamic +* `pypy-1.0.0-bin.tar.bz2`_ (precombiled binaries for linux, dynamic executable) - * `pypy-1.0.0-static.tar.bz2`_ (precombiled binaries for linux, static +* `pypy-1.0.0-static-bin.tar.bz2`_ (precombiled binaries for linux, static executable, GLIBC 2.5) - * `pypy-1.0.0-minimal-dynamic.tar.bz2`_ (precompiled minimal binary for +* `pypy-1.0.0-minimal-bin.tar.bz2`_ (precompiled minimal binary for linux, dynamic executable) - * `pypy-1.0.0-minimal-static.tar.bz2`_ (precompiled minimal binary for +* `pypy-1.0.0-minimal-bin-static.tar.bz2`_ (precompiled minimal binary for linux, statix executable, GLIBC 2.5) .. _`pypy-1.0.0.tar.bz2`: http://codespeak.net/download/pypy/pypy-1.0.0.tar.bz2 .. _`pypy-1.0.0.zip`: http://codespeak.net/download/pypy/pypy-1.0.0.zip .. _`pypy-1.0.0.tar.gz`: http://codespeak.net/download/pypy/pypy-1.0.0.tar.gz .. _`pypy-1.0.0-win32.zip`: http://codespeak.net/download/pypy/pypy-1.0.0-win32.zip -.. _`pypy-1.0.0-minimal-dynamic.tar.bz2`: http://codespeak.net/download/pypy/pypy-1.0.0-minimal.tar.bz2 -.. _`pypy-1.0.0-minimal-static.tar.bz2`: http://codespeak.net/download/pypy/pypy-1.0.0-minimal-static.tar.bz2 -.. _`pypy-1.0.0-dynamic.tar.bz2`: http://codespeak.net/download/pypy/pypy-1.0.0-dynamic.tar.bz2 -.. _`pypy-1.0.0-static.tar.bz2`: http://codespeak.net/download/pypy/pypy-1.0.0-static.tar.bz2 +.. _`pypy-1.0.0-minimal-bin.tar.bz2`: http://codespeak.net/download/pypy/pypy-1.0.0-minimal-bin.tar.bz2 +.. _`pypy-1.0.0-minimal-bin-static.tar.bz2`: http://codespeak.net/download/pypy/pypy-1.0.0-minimal-static-bin.tar.bz2 +.. _`pypy-1.0.0-bin.tar.bz2`: http://codespeak.net/download/pypy/pypy-1.0.0-bin.tar.bz2 +.. _`pypy-1.0.0-static-bin.tar.bz2`: http://codespeak.net/download/pypy/pypy-1.0.0-static-bin.tar.bz2 From pedronis at codespeak.net Sun May 13 14:25:56 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sun, 13 May 2007 14:25:56 +0200 (CEST) Subject: [pypy-svn] r43306 - pypy/dist/pypy/doc Message-ID: <20070513122556.9AB6F8082@code0.codespeak.net> Author: pedronis Date: Sun May 13 14:25:55 2007 New Revision: 43306 Modified: pypy/dist/pypy/doc/news.txt Log: fix formatting and date format Modified: pypy/dist/pypy/doc/news.txt ============================================================================== --- pypy/dist/pypy/doc/news.txt (original) +++ pypy/dist/pypy/doc/news.txt Sun May 13 14:25:55 2007 @@ -23,7 +23,7 @@ Activity Report`_ summarizes what we did and what we have in mind on technical, scientific and community levels. It also contains reflections and recommendations possibly interesting to other projects aiming at -EU funded Open Source research.*(2007-05-12)* +EU funded Open Source research. *(12th May, 2007)* .. _`submitted last documents`: http://codespeak.net/pypy/dist/pypy/doc/index-report.html .. _`PyPy EU Final Activity Report`: http://codespeak.net/pypy/extradoc/eu-report/PYPY-EU-Final-Activity-Report.pdf From pedronis at codespeak.net Sun May 13 14:27:36 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sun, 13 May 2007 14:27:36 +0200 (CEST) Subject: [pypy-svn] r43307 - pypy/extradoc/pypy.org Message-ID: <20070513122736.29F7F8081@code0.codespeak.net> Author: pedronis Date: Sun May 13 14:27:33 2007 New Revision: 43307 Modified: pypy/extradoc/pypy.org/news.txt Log: formatting and date format Modified: pypy/extradoc/pypy.org/news.txt ============================================================================== --- pypy/extradoc/pypy.org/news.txt (original) +++ pypy/extradoc/pypy.org/news.txt Sun May 13 14:27:33 2007 @@ -7,7 +7,7 @@ Activity Report`_ summarizes what we did and what we have in mind on technical, scientific and community levels. It also contains reflections and recommendations possibly interesting to other projects aiming at -EU funded Open Source research.*(2007-05-12)* +EU funded Open Source research. *(12th May, 2007)* .. _`submitted last documents`: http://codespeak.net/pypy/dist/pypy/doc/index-report.html .. _`PyPy EU Final Activity Report`: http://codespeak.net/pypy/extradoc/eu-report/PYPY-EU-Final-Activity-Report.pdf From arigo at codespeak.net Sun May 13 23:25:45 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 13 May 2007 23:25:45 +0200 (CEST) Subject: [pypy-svn] r43321 - in pypy/branch/pypy-string-formatting/objspace/std: . test Message-ID: <20070513212545.8F0BF808F@code0.codespeak.net> Author: arigo Date: Sun May 13 23:25:44 2007 New Revision: 43321 Modified: pypy/branch/pypy-string-formatting/objspace/std/formatting.py pypy/branch/pypy-string-formatting/objspace/std/test/test_stringformat.py Log: Bug and test. Modified: pypy/branch/pypy-string-formatting/objspace/std/formatting.py ============================================================================== --- pypy/branch/pypy-string-formatting/objspace/std/formatting.py (original) +++ pypy/branch/pypy-string-formatting/objspace/std/formatting.py Sun May 13 23:25:44 2007 @@ -151,7 +151,12 @@ i0 = i pcount = 1 while 1: - c = fmt[i] + try: + c = fmt[i] + except IndexError: + space = self.space + raise OperationError(space.w_ValueError, + space.wrap("incomplete format key")) if c == ')': pcount -= 1 if pcount == 0: Modified: pypy/branch/pypy-string-formatting/objspace/std/test/test_stringformat.py ============================================================================== --- pypy/branch/pypy-string-formatting/objspace/std/test/test_stringformat.py (original) +++ pypy/branch/pypy-string-formatting/objspace/std/test/test_stringformat.py Sun May 13 23:25:44 2007 @@ -128,6 +128,7 @@ def test_incomplete_format(self): raises(ValueError, '%'.__mod__, ((23,),)) + raises(ValueError, '%('.__mod__, ({},)) def test_format_char(self): assert '%c' % 65 == 'A' From cfbolz at codespeak.net Mon May 14 15:20:16 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 14 May 2007 15:20:16 +0200 (CEST) Subject: [pypy-svn] r43364 - pypy/dist/pypy/lang/prolog/interpreter Message-ID: <20070514132016.A56F48080@code0.codespeak.net> Author: cfbolz Date: Mon May 14 15:20:16 2007 New Revision: 43364 Modified: pypy/dist/pypy/lang/prolog/interpreter/engine.py pypy/dist/pypy/lang/prolog/interpreter/term.py Log: simplify the unify hash thing a bit Modified: pypy/dist/pypy/lang/prolog/interpreter/engine.py ============================================================================== --- pypy/dist/pypy/lang/prolog/interpreter/engine.py (original) +++ pypy/dist/pypy/lang/prolog/interpreter/engine.py Mon May 14 15:20:16 2007 @@ -104,13 +104,14 @@ curr = curr.next return first, copy - def find_applicable_rule(self, query, uh1): + def find_applicable_rule(self, query): #import pdb;pdb.set_trace() while self: - uh2 = self.rule.unify_hash - assert len(uh1) == len(uh2) - for j in range(len(uh1)): - if uh1[j] != 0 and uh2[j] != 0 and uh1[j] != uh2[j]: + uh = self.rule.unify_hash + for j in range(len(uh)): + hash1 = uh[j] + hash2 = query.unify_hash_of_child(j) + if hash1 != 0 and hash2 != 0 and hash1 != hash2: break else: return self @@ -215,8 +216,7 @@ if function is None: error.throw_existence_error( "procedure", query.get_prolog_signature()) - unify_hash = query.get_deeper_unify_hash(self.heap) - rulechain = function.rulechain.find_applicable_rule(query, unify_hash) + rulechain = function.rulechain.find_applicable_rule(query) if rulechain is None: # none of the rules apply raise UnificationFailed() @@ -224,7 +224,7 @@ rulechain = rulechain.next oldstate = self.heap.branch() while rulechain: - rulechain = rulechain.find_applicable_rule(query, unify_hash) + rulechain = rulechain.find_applicable_rule(query) if rulechain is None: self.heap.discard(oldstate) break Modified: pypy/dist/pypy/lang/prolog/interpreter/term.py ============================================================================== --- pypy/dist/pypy/lang/prolog/interpreter/term.py (original) +++ pypy/dist/pypy/lang/prolog/interpreter/term.py Mon May 14 15:20:16 2007 @@ -38,13 +38,13 @@ def clone_compress_vars(self, vars_new_indexes, offset): return self - def get_unify_hash(self, heap=None): + def get_unify_hash(self): # if two non-var objects return two different numbers # they must not be unifiable raise NotImplementedError("abstract base class") - def get_deeper_unify_hash(self, heap=None): - return [self.get_unify_hash(heap)] + def unify_hash_of_child(self, i, heap=None): + raise KeyError @specialize.arg(3) def unify(self, other, heap, occurs_check=False): @@ -136,13 +136,8 @@ vars_new_indexes[self.index] = index return Var(index) - def get_unify_hash(self, heap=None): - if heap is None: - return 0 - self = self.dereference(heap) - if isinstance(self, Var): - return 0 - return self.get_unify_hash(heap) + def get_unify_hash(self): + return 0 def contains_var(self, var, heap): self = self.dereference(heap) @@ -238,7 +233,7 @@ else: raise UnificationFailed - def get_unify_hash(self, heap=None): + def get_unify_hash(self): return intmask(hash(self.name) << TAGBITS | self.TAG) def get_prolog_signature(self): @@ -281,7 +276,7 @@ def __repr__(self): return "Number(%r)" % (self.num, ) - def get_unify_hash(self, heap=None): + def get_unify_hash(self): return intmask(self.num << TAGBITS | self.TAG) @@ -307,7 +302,7 @@ else: raise UnificationFailed - def get_unify_hash(self, heap=None): + def get_unify_hash(self): #XXX no clue whether this is a good idea... m, e = math.frexp(self.num) m = intmask(int(m / 2 * 2 ** (32 - TAGBITS))) @@ -339,7 +334,7 @@ else: raise UnificationFailed - def get_unify_hash(self, heap=None): + def get_unify_hash(self): return intmask(id(self) << TAGBITS | self.TAG) @@ -436,14 +431,11 @@ else: return self - def get_unify_hash(self, heap=None): + def get_unify_hash(self): return intmask(hash(self.signature) << TAGBITS | self.TAG) - def get_deeper_unify_hash(self, heap=None): - result = [0] * len(self.args) - for i in range(len(self.args)): - result[i] = self.args[i].get_unify_hash(heap) - return result + def unify_hash_of_child(self, i): + return self.args[i].get_unify_hash() def get_prolog_signature(self): return Term("/", [Atom.newatom(self.name), Number(len(self.args))]) @@ -456,6 +448,7 @@ class Rule(object): + unify_hash = [] def __init__(self, head, body): from pypy.lang.prolog.interpreter import helper d = {} @@ -469,7 +462,8 @@ self.body = None self.numvars = len(d) self.signature = self.head.signature - self.unify_hash = self.head.get_deeper_unify_hash() + if isinstance(head, Term): + self.unify_hash = [arg.get_unify_hash() for arg in head.args] self._does_contain_cut() def _does_contain_cut(self): From cfbolz at codespeak.net Mon May 14 15:24:05 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 14 May 2007 15:24:05 +0200 (CEST) Subject: [pypy-svn] r43366 - pypy/branch/prolog-jit-experiments Message-ID: <20070514132405.26C238080@code0.codespeak.net> Author: cfbolz Date: Mon May 14 15:24:04 2007 New Revision: 43366 Added: pypy/branch/prolog-jit-experiments/ - copied from r43364, pypy/dist/ Log: Be afraid. Very afraid. A branch to experiment with rewriting the Prolog interpreter to make it work with the JIT. From cfbolz at codespeak.net Mon May 14 15:29:25 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 14 May 2007 15:29:25 +0200 (CEST) Subject: [pypy-svn] r43368 - pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter Message-ID: <20070514132925.C41CD8081@code0.codespeak.net> Author: cfbolz Date: Mon May 14 15:29:25 2007 New Revision: 43368 Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/term.py Log: the jit doesn't like for-loops Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/term.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/term.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/term.py Mon May 14 15:29:25 2007 @@ -20,6 +20,11 @@ class PrologObject(object): __slots__ = () + + def __init__(self): + raise NotImplementedError("abstract base class") + return self + def getvalue(self, heap): return self @@ -318,6 +323,9 @@ # meant to be subclassed TAG = tag() STANDARD_ORDER = 4 + def __init__(self): + pass + @specialize.arg(3) def basic_unify(self, other, heap, occurs_check=False): if self is other: @@ -486,10 +494,13 @@ if isinstance(head, Term): h2 = self.head assert isinstance(h2, Term) - for i in range(len(h2.args)): - arg1 = h2.args[i] - arg2 = head.args[i] - arg1.copy_and_unify(arg2, heap, memo) + i = 0 + while i < len(h2.args): + i = hint(i, concrete=True) + arg2 = h2.args[i] + arg1 = head.args[i] + arg2.copy_and_unify(arg1, heap, memo) + i += 1 body = self.body if body is None: return None From arigo at codespeak.net Mon May 14 18:18:56 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 14 May 2007 18:18:56 +0200 (CEST) Subject: [pypy-svn] r43374 - pypy/dist/pypy/annotation/test Message-ID: <20070514161856.1D0F7807A@code0.codespeak.net> Author: arigo Date: Mon May 14 18:18:54 2007 New Revision: 43374 Added: pypy/dist/pypy/annotation/test/test_annrpython.py.merge.tmp - copied, changed from r43373, pypy/dist/pypy/annotation/test/test_annrpython.py Log: merging of svn+ssh://codespeak.net/svn/pypy/branch/pypy-string-formatting/annotation/test/test_annrpython.py revisions 42144 to 43373: ------------------------------------------------------------------------ r43298 | arigo | 2007-05-12 16:20:56 +0200 (Sat, 12 May 2007) | 3 lines Annotation: int(mybool) and int(mynonnegint) should return a *nonneg* integer. ------------------------------------------------------------------------ From arigo at codespeak.net Mon May 14 18:20:01 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 14 May 2007 18:20:01 +0200 (CEST) Subject: [pypy-svn] r43375 - in pypy/dist/pypy: annotation annotation/test interpreter module/__builtin__ module/math objspace/cpy objspace/fake objspace/std objspace/std/test rlib rlib/test rpython rpython/test translator Message-ID: <20070514162001.7E40F8080@code0.codespeak.net> Author: arigo Date: Mon May 14 18:20:01 2007 New Revision: 43375 Added: pypy/dist/pypy/annotation/binaryop.py - copied unchanged from r43374, pypy/branch/pypy-string-formatting/annotation/binaryop.py pypy/dist/pypy/annotation/builtin.py - copied unchanged from r43374, pypy/branch/pypy-string-formatting/annotation/builtin.py pypy/dist/pypy/annotation/test/test_annrpython.py - copied unchanged from r43374, pypy/dist/pypy/annotation/test/test_annrpython.py.merge.tmp pypy/dist/pypy/interpreter/ - copied from r43374, pypy/branch/pypy-string-formatting/interpreter/ pypy/dist/pypy/module/__builtin__/ - copied from r43374, pypy/branch/pypy-string-formatting/module/__builtin__/ pypy/dist/pypy/module/math/ - copied from r43374, pypy/branch/pypy-string-formatting/module/math/ pypy/dist/pypy/objspace/cpy/ - copied from r43374, pypy/branch/pypy-string-formatting/objspace/cpy/ pypy/dist/pypy/objspace/fake/ - copied from r43374, pypy/branch/pypy-string-formatting/objspace/fake/ pypy/dist/pypy/objspace/std/floatobject.py - copied unchanged from r43374, pypy/branch/pypy-string-formatting/objspace/std/floatobject.py pypy/dist/pypy/objspace/std/formatting.py - copied unchanged from r43374, pypy/branch/pypy-string-formatting/objspace/std/formatting.py pypy/dist/pypy/objspace/std/longobject.py - copied unchanged from r43374, pypy/branch/pypy-string-formatting/objspace/std/longobject.py pypy/dist/pypy/objspace/std/marshal_impl.py - copied unchanged from r43374, pypy/branch/pypy-string-formatting/objspace/std/marshal_impl.py pypy/dist/pypy/objspace/std/stringobject.py - copied unchanged from r43374, pypy/branch/pypy-string-formatting/objspace/std/stringobject.py pypy/dist/pypy/objspace/std/test/test_stringformat.py - copied unchanged from r43374, pypy/branch/pypy-string-formatting/objspace/std/test/test_stringformat.py pypy/dist/pypy/objspace/std/unicodeobject.py - copied unchanged from r43374, pypy/branch/pypy-string-formatting/objspace/std/unicodeobject.py pypy/dist/pypy/rlib/rarithmetic.py - copied unchanged from r43374, pypy/branch/pypy-string-formatting/rlib/rarithmetic.py pypy/dist/pypy/rlib/rbigint.py - copied unchanged from r43374, pypy/branch/pypy-string-formatting/rlib/rbigint.py pypy/dist/pypy/rlib/test/ - copied from r43374, pypy/branch/pypy-string-formatting/rlib/test/ pypy/dist/pypy/rpython/rlist.py - copied unchanged from r43374, pypy/branch/pypy-string-formatting/rpython/rlist.py pypy/dist/pypy/rpython/rstr.py - copied unchanged from r43374, pypy/branch/pypy-string-formatting/rpython/rstr.py pypy/dist/pypy/rpython/rtyper.py - copied unchanged from r43374, pypy/branch/pypy-string-formatting/rpython/rtyper.py pypy/dist/pypy/rpython/test/ - copied from r43374, pypy/branch/pypy-string-formatting/rpython/test/ pypy/dist/pypy/translator/transform.py - copied unchanged from r43374, pypy/branch/pypy-string-formatting/translator/transform.py Removed: pypy/dist/pypy/annotation/test/test_annrpython.py.merge.tmp Log: Merge the string formatting branch. From arigo at codespeak.net Mon May 14 18:21:46 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 14 May 2007 18:21:46 +0200 (CEST) Subject: [pypy-svn] r43376 - pypy/branch/pypy-string-formatting Message-ID: <20070514162146.BF2C38080@code0.codespeak.net> Author: arigo Date: Mon May 14 18:21:46 2007 New Revision: 43376 Removed: pypy/branch/pypy-string-formatting/ Log: Delete merged branch. From cfbolz at codespeak.net Mon May 14 21:05:09 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 14 May 2007 21:05:09 +0200 (CEST) Subject: [pypy-svn] r43382 - in pypy/branch/prolog-jit-experiments/pypy: lang/prolog/interpreter translator/goal Message-ID: <20070514190509.7279D807A@code0.codespeak.net> Author: cfbolz Date: Mon May 14 21:05:08 2007 New Revision: 43382 Added: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/portal.py (contents, props changed) Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py pypy/branch/prolog-jit-experiments/pypy/translator/goal/targetprologstandalone.py Log: steps in the direction of jitting prolog: add a portal file (which is a copy of the one in module/pypyjit/portal.py) in many parts, add a few hints and change the target a bit. Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py Mon May 14 21:05:08 2007 @@ -3,6 +3,7 @@ from pypy.lang.prolog.interpreter.error import UnificationFailed, FunctionNotFound, \ CutException from pypy.lang.prolog.interpreter import error +from pypy.rlib.objectmodel import hint, specialize DEBUG = True @@ -162,7 +163,7 @@ signature = rule.signature else: error.throw_type_error("callable", rule) - assert 0, "unreachable" # XXX make annotator happy + assert 0, "unreachable" # make annotator happy if signature in builtin.builtins: error.throw_permission_error( "modify", "static_procedure", rule.head.get_prolog_signature()) @@ -203,6 +204,7 @@ debug_print("calling", query) signature = query.signature # check for builtins + builtins = hint(builtins, deepfreeze=True) builtin = builtins.get(signature, None) if builtin is not None: return builtin.call(self, query, continuation) @@ -261,6 +263,9 @@ return self.try_rule(rule, query, continuation) def try_rule(self, rule, query, continuation=DONOTHING): + hint(None, global_merge_point=True) + rule = hint(rule, promote=True) + rule = hint(rule, deepfreeze=True) if DEBUG: debug_print("trying rule", rule, query, self.heap.vars[:self.heap.needed_vars]) # standardizing apart @@ -268,8 +273,9 @@ if DEBUG: debug_print("worked", rule, query, self.heap.vars[:self.heap.needed_vars]) if nextcall is not None: - return self.call(nextcall, continuation) - return continuation.call(self) + self.call(nextcall, continuation) + else: + continuation.call(self) def continue_after_cut(self, continuation, lsc=None): Added: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/portal.py ============================================================================== --- (empty file) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/portal.py Mon May 14 21:05:08 2007 @@ -0,0 +1,140 @@ +from pypy.jit.hintannotator.annotator import HintAnnotatorPolicy +from pypy.lang.prolog.interpreter import term, engine +from pypy.translator.translator import graphof +from pypy.annotation.specialize import getuniquenondirectgraph + + +forbidden_modules = {'pypy.lang.prolog.interpreter.parser': True, + } + +PORTAL = engine.Engine.try_rule + +class PyrologHintAnnotatorPolicy(HintAnnotatorPolicy): + novirtualcontainer = True + oopspec = True + + def __init__(self, timeshift_graphs): + self.timeshift_graphs = timeshift_graphs + + def look_inside_graph(self, graph): + if graph in self.timeshift_graphs: + return self.timeshift_graphs[graph] + try: + func = graph.func + except AttributeError: + return True + if hasattr(func, '_look_inside_me_'): + return func._look_inside_me_ + mod = func.__module__ or '?' + if mod in forbidden_modules: + return False +# if "pypy.lang.prolog.builtin" in mod: +# return True + return False + +def jitme(func): + func._look_inside_me_ = True + return func + +def dontjitme(func): + func._look_inside_me_ = False + return func + + +def enumerate_reachable_graphs(translator, startgraph): + from pypy.translator.backendopt.support import find_calls_from + pending = [(startgraph, None)] + yield pending[0] + seen = {startgraph: True} + while pending: + yield None # hack: a separator meaning "length increases now" + nextlengthlist = [] + nextseen = {} + for node in pending: + head, tail = node + for block, callee in find_calls_from(translator, head): + if callee not in seen: + newnode = callee, node + yield newnode + nextlengthlist.append(newnode) + nextseen[callee] = True + pending = nextlengthlist + seen.update(nextseen) + yield None + +def graphs_on_the_path_to(translator, startgraph, targetgraphs): + targetgraphs = targetgraphs.copy() + result = {} + found = {} + for node in enumerate_reachable_graphs(translator, startgraph): + if node is None: # hack: a separator meaning "length increases now" + for graph in found: + del targetgraphs[graph] + found.clear() + if not targetgraphs: + return result + elif node[0] in targetgraphs: + found[node[0]] = True + while node is not None: + head, tail = node + result[head] = True + node = tail + raise Exception("did not reach all targets:\nmissing %r" % ( + targetgraphs.keys(),)) + + +def timeshift_graphs(t, portal_graph, log): + import pypy + result_graphs = {} + + bk = t.annotator.bookkeeper + + def _graph(func): + func = getattr(func, 'im_func', func) + desc = bk.getdesc(func) + return getuniquenondirectgraph(desc) + + def seefunc(fromfunc, *tofuncs): + targetgraphs = {} + for tofunc in tofuncs: + targetgraphs[_graph(tofunc)] = True + graphs = graphs_on_the_path_to(t, _graph(fromfunc), targetgraphs) + for graph in graphs: + if graph not in result_graphs: + log('including graph %s' % (graph,)) + result_graphs[graph] = True + + def seepath(*path): + for i in range(1, len(path)): + seefunc(path[i-1], path[i]) + + def seegraph(func, look=True): + graph = _graph(func) + if look: + extra = "" + if look != True: + extra = " substituted with %s" % look + log('including graph %s%s' % (graph, extra)) + else: + log('excluding graph %s' % (graph,)) + result_graphs[graph] = look + + for cls in [term.Var, term.Term, term.Number, term.Float, term.Atom]: + seegraph(cls.copy) + seegraph(cls.__init__) + seegraph(cls.copy_and_unify) + for cls in [term.Term, term.Number, term.Atom]: + seegraph(cls.copy_and_basic_unify) + seegraph(pypy.lang.prolog.interpreter.engine.Heap.newvar) + seegraph(pypy.lang.prolog.interpreter.engine.Engine.try_rule) + seegraph(pypy.lang.prolog.interpreter.term.Rule.clone_and_unify_head) + return result_graphs + +def get_portal(drv): + t = drv.translator + portal = getattr(PORTAL, 'im_func', PORTAL) + portal_graph = graphof(t, portal) + + policy = PyrologHintAnnotatorPolicy(timeshift_graphs(t, portal_graph, + drv.log)) + return portal, policy Modified: pypy/branch/prolog-jit-experiments/pypy/translator/goal/targetprologstandalone.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/translator/goal/targetprologstandalone.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/translator/goal/targetprologstandalone.py Mon May 14 21:05:08 2007 @@ -25,11 +25,16 @@ # _____ Define and setup target ___ def handle_config(config): + return config.translation.stackless = True def target(driver, args): driver.exe_name = 'pyrolog-%(backend)s' return entry_point, None +def portal(driver): + from pypy.lang.prolog.interpreter.portal import get_portal + return get_portal(driver) + if __name__ == '__main__': entry_point(sys.argv) From cfbolz at codespeak.net Mon May 14 21:45:20 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 14 May 2007 21:45:20 +0200 (CEST) Subject: [pypy-svn] r43383 - pypy/dist/pypy/jit/timeshifter/test Message-ID: <20070514194520.8EE27807A@code0.codespeak.net> Author: cfbolz Date: Mon May 14 21:45:19 2007 New Revision: 43383 Modified: pypy/dist/pypy/jit/timeshifter/test/test_portal.py Log: a failing test Modified: pypy/dist/pypy/jit/timeshifter/test/test_portal.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/test/test_portal.py (original) +++ pypy/dist/pypy/jit/timeshifter/test/test_portal.py Mon May 14 21:45:19 2007 @@ -529,3 +529,16 @@ res = self.timeshift_from_portal(f, g, [], policy=P_NOVIRTUAL) assert res == 0 + + def test_portal_returns_none(self): + #py.test.skip("portal returning None is not supported") + def g(x): + x = hint(x, promote=True) + if x == 42: + return None + def f(x): + return g(x) + + res = self.timeshift_from_portal(f, g, [42], policy=P_NOVIRTUAL) + + From cfbolz at codespeak.net Mon May 14 21:52:25 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 14 May 2007 21:52:25 +0200 (CEST) Subject: [pypy-svn] r43384 - pypy/dist/pypy/jit/timeshifter/test Message-ID: <20070514195225.169DE807A@code0.codespeak.net> Author: cfbolz Date: Mon May 14 21:52:24 2007 New Revision: 43384 Modified: pypy/dist/pypy/jit/timeshifter/test/test_portal.py Log: another failing test Modified: pypy/dist/pypy/jit/timeshifter/test/test_portal.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/test/test_portal.py (original) +++ pypy/dist/pypy/jit/timeshifter/test/test_portal.py Mon May 14 21:52:24 2007 @@ -531,7 +531,7 @@ assert res == 0 def test_portal_returns_none(self): - #py.test.skip("portal returning None is not supported") + py.test.skip("portal returning None is not supported") def g(x): x = hint(x, promote=True) if x == 42: @@ -541,4 +541,20 @@ res = self.timeshift_from_portal(f, g, [42], policy=P_NOVIRTUAL) + def test_portal_returns_none_with_origins(self): + py.test.skip("portal returning None is not supported") + def returnNone(): + pass + def returnNone2(): + pass + def g(x): + x = hint(x, promote=True) + if x == 42: + return returnNone() + return returnNone2() + def f(x): + return g(x) + + res = self.timeshift_from_portal(f, g, [42], policy=P_NOVIRTUAL) + From santagada at codespeak.net Mon May 14 22:37:50 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Mon, 14 May 2007 22:37:50 +0200 (CEST) Subject: [pypy-svn] r43385 - in pypy/dist/pypy/lang/js: . test Message-ID: <20070514203750.BD7C7807A@code0.codespeak.net> Author: santagada Date: Mon May 14 22:37:49 2007 New Revision: 43385 Added: pypy/dist/pypy/lang/js/astbuilder.py Modified: pypy/dist/pypy/lang/js/jsgrammar.txt pypy/dist/pypy/lang/js/jsobj.py pypy/dist/pypy/lang/js/operations.py pypy/dist/pypy/lang/js/test/test_new_parser.py Log: new astbuilder, continuing with the tests until I can migrate to use the new parser Added: pypy/dist/pypy/lang/js/astbuilder.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/js/astbuilder.py Mon May 14 22:37:49 2007 @@ -0,0 +1,96 @@ +from pypy.rlib.parsing.tree import RPythonVisitor, Symbol +from pypy.lang.js import operations + +class ASTBuilder(RPythonVisitor): + BINOP_TO_CLS = { + '+': operations.Plus, + '-': operations.Minus, + '*': operations.Mult, + '/': operations.Div, + '%': operations.Mod, + } + UNOP_TO_CLS = { + '+': operations.UPlus, + '-': operations.UMinus, + '++': operations.Increment, + '--': operations.Decrement, + } + LISTOP_TO_CLS = { + '[': operations.Array, + '{': operations.ObjectInit, + } + + def get_instance(self, symbol, cls): + if isinstance(symbol, Symbol): + source_pos = symbol.token.source_pos + # XXX some of the source positions are not perfect + return cls(None, + symbol.additional_info, + source_pos.lineno, + source_pos.columnno, + source_pos.columnno + len(symbol.additional_info)) + else: + return cls(None, '', -1, -1, -1) + + def visit_DECIMALLITERAL(self, node): + result = self.get_instance(node, operations.Number) + result.num = float(node.additional_info) + return result + + def string(self,node): + print node.additional_info + result = self.get_instance(node, operations.String) + result.strval = node.additional_info[1:-1] #XXX should do unquoting + return result + + visit_DOUBLESTRING = string + visit_SINGLESTRING = string + + def binaryop(self, node): + left = self.dispatch(node.children[0]) + for i in range((len(node.children) - 1) // 2): + op = node.children[i * 2 + 1] + result = self.get_instance( + op, self.BINOP_TO_CLS[op.additional_info]) + right = self.dispatch(node.children[i * 2 + 2]) + result.left = left + result.right = right + left = result + return left + visit_additiveexpression = binaryop + visit_multiplicativeexpression = binaryop + + def visit_unaryexpression(self, node): + op = node.children[0] + result = self.get_instance( + op, self.UNOP_TO_CLS[op.additional_info]) + child = self.dispatch(node.children[1]) + result.expr = child + result.postfix = False + return result + + def listop(self, node): + op = node.children[0] + result = self.get_instance( + op, self.LISTOP_TO_CLS[op.additional_info]) + l = [self.dispatch(child) for child in node.children[1:]] + result.list = l + return result + visit_arrayliteral = listop + visit_objectliteral = listop + + def visit_propertynameandvalue(self, node): + op = node.children[0] + result = self.get_instance( + op, operations.PropertyInit) + result.left = self.dispatch(node.children[0]) + result.right = self.dispatch(node.children[1]) + return result + + def visit_IDENTIFIERNAME(self, node): + print node.additional_info + result = self.get_instance(node, operations.Identifier) + result.name = node.additional_info + result.initializer = operations.astundef #XXX this is uneded now + print result + return result Modified: pypy/dist/pypy/lang/js/jsgrammar.txt ============================================================================== --- pypy/dist/pypy/lang/js/jsgrammar.txt (original) +++ pypy/dist/pypy/lang/js/jsgrammar.txt Mon May 14 22:37:49 2007 @@ -163,18 +163,18 @@ | "\(" "\)" ; -arrayliteral : ["["] elementlist ["]"] - | ["["] elementlist ["," "]"] +arrayliteral : "[" >elementlist< ["]"] + | "[" >elementlist< ["," "]"] ; -elementlist : assignmentexpression ("," assignmentexpression)* +elementlist : assignmentexpression ([","] assignmentexpression)* ; elision : ","+ ; -objectliteral : ["{"] ["}"] - | ["{"] >propertynameandvaluelist< ["}"] +objectliteral : "{" ["}"] + | "{" >propertynameandvaluelist< ["}"] ; propertynameandvaluelist : propertynameandvalue ([","] propertynameandvalue)* Modified: pypy/dist/pypy/lang/js/jsobj.py ============================================================================== --- pypy/dist/pypy/lang/js/jsobj.py (original) +++ pypy/dist/pypy/lang/js/jsobj.py Mon May 14 22:37:49 2007 @@ -483,7 +483,7 @@ return W_Reference(identifier) -def global_context(w_global): +def global_context(w_global=W_Object()): ctx = ExecutionContext() ctx.push_object(w_global) ctx.this = w_global Modified: pypy/dist/pypy/lang/js/operations.py ============================================================================== --- pypy/dist/pypy/lang/js/operations.py (original) +++ pypy/dist/pypy/lang/js/operations.py Mon May 14 22:37:49 2007 @@ -17,7 +17,7 @@ is used to match the AST operation to the efective execution node. """ opcode = None - def __init__(self, t=None, type_='', value='', lineno=0, start=0, end=0): + def __init__(self, t=None, value='', lineno=0, start=0, end=0): """ Not to be overriden by subclasses, this method thakes the basic node information from the AST needed for tracing and debuging. if you want @@ -25,7 +25,6 @@ call. """ if t is None: - self.type = type_ self.value = value self.lineno = lineno self.start = start @@ -306,7 +305,7 @@ return ""%(str(self.name), str(self.initializer)) def eval(self, ctx): - if self.initializer is not astundef: + if not isinstance(self.initializer, Undefined): ref = ctx.resolve_identifier(self.name) ref.PutValue(self.initializer.eval(ctx).GetValue(), ctx) return ctx.resolve_identifier(self.name) Modified: pypy/dist/pypy/lang/js/test/test_new_parser.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_new_parser.py (original) +++ pypy/dist/pypy/lang/js/test/test_new_parser.py Mon May 14 22:37:49 2007 @@ -4,7 +4,8 @@ from pypy.rlib.parsing.ebnfparse import parse_ebnf, make_parse_function from pypy.rlib.parsing.parsing import ParseError, Rule from pypy.rlib.parsing.tree import RPythonVisitor -from pypy.lang.js.newparser import EvalTreeBuilder +from pypy.lang.js.jsobj import global_context, ThrowException +from pypy.lang.js.astbuilder import ASTBuilder from pypy import conftest import sys @@ -277,36 +278,41 @@ self.parse('function x () {;}') self.parse('function z (a,b,c,d,e) {;}') - + class TestToEvalTree(BaseGrammarTest): def setup_class(cls): cls.parse = parse_func('expression') - def to_etree(self, s): - return EvalTreeBuilder().dispatch(self.parse(s)) + def to_ast(self, s): + return ASTBuilder().dispatch(self.parse(s)) def eval_expr(self, s): - etree = self.to_etree(s) - return etree.eval(None) + ast = self.to_ast(s) + return ast.eval(global_context()) - def test_primary(self): - etree = self.to_etree('(6)') - w_num = etree.eval(None) + def test_primaryexpression(self): + w_num = self.eval_expr('(6)') assert w_num.ToNumber() == 6 - etree = self.to_etree('((((6))))') - w_num = etree.eval(None) + w_num = self.eval_expr('((((6))))') assert w_num.ToNumber() == 6 - etree = self.to_etree('1 - 1 - 1') - w_num = etree.eval(None) + w_array = self.eval_expr('[1,2,3]') + assert w_array.ToString() == '1,2,3' + w_identifier = self.eval_expr('x') + py.test.raises(ThrowException, w_identifier.GetValue) + w_object = self.eval_expr('{x:1}') + assert w_object.ToString() == '[object Object]' + assert w_object.Get('x').ToNumber() == 1 + + def test_expression(self): + w_num = self.eval_expr('1 - 1 - 1') assert w_num.ToNumber() == -1 - etree = self.to_etree('-(6 * (6 * 6)) + 6 - 6') - w_num = etree.eval(None) + w_num = self.eval_expr('-(6 * (6 * 6)) + 6 - 6') assert w_num.ToNumber() == -216 - etree = self.to_etree('++5') - w_num = etree.eval(None) + w_num = self.eval_expr('++5') assert w_num.ToNumber() == 6 w_num = self.eval_expr('--5') assert w_num.ToNumber() == 4 w_str = self.eval_expr('"hello "+\'world\'') assert w_str.ToString() == 'hello world' + \ No newline at end of file From cfbolz at codespeak.net Mon May 14 23:18:12 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 14 May 2007 23:18:12 +0200 (CEST) Subject: [pypy-svn] r43387 - pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin Message-ID: <20070514211812.A8CCA807A@code0.codespeak.net> Author: cfbolz Date: Mon May 14 23:18:12 2007 New Revision: 43387 Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/register.py Log: fix annotation problem. "how did that ever work?" Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/register.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/register.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/register.py Mon May 14 23:18:12 2007 @@ -29,6 +29,10 @@ code.append(" if we_are_translated():") code.append(" raise error.UncatchableError('%s does not work in translated version')" % (name, )) subargs = ["engine"] + if len(unwrap_spec): + code.append(" assert isinstance(query, term.Term)") + else: + code.append(" assert isinstance(query, term.Atom)") for i, spec in enumerate(unwrap_spec): varname = "var%s" % (i, ) subargs.append(varname) From santagada at codespeak.net Mon May 14 23:19:09 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Mon, 14 May 2007 23:19:09 +0200 (CEST) Subject: [pypy-svn] r43388 - pypy/dist/pypy/lang/js Message-ID: <20070514211909.6C071807A@code0.codespeak.net> Author: santagada Date: Mon May 14 23:19:09 2007 New Revision: 43388 Modified: pypy/dist/pypy/lang/js/astbuilder.py Log: getting source_pos from one of the children of a nonterminal symbol Modified: pypy/dist/pypy/lang/js/astbuilder.py ============================================================================== --- pypy/dist/pypy/lang/js/astbuilder.py (original) +++ pypy/dist/pypy/lang/js/astbuilder.py Mon May 14 23:19:09 2007 @@ -20,17 +20,24 @@ '{': operations.ObjectInit, } - def get_instance(self, symbol, cls): - if isinstance(symbol, Symbol): - source_pos = symbol.token.source_pos - # XXX some of the source positions are not perfect - return cls(None, - symbol.additional_info, - source_pos.lineno, - source_pos.columnno, - source_pos.columnno + len(symbol.additional_info)) + def get_instance(self, node, cls): + if isinstance(node, Symbol): + source_pos = node.token.source_pos else: - return cls(None, '', -1, -1, -1) + source_pos = None + for child in symbol.children: + if isinstance(child, Symbol): + source_pos = child.token.source_pos + break + if posnode is None: + return cls(None, '', -1, -1, -1) + + # XXX some of the source positions are not perfect + return cls(None, + node.additional_info, + source_pos.lineno, + source_pos.columnno, + source_pos.columnno + len(node.additional_info)) def visit_DECIMALLITERAL(self, node): result = self.get_instance(node, operations.Number) From santagada at codespeak.net Mon May 14 23:21:24 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Mon, 14 May 2007 23:21:24 +0200 (CEST) Subject: [pypy-svn] r43389 - pypy/dist/pypy/lang/js Message-ID: <20070514212124.9403F807A@code0.codespeak.net> Author: santagada Date: Mon May 14 23:21:24 2007 New Revision: 43389 Modified: pypy/dist/pypy/lang/js/astbuilder.py Log: typo Modified: pypy/dist/pypy/lang/js/astbuilder.py ============================================================================== --- pypy/dist/pypy/lang/js/astbuilder.py (original) +++ pypy/dist/pypy/lang/js/astbuilder.py Mon May 14 23:21:24 2007 @@ -29,7 +29,7 @@ if isinstance(child, Symbol): source_pos = child.token.source_pos break - if posnode is None: + if source_pos is None: return cls(None, '', -1, -1, -1) # XXX some of the source positions are not perfect From santagada at codespeak.net Mon May 14 23:31:32 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Mon, 14 May 2007 23:31:32 +0200 (CEST) Subject: [pypy-svn] r43390 - pypy/dist/pypy/lang/js Message-ID: <20070514213132.076C2807A@code0.codespeak.net> Author: santagada Date: Mon May 14 23:31:32 2007 New Revision: 43390 Modified: pypy/dist/pypy/lang/js/astbuilder.py Log: now it woks and tests pass. Modified: pypy/dist/pypy/lang/js/astbuilder.py ============================================================================== --- pypy/dist/pypy/lang/js/astbuilder.py (original) +++ pypy/dist/pypy/lang/js/astbuilder.py Mon May 14 23:31:32 2007 @@ -21,23 +21,26 @@ } def get_instance(self, node, cls): + value = '' + source_pos = None if isinstance(node, Symbol): source_pos = node.token.source_pos + value = node.additional_info else: - source_pos = None - for child in symbol.children: + for child in node.children: if isinstance(child, Symbol): source_pos = child.token.source_pos + value = child.additional_info break - if source_pos is None: - return cls(None, '', -1, -1, -1) + if source_pos is None: + return cls(None, '', -1, -1, -1) # XXX some of the source positions are not perfect return cls(None, - node.additional_info, + value, source_pos.lineno, source_pos.columnno, - source_pos.columnno + len(node.additional_info)) + source_pos.columnno + len(value)) def visit_DECIMALLITERAL(self, node): result = self.get_instance(node, operations.Number) @@ -87,15 +90,13 @@ visit_objectliteral = listop def visit_propertynameandvalue(self, node): - op = node.children[0] result = self.get_instance( - op, operations.PropertyInit) + node, operations.PropertyInit) result.left = self.dispatch(node.children[0]) result.right = self.dispatch(node.children[1]) return result def visit_IDENTIFIERNAME(self, node): - print node.additional_info result = self.get_instance(node, operations.Identifier) result.name = node.additional_info result.initializer = operations.astundef #XXX this is uneded now From cfbolz at codespeak.net Mon May 14 23:40:42 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 14 May 2007 23:40:42 +0200 (CEST) Subject: [pypy-svn] r43391 - pypy/dist/pypy/lang/js Message-ID: <20070514214042.BD14D807A@code0.codespeak.net> Author: cfbolz Date: Mon May 14 23:40:42 2007 New Revision: 43391 Modified: pypy/dist/pypy/lang/js/astbuilder.py Log: better heuristic Modified: pypy/dist/pypy/lang/js/astbuilder.py ============================================================================== --- pypy/dist/pypy/lang/js/astbuilder.py (original) +++ pypy/dist/pypy/lang/js/astbuilder.py Mon May 14 23:40:42 2007 @@ -27,13 +27,15 @@ source_pos = node.token.source_pos value = node.additional_info else: - for child in node.children: - if isinstance(child, Symbol): - source_pos = child.token.source_pos - value = child.additional_info + curr = node.children[0] + while not isinstance(curr, Symbol): + if len(curr.children): + curr = curr.children[0] + else: + source_pos = None break - if source_pos is None: - return cls(None, '', -1, -1, -1) + else: + source_pos = curr.token.source_pos # XXX some of the source positions are not perfect return cls(None, From santagada at codespeak.net Mon May 14 23:50:33 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Mon, 14 May 2007 23:50:33 +0200 (CEST) Subject: [pypy-svn] r43392 - pypy/dist/pypy/lang/js/test Message-ID: <20070514215033.CC10D807A@code0.codespeak.net> Author: santagada Date: Mon May 14 23:50:33 2007 New Revision: 43392 Modified: pypy/dist/pypy/lang/js/test/test_new_parser.py Log: new get_instance tests, good idea from cfbolz Modified: pypy/dist/pypy/lang/js/test/test_new_parser.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_new_parser.py (original) +++ pypy/dist/pypy/lang/js/test/test_new_parser.py Mon May 14 23:50:33 2007 @@ -290,6 +290,20 @@ ast = self.to_ast(s) return ast.eval(global_context()) + def test_get_instance(self): + from pypy.lang.js import operations + from pypy.rlib.parsing.tree import Symbol + astb = ASTBuilder() + t = self.parse('6') + assert isinstance(t, Symbol) + op = astb.get_instance(t, operations.Node) + assert op.value == '6' + assert op.lineno == 0 + t = self.parse('[1,]') + assert not isinstance(t, Symbol) + op = astb.get_instance(t, operations.Node) + assert op.lineno == 0 + def test_primaryexpression(self): w_num = self.eval_expr('(6)') assert w_num.ToNumber() == 6 From santagada at codespeak.net Tue May 15 03:27:37 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Tue, 15 May 2007 03:27:37 +0200 (CEST) Subject: [pypy-svn] r43396 - in pypy/dist/pypy/lang/js: . test Message-ID: <20070515012737.B0D5B8075@code0.codespeak.net> Author: santagada Date: Tue May 15 03:27:36 2007 New Revision: 43396 Added: pypy/dist/pypy/lang/js/jsparser.py pypy/dist/pypy/lang/js/test/test_parser.py - copied, changed from r43392, pypy/dist/pypy/lang/js/test/test_new_parser.py Modified: pypy/dist/pypy/lang/js/astbuilder.py pypy/dist/pypy/lang/js/interpreter.py pypy/dist/pypy/lang/js/test/test_interp.py pypy/dist/pypy/lang/js/test/test_new_parser.py Log: making the transition to the new parser... it is not ready for production yet, but I will not be mantaining the old one... so it is better to work on things as they are going to be (file names, apis, etc) Modified: pypy/dist/pypy/lang/js/astbuilder.py ============================================================================== --- pypy/dist/pypy/lang/js/astbuilder.py (original) +++ pypy/dist/pypy/lang/js/astbuilder.py Tue May 15 03:27:36 2007 @@ -46,7 +46,7 @@ def visit_DECIMALLITERAL(self, node): result = self.get_instance(node, operations.Number) - result.num = float(node.additional_info) + result.num = float(result.value) return result def string(self,node): Modified: pypy/dist/pypy/lang/js/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/js/interpreter.py (original) +++ pypy/dist/pypy/lang/js/interpreter.py Tue May 15 03:27:36 2007 @@ -1,6 +1,7 @@ import math -from pypy.lang.js.jsparser import parse, parse_bytecode +from pypy.lang.js.jsparser import parse +from pypy.lang.js.astbuilder import ASTBuilder from pypy.lang.js.operations import * from pypy.rlib.objectmodel import we_are_translated @@ -9,12 +10,9 @@ print x def load_source(script_source): + astb = ASTBuilder() temp_tree = parse(script_source) - return from_tree(temp_tree) - -def load_bytecode(bytecode): - temp_tree = parse_bytecode(bytecode) - return from_tree(temp_tree) + return astb.dispatch(temp_tree) import cPickle as pickle import os.path Added: pypy/dist/pypy/lang/js/jsparser.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/js/jsparser.py Tue May 15 03:27:36 2007 @@ -0,0 +1,17 @@ +from pypy.rlib.parsing.ebnfparse import parse_ebnf, make_parse_function +from pypy.rlib.parsing.parsing import ParseError, Rule +import py + +GFILE = py.magic.autopath().dirpath().join("jsgrammar.txt") + +try: + t = GFILE.read() + regexs, rules, ToAST = parse_ebnf(t) +except ParseError,e: + print e.nice_error_message(filename=str(GFILE),source=t) + raise + +parsef = make_parse_function(regexs, rules, eof=True) + +def parse(code): + parsef(code).visit(ToAST()) Modified: pypy/dist/pypy/lang/js/test/test_interp.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_interp.py (original) +++ pypy/dist/pypy/lang/js/test/test_interp.py Tue May 15 03:27:36 2007 @@ -15,6 +15,7 @@ py.test.skip("js binary not found") js_is_on_path() +py.test.skip("making the transition to the new parser") class TestInterp(object): def test_simple(self): Modified: pypy/dist/pypy/lang/js/test/test_new_parser.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_new_parser.py (original) +++ pypy/dist/pypy/lang/js/test/test_new_parser.py Tue May 15 03:27:36 2007 @@ -279,7 +279,7 @@ self.parse('function z (a,b,c,d,e) {;}') -class TestToEvalTree(BaseGrammarTest): +class TestToAST(BaseGrammarTest): def setup_class(cls): cls.parse = parse_func('expression') From santagada at codespeak.net Tue May 15 03:34:15 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Tue, 15 May 2007 03:34:15 +0200 (CEST) Subject: [pypy-svn] r43397 - in pypy/dist/pypy/lang/js: . test test/ecma Message-ID: <20070515013415.70C468075@code0.codespeak.net> Author: santagada Date: Tue May 15 03:34:15 2007 New Revision: 43397 Removed: pypy/dist/pypy/lang/js/test/test_new_parser.py Modified: pypy/dist/pypy/lang/js/operations.py pypy/dist/pypy/lang/js/test/ecma/conftest.py Log: some mistakes Modified: pypy/dist/pypy/lang/js/operations.py ============================================================================== --- pypy/dist/pypy/lang/js/operations.py (original) +++ pypy/dist/pypy/lang/js/operations.py Tue May 15 03:34:15 2007 @@ -6,7 +6,6 @@ #XXX * imports are bad from pypy.lang.js.jsobj import * -from pypy.lang.js.jsparser import JsSyntaxError from pypy.rlib.parsing.ebnfparse import Symbol, Nonterminal from pypy.rlib.rarithmetic import r_uint, intmask from constants import unescapedict, SLASH Modified: pypy/dist/pypy/lang/js/test/ecma/conftest.py ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/conftest.py (original) +++ pypy/dist/pypy/lang/js/test/ecma/conftest.py Tue May 15 03:34:15 2007 @@ -1,7 +1,7 @@ import py from pypy.lang.js.interpreter import * from pypy.lang.js.jsobj import W_Array, JsBaseExcept -from pypy.lang.js.jsparser import JsSyntaxError +from pypy.rlib.parsing.parsing import ParseError from py.__.test.outcome import Failed, ExceptionFailure import pypy.lang.js as js @@ -57,7 +57,7 @@ t = load_file(str(self.fspath)) try: t.execute(self.interp.global_context) - except JsSyntaxError: + except ParseError: raise Failed(msg="Syntax Error",excinfo=py.code.ExceptionInfo()) except JsBaseExcept: raise Failed(msg="Javascript Error", excinfo=py.code.ExceptionInfo()) From santagada at codespeak.net Tue May 15 04:45:00 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Tue, 15 May 2007 04:45:00 +0200 (CEST) Subject: [pypy-svn] r43398 - in pypy/dist/pypy/lang/js: . test Message-ID: <20070515024500.753828075@code0.codespeak.net> Author: santagada Date: Tue May 15 04:44:59 2007 New Revision: 43398 Modified: pypy/dist/pypy/lang/js/astbuilder.py pypy/dist/pypy/lang/js/jsgrammar.txt pypy/dist/pypy/lang/js/operations.py pypy/dist/pypy/lang/js/test/test_parser.py Log: some preliminar suport for javascript programs. Modified: pypy/dist/pypy/lang/js/astbuilder.py ============================================================================== --- pypy/dist/pypy/lang/js/astbuilder.py (original) +++ pypy/dist/pypy/lang/js/astbuilder.py Tue May 15 04:44:59 2007 @@ -104,3 +104,19 @@ result.initializer = operations.astundef #XXX this is uneded now print result return result + + def visit_program(self, node): + result = self.get_instance(node, operations.Program) + result.body = self.dispatch(node.children[0]) + return result + + def visit_sourceelements(self, node): + result = self.get_instance(node, operations.Script) + result.var_decl = None #XXX TODO + result.func_decl = None #XXX TODO + result.nodes = [self.dispatch(child) for child in node.children] + return result + + def visit_expressionstatement(self, node): + return self.dispatch(node.children[0]) + \ No newline at end of file Modified: pypy/dist/pypy/lang/js/jsgrammar.txt ============================================================================== --- pypy/dist/pypy/lang/js/jsgrammar.txt (original) +++ pypy/dist/pypy/lang/js/jsgrammar.txt Tue May 15 04:44:59 2007 @@ -188,17 +188,17 @@ | ; -functiondeclaration : ["function"] identifier ["("] formalparameterlist? [")"] ["{"] functionbody ["}"] +functiondeclaration : ["function"] identifier ["("] formalparameterlist? [")"] ["{"] >functionbody< ["}"] ; -functionexpression : ["function"] identifier? ["("] formalparameterlist? [")"] ["{"] functionbody ["}"] +functionexpression : ["function"] identifier? ["("] formalparameterlist? [")"] ["{"] >functionbody< ["}"] ; formalparameterlist : identifier [","] >formalparameterlist< | identifier ; -functionbody : +functionbody : sourceelements ; begmemberexpression : Modified: pypy/dist/pypy/lang/js/operations.py ============================================================================== --- pypy/dist/pypy/lang/js/operations.py (original) +++ pypy/dist/pypy/lang/js/operations.py Tue May 15 04:44:59 2007 @@ -868,6 +868,11 @@ print "exception in line: %s, on: %s"%(node.lineno, node.value) raise +class Program(Statement): + + def execute(self, ctx): + return self.body.execute(self, ctx) + class Semicolon(Statement): opcode = 'SEMICOLON' Modified: pypy/dist/pypy/lang/js/test/test_parser.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_parser.py (original) +++ pypy/dist/pypy/lang/js/test/test_parser.py Tue May 15 04:44:59 2007 @@ -279,7 +279,7 @@ self.parse('function z (a,b,c,d,e) {;}') -class TestToAST(BaseGrammarTest): +class TestToASTExpr(BaseGrammarTest): def setup_class(cls): cls.parse = parse_func('expression') @@ -328,5 +328,16 @@ assert w_num.ToNumber() == 4 w_str = self.eval_expr('"hello "+\'world\'') assert w_str.ToString() == 'hello world' - - \ No newline at end of file + + +class TestToAST(BaseGrammarTest): + def setup_class(cls): + cls.parse = parse_func() + + def to_ast(self, s): + ASTBuilder().dispatch(self.parse(s)) + + def test_simple(self): + self.to_ast("1;") + #self.to_ast("print(1+1);") + From cfbolz at codespeak.net Tue May 15 15:26:06 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 15 May 2007 15:26:06 +0200 (CEST) Subject: [pypy-svn] r43405 - in pypy/branch/prolog-jit-experiments/pypy: jit/timeshifter/test lang/prolog/interpreter lang/prolog/interpreter/test Message-ID: <20070515132606.371228082@code0.codespeak.net> Author: cfbolz Date: Tue May 15 15:26:05 2007 New Revision: 43405 Added: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/test/test_jit.py (contents, props changed) Modified: pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_portal.py pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_timeshift.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/portal.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/term.py Log: make a test out of the prolog jit so that I can look at the graphs. some more changes around. Modified: pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_portal.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_portal.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_portal.py Tue May 15 15:26:05 2007 @@ -65,8 +65,8 @@ self.hrtyper.specialize(origportalgraph=origportalgraph, view = conftest.option.view and self.small) - #if conftest.option.view and self.small: - # t.view() + if conftest.option.view and self.small: + t.viewcg() self.postprocess_timeshifting() self.readportalgraph = self.hrtyper.readportalgraph Modified: pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_timeshift.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_timeshift.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_timeshift.py Tue May 15 15:26:05 2007 @@ -61,6 +61,8 @@ backend_optimizations(t) if portal is None: portal = func + if hasattr(policy, "seetranslator"): + policy.seetranslator(t) graph1 = graphof(t, portal) # build hint annotator types hannotator = HintAnnotator(base_translator=t, policy=policy) @@ -69,7 +71,7 @@ for v in graph1.getargs()]) hannotator.simplify() if conftest.option.view: - hannotator.translator.view() + hannotator.translator.viewcg() return hs, hannotator, rtyper class TimeshiftingTests(object): Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py Tue May 15 15:26:05 2007 @@ -5,7 +5,7 @@ from pypy.lang.prolog.interpreter import error from pypy.rlib.objectmodel import hint, specialize -DEBUG = True +DEBUG = False class Continuation(object): def call(self, engine): @@ -32,6 +32,11 @@ self.needed_vars = 0 self.last_branch = 0 + def reset(self): + self.vars = [None] * len(self.vars) + self.trail = [] + self.last_branch = 0 + def clear(self, length): l = max(START_NUMBER_OF_VARS, length) self.vars = [None] * l @@ -274,8 +279,10 @@ debug_print("worked", rule, query, self.heap.vars[:self.heap.needed_vars]) if nextcall is not None: self.call(nextcall, continuation) + return "no" else: continuation.call(self) + return "yes" def continue_after_cut(self, continuation, lsc=None): Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/portal.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/portal.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/portal.py Tue May 15 15:26:05 2007 @@ -7,14 +7,20 @@ forbidden_modules = {'pypy.lang.prolog.interpreter.parser': True, } +good_modules = {'pypy.lang.prolog.builtin.control': True, + 'pypy.lang.prolog.builtin.register': True + } + PORTAL = engine.Engine.try_rule class PyrologHintAnnotatorPolicy(HintAnnotatorPolicy): novirtualcontainer = True oopspec = True - def __init__(self, timeshift_graphs): - self.timeshift_graphs = timeshift_graphs + def seetranslator(self, t): + portal = getattr(PORTAL, 'im_func', PORTAL) + portal_graph = graphof(t, portal) + self.timeshift_graphs = timeshift_graphs(t, portal_graph) def look_inside_graph(self, graph): if graph in self.timeshift_graphs: @@ -28,9 +34,11 @@ mod = func.__module__ or '?' if mod in forbidden_modules: return False -# if "pypy.lang.prolog.builtin" in mod: -# return True - return False + if mod in good_modules: + return True + if mod.startswith("pypy.lang.prolog"): + return False + return True def jitme(func): func._look_inside_me_ = True @@ -83,7 +91,7 @@ targetgraphs.keys(),)) -def timeshift_graphs(t, portal_graph, log): +def timeshift_graphs(t, portal_graph): import pypy result_graphs = {} @@ -100,8 +108,6 @@ targetgraphs[_graph(tofunc)] = True graphs = graphs_on_the_path_to(t, _graph(fromfunc), targetgraphs) for graph in graphs: - if graph not in result_graphs: - log('including graph %s' % (graph,)) result_graphs[graph] = True def seepath(*path): @@ -114,9 +120,6 @@ extra = "" if look != True: extra = " substituted with %s" % look - log('including graph %s%s' % (graph, extra)) - else: - log('excluding graph %s' % (graph,)) result_graphs[graph] = look for cls in [term.Var, term.Term, term.Number, term.Float, term.Atom]: @@ -125,16 +128,17 @@ seegraph(cls.copy_and_unify) for cls in [term.Term, term.Number, term.Atom]: seegraph(cls.copy_and_basic_unify) + seegraph(cls.copy_and_basic_unify) seegraph(pypy.lang.prolog.interpreter.engine.Heap.newvar) seegraph(pypy.lang.prolog.interpreter.engine.Engine.try_rule) seegraph(pypy.lang.prolog.interpreter.term.Rule.clone_and_unify_head) + seegraph(pypy.lang.prolog.interpreter.engine.Engine.call) return result_graphs def get_portal(drv): t = drv.translator portal = getattr(PORTAL, 'im_func', PORTAL) - portal_graph = graphof(t, portal) - policy = PyrologHintAnnotatorPolicy(timeshift_graphs(t, portal_graph, - drv.log)) + policy = PyrologHintAnnotatorPolicy() + policy.seetranslator(t) return portal, policy Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/term.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/term.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/term.py Tue May 15 15:26:05 2007 @@ -4,7 +4,7 @@ from pypy.lang.prolog.interpreter.error import UnificationFailed, UncatchableError from pypy.rlib.objectmodel import hint, specialize -DEBUG = True +DEBUG = False TAGBITS = 3 CURR_TAG = 1 @@ -20,6 +20,7 @@ class PrologObject(object): __slots__ = () + _immutable_ = True def __init__(self): raise NotImplementedError("abstract base class") @@ -121,6 +122,7 @@ def copy_and_unify(self, other, heap, memo): hint(self, concrete=True) + self = hint(self, deepfreeze=True) try: seen_value = memo[self.index] except KeyError: @@ -162,6 +164,7 @@ self.index == other.index) class NonVar(PrologObject): + __slots__ = () def dereference(self, heap): return self @@ -197,6 +200,7 @@ class Callable(NonVar): + __slots__ = ("name", "signature") name = "" signature = "" @@ -394,14 +398,13 @@ arg = self.args[i].copy(heap, memo) newargs.append(arg) i += 1 - return Term(self.name, newargs) + return Term(self.name, newargs, self.signature) def copy_and_basic_unify(self, other, heap, memo): hint(self, concrete=True) self = hint(self, deepfreeze=True) if (isinstance(other, Term) and - self.name == other.name and - len(self.args) == len(other.args)): + self.signature == other.signature): newargs = [None] * len(self.args) i = 0 while i < len(self.args): @@ -409,7 +412,7 @@ arg = self.args[i].copy_and_unify(other.args[i], heap, memo) newargs[i] = arg i += 1 - return Term(self.name, newargs) + return Term(self.name, newargs, self.signature) else: raise UnificationFailed @@ -456,6 +459,7 @@ class Rule(object): + _immutable_ = True unify_hash = [] def __init__(self, head, body): from pypy.lang.prolog.interpreter import helper @@ -491,9 +495,9 @@ def clone_and_unify_head(self, heap, head): memo = {} - if isinstance(head, Term): - h2 = self.head - assert isinstance(h2, Term) + h2 = self.head + if isinstance(h2, Term): + assert isinstance(head, Term) i = 0 while i < len(h2.args): i = hint(i, concrete=True) Added: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/test/test_jit.py ============================================================================== --- (empty file) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/test/test_jit.py Tue May 15 15:26:05 2007 @@ -0,0 +1,47 @@ +import py +from pypy.jit.timeshifter.test.test_portal import PortalTest +from pypy.lang.prolog.interpreter import portal +from pypy.lang.prolog.interpreter import engine, term +from pypy.lang.prolog.interpreter.parsing import parse_query_term, get_engine + +POLICY = portal.PyrologHintAnnotatorPolicy() + + +class TestPortal(PortalTest): + small = False + + def test_simple(self): + e = get_engine(""" + f(x, y). + f(a(X), b(b(Y))) :- f(X, Y). + """) + X = e.heap.newvar() + Y = e.heap.newvar() + larger = term.Term( + "f", [term.Term("a", [X]), term.Term("b", [term.Term("b", [Y])])]) + + def main(n): + e.heap.reset() + if n == 0: + e.call(term.Term("f", [X, Y])) + return isinstance(X.dereference(e.heap), term.Atom) + if n == 1: + e.call(larger) + return isinstance(X.dereference(e.heap), term.Atom) + else: + return False + + res = main(0) + assert res == True + res = main(1) + assert res == True + + + res = self.timeshift_from_portal(main, engine.Engine.try_rule.im_func, + [1], policy=POLICY) + assert res == True + + res = self.timeshift_from_portal(main, engine.Engine.try_rule.im_func, + [0], policy=POLICY) + assert res == True + From cfbolz at codespeak.net Tue May 15 16:17:03 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 15 May 2007 16:17:03 +0200 (CEST) Subject: [pypy-svn] r43409 - in pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter: . test Message-ID: <20070515141703.71C7D8082@code0.codespeak.net> Author: cfbolz Date: Tue May 15 16:17:03 2007 New Revision: 43409 Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/parsing.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/term.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/test/test_unification.py Log: some changes to keep the memo dict virtual Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py Tue May 15 16:17:03 2007 @@ -90,7 +90,7 @@ return self.needed_vars def newvar(self): - result = Var(self.maxvar()) + result = Var.newvar(self.maxvar()) self.extend(1) return result Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/parsing.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/parsing.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/parsing.py Tue May 15 16:17:03 2007 @@ -357,10 +357,10 @@ if varname == "_": pos = self.freevar self.freevar += 1 - return Var(pos) + return Var.newvar(pos) if varname in self.var_to_pos: return self.var_to_pos[varname] - res = Var(self.freevar) + res = Var.newvar(self.freevar) self.freevar += 1 self.var_to_pos[varname] = res return res Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/term.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/term.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/term.py Tue May 15 16:17:03 2007 @@ -78,6 +78,7 @@ STANDARD_ORDER = 0 __slots__ = ('index', ) + cache = {} def __init__(self, index): self.index = index @@ -115,18 +116,18 @@ def copy(self, heap, memo): hint(self, concrete=True) try: - return memo[self.index] + return memo[self] except KeyError: - newvar = memo[self.index] = heap.newvar() + newvar = memo[self] = heap.newvar() return newvar def copy_and_unify(self, other, heap, memo): hint(self, concrete=True) self = hint(self, deepfreeze=True) try: - seen_value = memo[self.index] + seen_value = memo[self] except KeyError: - memo[self.index] = other + memo[self] = other return other else: seen_value.unify(other, heap) @@ -138,10 +139,10 @@ def clone_compress_vars(self, vars_new_indexes, offset): if self.index in vars_new_indexes: - return Var(vars_new_indexes[self.index]) + return Var.newvar(vars_new_indexes[self.index]) index = len(vars_new_indexes) + offset vars_new_indexes[self.index] = index - return Var(index) + return Var.newvar(index) def get_unify_hash(self): return 0 @@ -163,6 +164,15 @@ return (self.__class__ == other.__class__ and self.index == other.index) + def newvar(index): + result = Var.cache.get(index, None) + if result is not None: + return result + Var.cache[index] = result = Var(index) + return result + newvar = staticmethod(newvar) + + class NonVar(PrologObject): __slots__ = () Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/test/test_unification.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/test/test_unification.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/test/test_unification.py Tue May 15 16:17:03 2007 @@ -11,13 +11,13 @@ py.test.raises(UnificationFailed, "a.unify(Atom.newatom('xxx'), None)") def test_var(): - b = Var(0) + b = Var.newvar(0) heap = Heap() heap.clear(1) b.unify(Atom.newatom("hallo"), heap) assert b.getvalue(heap).name == "hallo" - a = Var(0) - b = Var(1) + a = Var.newvar(0) + b = Var.newvar(1) heap.clear(2) a.unify(b, heap) a.unify(Atom.newatom("hallo"), heap) @@ -25,7 +25,7 @@ assert b.getvalue(heap).name == "hallo" def test_unify_var(): - b = Var(0) + b = Var.newvar(0) heap = Heap() heap.clear(1) b.unify(b, heap) @@ -33,15 +33,15 @@ py.test.raises(UnificationFailed, b.unify, Atom.newatom("bye"), heap) def test_recursive(): - b = Var(0) + b = Var.newvar(0) heap = Heap() heap.clear(1) b.unify(Term("hallo", [b]), heap) def test_term(): - X = Var(0) - Y = Var(1) + X = Var.newvar(0) + Y = Var.newvar(1) t1 = Term("f", [Atom.newatom("hallo"), X]) t2 = Term("f", [Y, Atom.newatom("HALLO")]) heap = Heap() @@ -61,12 +61,12 @@ def test_run(): e = Engine() e.add_rule(Term("f", [Atom.newatom("a"), Atom.newatom("b")])) - e.add_rule(Term("f", [Var(0), Var(0)])) - e.add_rule(Term(":-", [Term("f", [Var(0), Var(1)]), - Term("f", [Var(1), Var(0)])])) + e.add_rule(Term("f", [Var.newvar(0), Var.newvar(0)])) + e.add_rule(Term(":-", [Term("f", [Var.newvar(0), Var.newvar(1)]), + Term("f", [Var.newvar(1), Var.newvar(0)])])) X = e.heap.newvar() - assert e.run(Term("f", [Atom.newatom("b"), X])) is None + e.run(Term("f", [Atom.newatom("b"), X])) assert X.dereference(e.heap).name == "b" - assert e.run(Term("f", [Atom.newatom("b"), Atom.newatom("a")])) is None + e.run(Term("f", [Atom.newatom("b"), Atom.newatom("a")])) From arigo at codespeak.net Tue May 15 17:21:11 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 15 May 2007 17:21:11 +0200 (CEST) Subject: [pypy-svn] r43414 - pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter Message-ID: <20070515152111.CBB518084@code0.codespeak.net> Author: arigo Date: Tue May 15 17:21:11 2007 New Revision: 43414 Modified: pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/rtimeshift.py Log: Trying to fix the problem. Not tested yet. Modified: pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/rtimeshift.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/rtimeshift.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/rtimeshift.py Tue May 15 17:21:11 2007 @@ -453,6 +453,7 @@ if pending.returnbox is not None: pending.frame.local_boxes.insert(0, getreturnbox(pending)) pending.next = None + start_writing(pending, jitstate_chain) return pending n = 0 From fijal at codespeak.net Wed May 16 13:54:51 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 16 May 2007 13:54:51 +0200 (CEST) Subject: [pypy-svn] r43428 - pypy/extradoc/talk/ep2007 Message-ID: <20070516115451.409E0808F@code0.codespeak.net> Author: fijal Date: Wed May 16 13:54:49 2007 New Revision: 43428 Added: pypy/extradoc/talk/ep2007/ pypy/extradoc/talk/ep2007/planning.txt Log: Some kind of planning file for ep2007 Added: pypy/extradoc/talk/ep2007/planning.txt ============================================================================== --- (empty file) +++ pypy/extradoc/talk/ep2007/planning.txt Wed May 16 13:54:49 2007 @@ -0,0 +1,8 @@ +I'm listing here some ideas gathered around on IRC: + +1. "A usual pypy talk" - architecture, motivation etc. +2. An RPython talk: .NET, JVM, JS, extcompiler, possible uses, caveats, + probably some architecture +3. Interpreters talk: taint, thunk, persistance, distribution, probably + some stackless (?), interpreter for .NET (hpk, fijal) +4. Some agile talk (?) (Bea) From fijal at codespeak.net Wed May 16 14:36:08 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 16 May 2007 14:36:08 +0200 (CEST) Subject: [pypy-svn] r43430 - pypy/extradoc/talk/ep2007 Message-ID: <20070516123608.527DA8091@code0.codespeak.net> Author: fijal Date: Wed May 16 14:36:07 2007 New Revision: 43430 Added: pypy/extradoc/talk/ep2007/abstract_interpreter.txt Log: Add a draft of an abstract for interpreter talk Added: pypy/extradoc/talk/ep2007/abstract_interpreter.txt ============================================================================== --- (empty file) +++ pypy/extradoc/talk/ep2007/abstract_interpreter.txt Wed May 16 14:36:07 2007 @@ -0,0 +1,17 @@ +Why use PyPy interpreter? +===================================== + +XXX Better title? + +Speakers: Holger Krekel & Maciek Fijalkowski + +Abstract +--------- + +We would like to present features showcasing PyPy's flexible architecture, +in other words, what makes the pypy interpreter unique. Showcases will +include PyPy's approach to distribution, security (ruby-like tainting), +lazy computations and others. Talk will be need-based, showing various +usecases of our flexible interpreter in use. + +XXX finish \ No newline at end of file From cfbolz at codespeak.net Wed May 16 15:19:06 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 16 May 2007 15:19:06 +0200 (CEST) Subject: [pypy-svn] r43432 - pypy/dist/pypy/jit/timeshifter/test Message-ID: <20070516131906.953348090@code0.codespeak.net> Author: cfbolz Date: Wed May 16 15:19:05 2007 New Revision: 43432 Modified: pypy/dist/pypy/jit/timeshifter/test/test_portal.py Log: add a (skipped because failing) test for a recursive portal call. Modified: pypy/dist/pypy/jit/timeshifter/test/test_portal.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/test/test_portal.py (original) +++ pypy/dist/pypy/jit/timeshifter/test/test_portal.py Wed May 16 15:19:05 2007 @@ -557,4 +557,18 @@ res = self.timeshift_from_portal(f, g, [42], policy=P_NOVIRTUAL) - + def test_recursive_portal_call(self): + py.test.skip("recursive portal calls don't work right now") + def indirection(green, red): + return portal((green + red) % 100, red + 1) + def portal(green, red): + green = abs(green) + red = abs(red) + hint(green, concrete=True) + if green > 42: + return 0 + if red > 42: + return 1 + return indirection(green, red) + res = self.timeshift_from_portal(portal, portal, [41, 1], policy=P_NOVIRTUAL) + assert res == 0 From cfbolz at codespeak.net Wed May 16 15:28:07 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 16 May 2007 15:28:07 +0200 (CEST) Subject: [pypy-svn] r43433 - pypy/dist/pypy/jit/hintannotator/test Message-ID: <20070516132807.62BB08090@code0.codespeak.net> Author: cfbolz Date: Wed May 16 15:28:07 2007 New Revision: 43433 Modified: pypy/dist/pypy/jit/hintannotator/test/test_annotator.py Log: a test where I am not sure the hintannotator gives the right result. Modified: pypy/dist/pypy/jit/hintannotator/test/test_annotator.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/test/test_annotator.py (original) +++ pypy/dist/pypy/jit/hintannotator/test/test_annotator.py Wed May 16 15:28:07 2007 @@ -919,3 +919,30 @@ hs = hannotate(f, [], policy=P_NOVIRTUAL) assert not hs.is_green() + +def test_strange_green_result_after_red_switch(): + py.test.skip("is this right?") + class LinkedRules(object): + _immutable_ = True + def __init__(self, data, next=None): + self.data = data + self.next = next + + def find_applicable_rule(self, query): + # self is green, query isn't + # should the result really be green? + while self: + data = self.data + hint(data, concrete=True) + j = 0 + if self.data == query: + return self + self = self.next + return None + + chain = LinkedRules(1, LinkedRules(2, LinkedRules(0))) + def f(x): + rulechain = chain.find_applicable_rule(x) + return rulechain + hs = hannotate(f, [int], policy=P_OOPSPEC_NOVIRTUAL) + assert isinstance(hs, SomeLLAbstractVariable) From fijal at codespeak.net Wed May 16 21:15:05 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 16 May 2007 21:15:05 +0200 (CEST) Subject: [pypy-svn] r43437 - pypy/branch/kill-ctypes Message-ID: <20070516191505.B58948091@code0.codespeak.net> Author: fijal Date: Wed May 16 21:15:04 2007 New Revision: 43437 Added: pypy/branch/kill-ctypes/ - copied from r43436, pypy/dist/ Log: A branch which tries to create pypy without ctypes dependency, by approaching external functions via ll objects From antocuni at codespeak.net Thu May 17 14:43:18 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 17 May 2007 14:43:18 +0200 (CEST) Subject: [pypy-svn] r43452 - pypy/extradoc/talk/ep2007 Message-ID: <20070517124318.30F7C8091@code0.codespeak.net> Author: antocuni Date: Thu May 17 14:43:16 2007 New Revision: 43452 Added: pypy/extradoc/talk/ep2007/rpython.txt (contents, props changed) Log: draft for an RPython-centric talk Added: pypy/extradoc/talk/ep2007/rpython.txt ============================================================================== --- (empty file) +++ pypy/extradoc/talk/ep2007/rpython.txt Thu May 17 14:43:16 2007 @@ -0,0 +1,18 @@ +RPython: need for speed +======================= + +XXX: suggestions for title are welcome :-) + +Speakers: Antonio Cuni & Maciek Fijalkowski + +Abstract +-------- + +Restricted Python (RPython) is a subset of Python designed to be +compiled into efficient executables, up to 300 times faster than +CPython. Although it was born as an implementation detail of PyPy, +experience showed that it can be useful by itself. This talk will +present some typical RPython's use cases, including: how to write an +extension module for CPython, how to produce .NET +libraries/executables as fast as C#, how to write AJAX applications +without a single line of Javascript. From niko at codespeak.net Thu May 17 15:04:27 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Thu, 17 May 2007 15:04:27 +0200 (CEST) Subject: [pypy-svn] r43453 - pypy/dist/pypy/translator/jvm Message-ID: <20070517130427.075658078@code0.codespeak.net> Author: niko Date: Thu May 17 15:04:26 2007 New Revision: 43453 Modified: pypy/dist/pypy/translator/jvm/database.py pypy/dist/pypy/translator/jvm/generator.py pypy/dist/pypy/translator/jvm/node.py pypy/dist/pypy/translator/jvm/typesystem.py Log: interim check-in: treat the Object RPython class very specially, so that it is generated as an interface with two implements, one of which descends from Object and one of which descends from Throwable. RPython exceptions use the latter, which should enable them to be directly thrown by "throw" statements in the JVM. Note: exception throwing, for now, still uses the old mechanism. Hat tip for the idea to pedronis. Modified: pypy/dist/pypy/translator/jvm/database.py ============================================================================== --- pypy/dist/pypy/translator/jvm/database.py (original) +++ pypy/dist/pypy/translator/jvm/database.py Thu May 17 15:04:26 2007 @@ -10,9 +10,11 @@ from pypy.translator.jvm import node, methods from pypy.translator.jvm.option import getoption import pypy.translator.jvm.generator as jvmgen +from pypy.translator.jvm.generator import Method, Property, Field import pypy.translator.jvm.constant as jvmconst from pypy.translator.jvm.typesystem import \ - jStringBuilder, jInt, jVoid, jString, jChar, jPyPyConst, jObject + jStringBuilder, jInt, jVoid, jString, jChar, jPyPyConst, jObject, \ + jThrowable from pypy.translator.jvm.builtin import JvmBuiltInType from pypy.translator.oosupport.database import Database as OODatabase @@ -41,6 +43,11 @@ self._constants = {} # flowmodel.Variable --> jvmgen.Const + # Special fields for the Object class, see _translate_Object + self._object_interf = None + self._object_impl = None + self._object_exc_impl = None + # _________________________________________________________________ # Java String vs Byte Array # @@ -74,6 +81,9 @@ """ Returns list of files we need to run jasmin on """ return self._jasmin_files + def is_Object(self, OOTYPE): + return isinstance(OOTYPE, ootype.Instance) and OOTYPE._name == "Object" + # _________________________________________________________________ # Node Creation # @@ -151,7 +161,82 @@ self.pending_node(clsobj) return clsobj - + + def _translate_Object(self, OBJ): + """ + We handle the class 'Object' quite specially: we translate it + into an interface with two implementations. One + implementation serves as the root of most objects, and the + other as the root for all exceptions. + """ + assert self.is_Object(OBJ) + assert OBJ._superclass == ootype.ROOT + + # Have we already translated Object? + if self._object_interf: return self._object_interf + + # Create the interface and two implementations: + def gen_name(): return self._pkg(self._uniq(OBJ._name)) + internm, implnm, exc_implnm = gen_name(), gen_name(), gen_name() + self._object_interf = node.Interface(internm) + self._object_impl = node.Class(implnm, supercls=jObject) + self._object_exc_impl = node.Class(exc_implnm, supercls=jThrowable) + self._object_impl.add_interface(self._object_interf) + self._object_exc_impl.add_interface(self._object_interf) + + # Translate the fields into properties on the interface, + # and into actual fields on the implementations. + for fieldnm, (FIELDOOTY, fielddef) in OBJ._fields.iteritems(): + if FIELDOOTY is ootype.Void: continue + fieldty = self.lltype_to_cts(FIELDOOTY) + + # Currently use hacky convention of _jvm_FieldName for the name + methodnm = "_jvm_"+fieldnm + + def getter_method_obj(node): + return Method.v(node, methodnm+"_g", [], fieldty) + def putter_method_obj(node): + return Method.v(node, methodnm+"_p", [fieldty], jVoid) + + # Add get/put methods to the interface: + prop = Property( + fieldnm, + getter_method_obj(self._object_interf), + putter_method_obj(self._object_interf), + OOTYPE=FIELDOOTY) + self._object_interf.add_property(prop) + + # Generate implementations: + def generate_impl(clsobj): + clsnm = clsobj.name + fieldobj = Field(clsnm, fieldnm, fieldty, False, FIELDOOTY) + clsobj.add_field(fieldobj, fielddef) + clsobj.add_method(node.GetterFunction( + self, clsobj, getter_method_obj(clsobj), fieldobj)) + clsobj.add_method(node.PutterFunction( + self, clsobj, putter_method_obj(clsobj), fieldobj)) + generate_impl(self._object_impl) + generate_impl(self._object_exc_impl) + + # Ensure that we generate all three classes. + self.pending_node(self._object_interf) + self.pending_node(self._object_impl) + self.pending_node(self._object_exc_impl) + + def _translate_superclass_of(self, OOSUB): + """ + Invoked to translate OOSUB's super class. Normally just invokes + pending_class, but we treat "Object" differently so that we can + make all exceptions descend from Throwable. + """ + OOSUPER = OOSUB._superclass + if not self.is_Object(OOSUPER): + return self.pending_class(OOSUPER) + self._translate_Object(OOSUPER) # ensure this has been done + if OOSUB._name == "exceptions.Exception": + return self._object_exc_impl + return self._object_impl + def _translate_instance(self, OOTYPE): assert isinstance(OOTYPE, ootype.Instance) assert OOTYPE is not ootype.ROOT @@ -167,7 +252,7 @@ # Resolve super class assert OOTYPE._superclass - supercls = self.pending_class(OOTYPE._superclass) + supercls = self._translate_superclass_of(OOTYPE) clsobj.set_super_class(supercls) # TODO --- mangle field and method names? Must be @@ -175,7 +260,6 @@ # classes? # Add fields: - # Add fields: self._translate_class_fields(clsobj, OOTYPE) # Add methods: @@ -368,7 +452,7 @@ ootype.Char: jvmtype.jChar, # byte would be sufficient, but harder ootype.UniChar: jvmtype.jChar, ootype.Class: jvmtype.jClass, - ootype.ROOT: jvmtype.jObject, # count this as a scalar... + ootype.ROOT: jvmtype.jObject # treat like a scalar } # Dictionary for non-scalar types; in this case, if we see the key, we @@ -399,6 +483,8 @@ # Handle non-built-in-types: if isinstance(OOT, ootype.Instance): + if self.is_Object(OOT): + return self._translate_Object(OOT) return self._translate_instance(OOT) if isinstance(OOT, ootype.Record): return self._translate_record(OOT) Modified: pypy/dist/pypy/translator/jvm/generator.py ============================================================================== --- pypy/dist/pypy/translator/jvm/generator.py (original) +++ pypy/dist/pypy/translator/jvm/generator.py Thu May 17 15:04:26 2007 @@ -151,11 +151,12 @@ class ArrayOpcodeFamily(OpcodeFamily): """ Opcode family specialized for array access instr """ def for_type(self, argtype): - if argtype == 'J': return self._o("l") # Integers - if argtype == 'D': return self._o("d") # Doubles - if argtype == 'C': return self._o("c") # Characters - if argtype == 'B': return self._o("b") # Bytes - if argtype == 'Z': return self._o("b") # Boolean (access as bytes) + desc = argtype.descriptor + if desc == 'J': return self._o("l") # Integers + if desc == 'D': return self._o("d") # Doubles + if desc == 'C': return self._o("c") # Characters + if desc == 'B': return self._o("b") # Bytes + if desc == 'Z': return self._o("b") # Boolean (access as bytes) return OpcodeFamily.for_type(self, argtype) # Define the opcodes for IFNE, IFEQ, IFLT, IF_ICMPLT, etc. The IFxx @@ -295,6 +296,8 @@ not the this ptr 'rettype' - JvmType for return type """ + assert argtypes is not None + assert rettype is not None classnm = classty.name if isinstance(classty, jvmtype.JvmInterfaceType): opc = INVOKEINTERFACE @@ -341,7 +344,7 @@ # A weird, inexplicable quirk of Jasmin syntax is that it requires # the number of arguments after an invokeinterface call: if self.opcode == INVOKEINTERFACE: - res += " %d" % (len(self.argument_types),) + res += " %d" % (len(self.argument_types)+1,) return res OBJHASHCODE = Method.v(jObject, 'hashCode', (), jInt) @@ -418,6 +421,29 @@ self.field_name, self.jtype.descriptor) +class Property(object): + """ + An object which acts like a Field, but when a value is loaded or + stored it actually invokes accessor methods. + """ + def __init__(self, field_name, get_method, put_method, OOTYPE=None): + self.get_method = get_method + self.put_method = put_method + self.field_name = field_name + self.OOTYPE = OOTYPE + + # Synthesize the Field attributes from the get_method/put_method: + self.class_name = get_method.class_name + assert put_method.class_name == self.class_name + self.jtype = get_method.return_type + self.is_static = get_method.is_static + def load(self, gen): + self.get_method.invoke(gen) + def store(self, gen): + self.put_method.invoke(gen) + # jasmin_syntax is not needed, since this object itself never appears + # as an argument an Opcode + SYSTEMOUT = Field('java.lang.System', 'out', jPrintStream, True) SYSTEMERR = Field('java.lang.System', 'err', jPrintStream, True) DOUBLENAN = Field('java.lang.Double', 'NaN', jDouble, True) @@ -488,7 +514,8 @@ # If the name does not begin with '_', it will be called from # outside the generator. - def begin_class(self, classty, superclsty, abstract=False): + def begin_class(self, classty, superclsty, + abstract=False, interface=False): """ Begins a class declaration. Overall flow of class declaration looks like: @@ -507,7 +534,7 @@ """ assert not self.curclass self.curclass = ClassState(classty, superclsty) - self._begin_class(abstract) + self._begin_class(abstract, interface) def end_class(self): self._end_class() @@ -519,7 +546,7 @@ begin_class() has not been called, returns None. """ return self.curclass.class_type - def _begin_class(self, abstract): + def _begin_class(self, abstract, interface): """ Main implementation of begin_class """ raise NotImplementedError @@ -557,6 +584,19 @@ self.return_val(jVoid) self.end_function() + def begin_j_function(self, cls_obj, method_obj, abstract=False): + """ + A convenience function that invokes begin_function() with the + appropriate arguments to define a method on class 'cls_obj' that + could be invoked with 'method_obj'. + """ + return self.begin_function(method_obj.method_name, + [], + [cls_obj]+method_obj.argument_types, + method_obj.return_type, + static=method_obj.is_static(), + abstract=abstract) + def begin_function(self, funcname, argvars, argtypes, rettype, static=False, abstract=False): """ @@ -635,13 +675,13 @@ # XXX --- support byte arrays here? Would be trickier! self._instr(LDC, res) - def load_jvm_var(self, vartype, varidx): + def load_jvm_var(self, jvartype, varidx): """ Loads from jvm slot #varidx, which is expected to hold a value of type vartype """ assert varidx < self.curfunc.next_offset - opc = LOAD.for_type(vartype) - self.add_comment(" load_jvm_jar: vartype=%s varidx=%s" % ( - repr(vartype), repr(varidx))) + opc = LOAD.for_type(jvartype) + self.add_comment(" load_jvm_jar: jvartype=%s varidx=%s" % ( + repr(jvartype), repr(varidx))) self._instr(opc, varidx) def store_jvm_var(self, vartype, varidx): @@ -858,7 +898,7 @@ if isinstance(instr, Method): return instr.invoke(self) - if isinstance(instr, Field): + if isinstance(instr, Field) or isinstance(instr, Property): return instr.load(self) raise Exception("Unknown object in call to emit(): "+repr(instr)) @@ -1169,9 +1209,14 @@ JVMGenerator.__init__(self, db) self.outdir = outdir - def _begin_class(self, abstract): + def _begin_class(self, abstract, interface): """ - classnm --- full Java name of the class (i.e., "java.lang.String") + Invoked by begin_class. It is expected that self.curclass will + be set when this method is invoked. + + abstract: True if the class to generate is abstract + + interface: True if the 'class' to generate is an interface """ iclassnm = self.current_type().descriptor.int_class_name() @@ -1186,10 +1231,15 @@ self.curclass.file = open(jfile, 'w') self.db.add_jasmin_file(jfile) + # Determine the "declaration string" + if interface: decl_str = "interface" + else: decl_str = "class" + # Write the JasminXT header fields = ["public"] if abstract: fields.append('abstract') - self.curclass.out(".class %s %s\n" % (" ".join(fields), iclassnm)) + self.curclass.out(".%s %s %s\n" % ( + decl_str, " ".join(fields), iclassnm)) self.curclass.out(".super %s\n" % isuper) def _end_class(self): Modified: pypy/dist/pypy/translator/jvm/node.py ============================================================================== --- pypy/dist/pypy/translator/jvm/node.py (original) +++ pypy/dist/pypy/translator/jvm/node.py Thu May 17 15:04:26 2007 @@ -21,7 +21,8 @@ ootype, rclass from pypy.translator.jvm.typesystem import \ JvmClassType, jString, jStringArray, jVoid, jThrowable, jInt, jPyPyMain, \ - jObject, JvmType, jStringBuilder, jPyPyInterlink, jCallbackInterfaces + jObject, JvmType, jStringBuilder, jPyPyInterlink, jCallbackInterfaces, \ + JvmInterfaceType from pypy.translator.jvm.opcodes import \ opcodes from pypy.translator.jvm.option import \ @@ -172,7 +173,47 @@ """ Returns a jvmgen.Method object that would allow this function to be invoked. """ raise NotImplementedError + +class GetterFunction(Function): + def __init__(self, db, cls_obj, method_obj, field_obj): + self.db = db + self.name = method_obj.method_name + self.cls_obj = cls_obj + self.method_obj = method_obj + self.field_obj = field_obj + + def method(self): + return self.method_obj + + def render(self, gen): + gen.begin_function( + self.method_obj.method_name, [], + [self.cls_obj], self.field_obj.jtype) + gen.load_this_ptr() + self.field_obj.load(gen) + gen.return_val(self.field_obj.jtype) + gen.end_function() +class PutterFunction(Function): + def __init__(self, db, cls_obj, method_obj, field_obj): + self.db = db + self.cls_obj = cls_obj + self.method_obj = method_obj + self.field_obj = field_obj + + def method(self): + return self.method_obj + + def render(self, gen): + gen.begin_function( + self.method_obj.method_name, [], + [self.cls_obj, self.field_obj.jtype], jVoid) + gen.load_this_ptr() + gen.load_function_argument(1) + self.field_obj.store(gen) + gen.return_val(jVoid) + gen.end_function() + class GraphFunction(OOFunction, Function): """ Represents a function that is generated from a graph. """ @@ -266,12 +307,7 @@ if isinstance(link.last_exception, flowmodel.Variable): self.ilasm.emit(jvmgen.DUP) self.ilasm.store(link.last_exc_value) - fld = jvmgen.Field( - self.db.lltype_to_cts(rclass.OBJECT).name, - 'meta', - self.db.lltype_to_cts(rclass.CLASSTYPE), - False, - rclass.OBJECT) + fld = self.db.lltype_to_cts(rclass.OBJECT).lookup_field('meta') self.ilasm.emit(fld) self.ilasm.store(link.last_exception) else: @@ -381,14 +417,15 @@ self.java_argument_types[1:], self.java_return_type) def lookup_field(self, fieldnm): - """ Given a field name, returns a jvmgen.Field object """ raise KeyError(fieldnm) # no fields + def lookup_method(self, methodnm): """ Given the method name, returns a jvmgen.Method object """ assert isinstance(self.java_return_type, JvmType) if methodnm == 'invoke': return self.invoke_method_obj raise KeyError(methodnm) # only one method + def render(self, gen): assert isinstance(self.java_return_type, JvmType) @@ -535,12 +572,54 @@ gen.end_function() gen.end_class() +class Interface(Node, JvmInterfaceType): + """ + Represents an interface to be generated. The only class that we + currently generate into an interface is ootype.ROOT. + """ + def __init__(self, name): + JvmClassType.__init__(self, name) + self.super_class = jObject + self.rendered = False + self.properties = {} + self.methods = {} + + def lookup_field(self, fieldnm): + # Right now, we don't need inheritance between interfaces. + return self.properties[fieldnm] + + def lookup_method(self, methodnm): + # Right now, we don't need inheritance between interfaces. + return self.methods[methodnm] + + def add_property(self, prop): + self.properties[prop.field_name] = prop + + def add_method(self, method): + self.methods[method.name] = method + + def render(self, gen): + self.rendered = True + gen.begin_class(self, self.super_class, interface=True) + + def emit_method(method): + gen.begin_j_function(self, method, abstract=True) + gen.end_function() + + for method in self.methods.values(): + emit_method(method) + for prop in self.properties.values(): + emit_method(prop.get_method) + emit_method(prop.put_method) + + gen.end_class() + class Class(Node, JvmClassType): """ Represents a class to be emitted. Note that currently, classes are emitted all in one shot, not piecemeal. """ - def __init__(self, name, supercls=None, initialize_fields=True): + def __init__(self, name, supercls=None): """ 'name' should be a fully qualified Java class name like "java.lang.String", supercls is a Class object @@ -577,7 +656,6 @@ self.interfaces.append(inter) def lookup_field(self, fieldnm): - """ Given a field name, returns a jvmgen.Field object """ if fieldnm in self.fields: return self.fields[fieldnm][0] return self.super_class.lookup_field(fieldnm) @@ -629,9 +707,7 @@ method.render(gen) for method in self.abstract_methods.values(): - gen.begin_function( - method.method_name, None, method.argument_types, - method.return_type, abstract=True) + gen.begin_j_function(self, method, abstract=True) gen.end_function() gen.end_class() Modified: pypy/dist/pypy/translator/jvm/typesystem.py ============================================================================== --- pypy/dist/pypy/translator/jvm/typesystem.py (original) +++ pypy/dist/pypy/translator/jvm/typesystem.py Thu May 17 15:04:26 2007 @@ -115,9 +115,10 @@ self.name = None # public, string like "java.lang.Object" # (None for scalars and arrays) def lookup_field(self, fieldnm): - """ Returns a jvm.generator.Field object representing the field - with the given name, or raises KeyError if that field does not - exist on this type. """ + """ If the class has a field named 'fieldnm', returns a + jvmgen.Field or jvmgen.Property object that represents it and can + be used with the interpreter to load/store it. If no such field + exists, or this is not a class, then raises KeyError. """ raise NotImplementedException def lookup_method(self, methodnm): """ Returns a jvm.generator.Method object representing the method From santagada at codespeak.net Thu May 17 17:45:04 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Thu, 17 May 2007 17:45:04 +0200 (CEST) Subject: [pypy-svn] r43458 - in pypy/dist/pypy/lang/js: . test Message-ID: <20070517154504.05CE2807B@code0.codespeak.net> Author: santagada Date: Thu May 17 17:45:04 2007 New Revision: 43458 Modified: pypy/dist/pypy/lang/js/astbuilder.py pypy/dist/pypy/lang/js/operations.py pypy/dist/pypy/lang/js/test/test_parser.py Log: removed from_tree and made a position object (maybe I can just grab the one on the symbol) Modified: pypy/dist/pypy/lang/js/astbuilder.py ============================================================================== --- pypy/dist/pypy/lang/js/astbuilder.py (original) +++ pypy/dist/pypy/lang/js/astbuilder.py Thu May 17 17:45:04 2007 @@ -20,40 +20,37 @@ '{': operations.ObjectInit, } - def get_instance(self, node, cls): + def get_pos(self, node): value = '' source_pos = None if isinstance(node, Symbol): - source_pos = node.token.source_pos value = node.additional_info + source_pos = node.token.source_pos else: curr = node.children[0] while not isinstance(curr, Symbol): if len(curr.children): curr = curr.children[0] else: - source_pos = None break else: + value = curr.additional_info source_pos = curr.token.source_pos # XXX some of the source positions are not perfect - return cls(None, - value, + return operations.Position( source_pos.lineno, source_pos.columnno, source_pos.columnno + len(value)) def visit_DECIMALLITERAL(self, node): - result = self.get_instance(node, operations.Number) - result.num = float(result.value) - return result + pos = self.get_pos(node) + number = operations.Number(pos, float(node.additional_info)) + return number def string(self,node): - print node.additional_info - result = self.get_instance(node, operations.String) - result.strval = node.additional_info[1:-1] #XXX should do unquoting - return result + pos = self.get_pos(node) + return operations.String(pos, node.additional_info) visit_DOUBLESTRING = string visit_SINGLESTRING = string @@ -62,11 +59,9 @@ left = self.dispatch(node.children[0]) for i in range((len(node.children) - 1) // 2): op = node.children[i * 2 + 1] - result = self.get_instance( - op, self.BINOP_TO_CLS[op.additional_info]) + pos = self.get_pos(op) right = self.dispatch(node.children[i * 2 + 2]) - result.left = left - result.right = right + result = self.BINOP_TO_CLS[op.additional_info](pos, left, right) left = result return left visit_additiveexpression = binaryop @@ -74,48 +69,41 @@ def visit_unaryexpression(self, node): op = node.children[0] - result = self.get_instance( - op, self.UNOP_TO_CLS[op.additional_info]) + pos = self.get_pos(op) child = self.dispatch(node.children[1]) - result.expr = child - result.postfix = False - return result + return self.UNOP_TO_CLS[op.additional_info](pos, child) def listop(self, node): op = node.children[0] - result = self.get_instance( - op, self.LISTOP_TO_CLS[op.additional_info]) + pos = self.get_pos(op) l = [self.dispatch(child) for child in node.children[1:]] - result.list = l - return result + return self.LISTOP_TO_CLS[op.additional_info](pos, l) visit_arrayliteral = listop visit_objectliteral = listop def visit_propertynameandvalue(self, node): - result = self.get_instance( - node, operations.PropertyInit) - result.left = self.dispatch(node.children[0]) - result.right = self.dispatch(node.children[1]) - return result + pos = self.get_pos(node) + left = self.dispatch(node.children[0]) + right = self.dispatch(node.children[1]) + return operations.PropertyInit(pos,left,right) def visit_IDENTIFIERNAME(self, node): - result = self.get_instance(node, operations.Identifier) - result.name = node.additional_info - result.initializer = operations.astundef #XXX this is uneded now - print result - return result + pos = self.get_pos(node) + name = node.additional_info + initializer = operations.astundef #XXX this is uneded now + return operations.Identifier(pos, name, initializer) def visit_program(self, node): - result = self.get_instance(node, operations.Program) - result.body = self.dispatch(node.children[0]) - return result + pos = self.get_pos(node) + body = self.dispatch(node.children[0]) + return operations.Program(pos, body) def visit_sourceelements(self, node): - result = self.get_instance(node, operations.Script) - result.var_decl = None #XXX TODO - result.func_decl = None #XXX TODO - result.nodes = [self.dispatch(child) for child in node.children] - return result + pos = self.get_pos(node) + var_decl = None #XXX TODO + func_decl = None #XXX TODO + nodes = [self.dispatch(child) for child in node.children] + return operations.Script(pos, var_decl, func_decl, nodes) def visit_expressionstatement(self, node): return self.dispatch(node.children[0]) Modified: pypy/dist/pypy/lang/js/operations.py ============================================================================== --- pypy/dist/pypy/lang/js/operations.py (original) +++ pypy/dist/pypy/lang/js/operations.py Thu May 17 17:45:04 2007 @@ -10,40 +10,18 @@ from pypy.rlib.rarithmetic import r_uint, intmask from constants import unescapedict, SLASH +class Position(object): + def __init__(self, lineno=-1, start=-1, end=-1): + self.lineno = lineno + self.start = start + self.end = end + + class Node(object): """ - Node is the base class for all the other nodes, the opcode parameter - is used to match the AST operation to the efective execution node. + Node is the base class for all the other nodes. """ - opcode = None - def __init__(self, t=None, value='', lineno=0, start=0, end=0): - """ - Not to be overriden by subclasses, this method thakes the basic node - information from the AST needed for tracing and debuging. if you want - to override behavior for the creation of a node do it on the from_tree - call. - """ - if t is None: - self.value = value - self.lineno = lineno - self.start = start - self.end = end - else: - self.type = get_string(t, 'type') - self.value = get_string(t, 'value') - self.lineno = int(get_string(t, 'lineno')) - - try: - self.start = int(get_string(t, 'start')) - except ValueError, e: - self.start = 0 - try: - self.end = int(get_string(t, 'end')) - except Exception, e: - self.end = 0 - self.from_tree(t) - - def from_tree(self, t): + def __init__(self, pos): """ Initializes the content from the AST specific for each node type """ @@ -71,7 +49,8 @@ return ""%(self.opcode, self.value) class Statement(Node): - pass + def __init__(self, pos): + self.pos = pos class Expression(Statement): def eval(self, ctx): @@ -81,18 +60,21 @@ return self.eval(ctx) class ListOp(Expression): - def from_tree(self, t): - self.list = get_objects(t) + def __init__(self, pos, nodelist): + self.pos = pos + self.nodelist = nodelist class UnaryOp(Expression): - def from_tree(self, t): - self.expr = get_obj(t, '0') - self.postfix = bool(get_string(t, 'postfix')) + def __init__(self, pos, expr, postfix=False): + self.pos = pos + self.expr = expr + self.postfix = postfix class BinaryOp(Expression): - def from_tree(self, t): - self.left = get_obj(t,'0') - self.right = get_obj(t, '1') + def __init__(self, pos, left, right): + self.pos = pos + self.left = left + self.right = right class BinaryComparisonOp(BinaryOp): def eval(self, ctx): @@ -112,25 +94,17 @@ def decision(self, ctx, op1, op2): raise NotImplementedError -class BinaryLogicOp(BinaryOp): - pass - class PropertyInit(BinaryOp): - opcode = 'PROPERTY_INIT' + pass -class Array(ListOp): - opcode = 'ARRAY_INIT' - +class Array(ListOp): def eval(self, ctx): array = W_Array() - for i in range(len(self.list)): - array.Put(str(i), self.list[i].eval(ctx).GetValue()) + for i in range(len(self.nodelist)): + array.Put(str(i), self.nodelist[i].eval(ctx).GetValue()) return array - -class Assign(BinaryOp): - opcode = 'ASSIGN' - +class Assign(BinaryOp): def eval(self, ctx): v1 = self.left.eval(ctx) v3 = self.right.eval(ctx).GetValue() @@ -159,9 +133,7 @@ return val class Block(Statement): - opcode = 'BLOCK' - - def from_tree(self, t): + def __init__(self, pos, t): self.nodes = get_objects(t) def execute(self, ctx): @@ -203,7 +175,7 @@ return W_Number(op1^op2) class Unconditional(Statement): - def from_tree(self, t): + def __init__(self, pos, t): pieces = get_string(t, 'target').split(',') self.targtype = pieces[0] self.targlineno = pieces[1] @@ -256,7 +228,7 @@ class Conditional(Expression): opcode = 'CONDITIONAL' - def from_tree(self, t): + def __init__(self, pos, t): self.logicalexpr = get_obj(t, '0') self.trueop = get_obj(t, '1') self.falseop = get_obj(t, '2') @@ -278,7 +250,7 @@ class Function(Expression): opcode = 'FUNCTION' - def from_tree(self, t): + def __init__(self, pos, t): self.name = get_string(t, 'name') self.body = get_obj(t, 'body') params = get_string(t, 'params') @@ -296,9 +268,9 @@ class Identifier(Expression): opcode = 'IDENTIFIER' - def from_tree(self, t): - self.name = get_string(t,'value') - self.initializer = get_obj(t, 'initializer') + def __init__(self, pos, name, initializer): + self.name = name + self.initializer = initializer def __str__(self): return ""%(str(self.name), str(self.initializer)) @@ -318,7 +290,7 @@ class If(Statement): opcode = 'IF' - def from_tree(self, t): + def __init__(self, pos, t): self.condition = get_obj(t, 'condition') self.thenPart = get_obj(t, 'thenPart') self.elsePart = get_obj(t, 'elsePart') @@ -362,8 +334,7 @@ else: return -1 -class Or(BinaryLogicOp): - opcode = 'OR' +class Or(BinaryOp): def eval(self, ctx): s2 = self.left.eval(ctx).GetValue() @@ -372,7 +343,7 @@ s4 = self.right.eval(ctx).GetValue() return s4 -class And(BinaryLogicOp): +class And(BinaryOp): opcode = 'AND' def eval(self, ctx): @@ -612,7 +583,7 @@ thing = self.expr.eval(ctx) val = thing.GetValue() x = val.ToNumber() - resl = Plus().mathop(ctx, W_Number(x), W_Number(1)) + resl = Plus.mathop(ctx, W_Number(x), W_Number(1)) thing.PutValue(resl, ctx) if self.postfix: return val @@ -630,7 +601,7 @@ thing = self.expr.eval(ctx) val = thing.GetValue() x = val.ToNumber() - resl = Plus().mathop(ctx, W_Number(x), W_Number(-1)) + resl = Plus.mathop(ctx, W_Number(x), W_Number(-1)) thing.PutValue(resl, ctx) if self.postfix: return val @@ -667,9 +638,8 @@ return result class Plus(BinaryNumberOp): - opcode = 'PLUS' - - def mathop(self, ctx, nleft, nright): + @staticmethod + def mathop(ctx, nleft, nright): if isinstance(nleft, W_String) or isinstance(nright, W_String): sleft = nleft.ToString() sright = nright.ToString() @@ -680,16 +650,12 @@ return W_Number(fleft + fright) class Mult(BinaryNumberOp): - opcode = 'MUL' - def mathop(self, ctx, nleft, nright): fleft = nleft.ToNumber() fright = nright.ToNumber() return W_Number(fleft * fright) class Mod(BinaryNumberOp): - opcode = 'MOD' - def mathop(self, ctx, nleft, nright): fleft = nleft.ToInt32() fright = nright.ToInt32() @@ -716,7 +682,7 @@ class Null(Expression): opcode = 'NULL' - def from_tree(self, t): + def __init__(self, pos, t): pass def eval(self, ctx): @@ -749,11 +715,10 @@ args = self.right.eval(ctx).get_args() return x.Construct(ctx=ctx, args=args) -class Number(Expression): - opcode = 'NUMBER' - - def from_tree(self, t): - self.num = float(get_string(t, 'value')) +class Number(Expression): + def __init__(self, pos, num): + assert isinstance(num, float) + self.num = num def eval(self, ctx): return W_Number(self.num) @@ -761,8 +726,8 @@ class String(Expression): opcode = 'STRING' - def from_tree(self, t): - self.strval = self.string_unquote(get_string(t, 'value')) + def __init__(self, pos, strval): + self.strval = self.string_unquote(strval) def eval(self, ctx): return W_String(self.strval) @@ -778,17 +743,13 @@ #removing the begining quotes (" or \') if string.startswith('"'): singlequote = False - if stop < 0: - print stop - raise JsSyntaxError() - internalstring = string[1:stop] else: singlequote = True - stop -= 1 - if stop < 0: - print stop - raise JsSyntaxError() - internalstring = string[2:stop] + + if stop < 0: + print stop + raise JsSyntaxError() + internalstring = string[1:stop] for c in internalstring: if last == SLASH: @@ -807,8 +768,8 @@ def eval(self, ctx): w_obj = W_Object() - for prop in self.list: - name = prop.left.value + for prop in self.nodelist: + name = prop.left.name w_expr = prop.right.eval(ctx).GetValue() w_obj.Put(name, w_expr) return w_obj @@ -823,28 +784,10 @@ """ Script nodes are found on each function declaration and in global code """ - opcode = 'SCRIPT' - - def from_tree(self, t): - f = get_tree_item(t, 'funDecls') - if f.symbol == "dict": - func_decl = [from_tree(f),] - elif f.symbol == "list": - func_decl = [from_tree(x) for x in f.children] - else: - func_decl = [] - - v = get_tree_item(t, 'varDecls') - if v.symbol == "dict": - var_decl = [from_tree(v),] - elif v.symbol == "list": - var_decl = [from_tree(x) for x in v.children] - else: - var_decl = [] - - self.nodes = get_objects(t) + def __init__(self, pos, var_decl, func_decl, nodes): self.var_decl = var_decl self.func_decl = func_decl + self.nodes = nodes def execute(self, ctx): for var in self.var_decl: @@ -869,6 +812,9 @@ raise class Program(Statement): + def __init__(self, pos, body): + self.pos = pos + self.body = body def execute(self, ctx): return self.body.execute(self, ctx) @@ -876,7 +822,7 @@ class Semicolon(Statement): opcode = 'SEMICOLON' - def from_tree(self, t): + def __init__(self, pos, t): self.expr = get_obj(t, 'expression') def execute(self, ctx): @@ -887,7 +833,7 @@ class Return(Statement): opcode = 'RETURN' - def from_tree(self, t): + def __init__(self, pos, t): self.expr = get_obj(t, 'value') def execute(self, ctx): @@ -899,7 +845,7 @@ class Throw(Statement): opcode = 'THROW' - def from_tree(self, t): + def __init__(self, pos, t): self.exception = get_obj(t, 'exception') def execute(self, ctx): @@ -908,7 +854,7 @@ class Try(Statement): opcode = 'TRY' - def from_tree(self, t): + def __init__(self, pos, t): self.tryblock = get_obj(t, 'tryBlock') self.finallyblock = get_obj(t, 'finallyBlock') catch = get_tree_item(t, 'catchClauses') @@ -959,7 +905,7 @@ class Vars(Statement): opcode = 'VAR' - def from_tree(self, t): + def __init__(self, pos, t): self.nodes = get_objects(t) def execute(self, ctx): @@ -980,7 +926,7 @@ class With(Statement): opcode = "WITH" - def from_tree(self, t): + def __init__(self, pos, t): self.object = get_obj(t, 'object') self.body = get_obj(t, 'body') @@ -996,7 +942,7 @@ class WhileBase(Statement): - def from_tree(self, t): + def __init__(self, pos, t): self.condition = get_obj(t, 'condition') self.body = get_obj(t, 'body') @@ -1034,9 +980,7 @@ continue class ForIn(Statement): - opcode = 'FOR_IN' - - def from_tree(self, t): + def __init__(self, pos, t): self.object = get_obj(t, 'object') self.body = get_obj(t, 'body') self.iterator = get_obj(t, 'iterator') @@ -1057,9 +1001,7 @@ continue class For(Statement): - opcode = 'FOR' - - def from_tree(self, t): + def __init__(self, pos, t): self.setup = get_obj(t, 'setup') self.condition = get_obj(t, 'condition') self.update = get_obj(t, 'update') @@ -1079,7 +1021,7 @@ continue class Boolean(Expression): - def from_tree(self, t): + def __init__(self, pos, t): if self.opcode == 'TRUE': self.bool = True else: @@ -1089,71 +1031,22 @@ return W_Boolean(self.bool) class BTrue(Boolean): - opcode = 'TRUE' + pass class BFalse(Boolean): - opcode = 'FALSE' + pass class Not(UnaryOp): - opcode = 'NOT' - def eval(self, ctx): return W_Boolean(not self.expr.eval(ctx).GetValue().ToBoolean()) class UMinus(UnaryOp): - opcode = 'UNARY_MINUS' - def eval(self, ctx): return W_Number(-self.expr.eval(ctx).GetValue().ToNumber()) -class UPlus(UnaryOp): - opcode = 'UNARY_PLUS' - +class UPlus(UnaryOp): def eval(self, ctx): return W_Number(+self.expr.eval(ctx).GetValue().ToNumber()) -astundef = Undefined() -def get_obj(t, objname): - item = get_tree_item(t, objname) - if isinstance(item, Nonterminal): - return from_tree(item) - else: - return astundef - -def get_string(t, string): - simb = get_tree_item(t, string) - if isinstance(simb, Symbol): - return str(simb.additional_info) - else: - return '' - -def get_objects(t): - item = get_tree_item(t, 'length') - if item is None: - return [] - lgt = int(item.additional_info) - output = [get_obj(t, str(i)) for i in range(lgt)] - return output - -def get_tree_item(t, name): - for x in t.children: - if isinstance(x.children[0], Symbol): - if x.children[0].additional_info == name: - return x.children[1] - return None - -opcodedict = {} -for i in locals().values(): - if isinstance(i, type(Node)) and issubclass(i, Node): - if i.opcode is not None: - opcodedict[i.opcode] = i - -def from_tree(t): - if t is None: - return None - opcode = get_string(t, 'type') - if opcode in opcodedict: - return opcodedict[opcode](t) - else: - raise NotImplementedError("Dont know how to handle %s" % opcode) +astundef = Undefined(Position()) Modified: pypy/dist/pypy/lang/js/test/test_parser.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_parser.py (original) +++ pypy/dist/pypy/lang/js/test/test_parser.py Thu May 17 17:45:04 2007 @@ -290,19 +290,18 @@ ast = self.to_ast(s) return ast.eval(global_context()) - def test_get_instance(self): + def test_get_pos(self): from pypy.lang.js import operations from pypy.rlib.parsing.tree import Symbol astb = ASTBuilder() t = self.parse('6') assert isinstance(t, Symbol) - op = astb.get_instance(t, operations.Node) - assert op.value == '6' - assert op.lineno == 0 + pos = astb.get_pos(t) + assert pos.lineno == 0 t = self.parse('[1,]') assert not isinstance(t, Symbol) - op = astb.get_instance(t, operations.Node) - assert op.lineno == 0 + pos = astb.get_pos(t) + assert pos.start == 0 def test_primaryexpression(self): w_num = self.eval_expr('(6)') @@ -330,7 +329,7 @@ assert w_str.ToString() == 'hello world' -class TestToAST(BaseGrammarTest): +class TestToASTProgram(BaseGrammarTest): def setup_class(cls): cls.parse = parse_func() @@ -339,5 +338,6 @@ def test_simple(self): self.to_ast("1;") + #self.to_ast("var x=1;") #self.to_ast("print(1+1);") From fijal at codespeak.net Thu May 17 23:32:39 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 17 May 2007 23:32:39 +0200 (CEST) Subject: [pypy-svn] r43462 - pypy/extradoc/talk/ep2007 Message-ID: <20070517213239.9E1468092@code0.codespeak.net> Author: fijal Date: Thu May 17 23:32:38 2007 New Revision: 43462 Modified: pypy/extradoc/talk/ep2007/rpython.txt Log: Few rewordings, add some content Modified: pypy/extradoc/talk/ep2007/rpython.txt ============================================================================== --- pypy/extradoc/talk/ep2007/rpython.txt (original) +++ pypy/extradoc/talk/ep2007/rpython.txt Thu May 17 23:32:38 2007 @@ -2,17 +2,26 @@ ======================= XXX: suggestions for title are welcome :-) +XXX: another proposition - C considered harmful Speakers: Antonio Cuni & Maciek Fijalkowski Abstract -------- -Restricted Python (RPython) is a subset of Python designed to be -compiled into efficient executables, up to 300 times faster than -CPython. Although it was born as an implementation detail of PyPy, -experience showed that it can be useful by itself. This talk will -present some typical RPython's use cases, including: how to write an -extension module for CPython, how to produce .NET -libraries/executables as fast as C#, how to write AJAX applications -without a single line of Javascript. +Restricted Python (RPython) is a subset of a Python language designed to be +compiled into lower-level languages, suitable for direct compilation +into C, CLI, JVM or others. RPython cuts some of python dynamism (allows for +full type inference and creation of flow graphs), but doesn't require explicit +type annotations. +Results might be up to 300 times faster than the +original python program. Although it was born as an implementation +detail of the PyPy interpreter, experience showed that it can be useful +by itself. This talk will present some typical RPython's use cases, including: + * how to write a standalone executable in RPython. + * how to write an extension module for CPython. + * how to produce .NET libraries/executables as fast as C# + * how to write AJAX applications without a single line of Javascript. + (and with tests!) + * how to create bub-n-bros client in just few lines of code running on + a browser :-) From fijal at codespeak.net Fri May 18 00:04:13 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 18 May 2007 00:04:13 +0200 (CEST) Subject: [pypy-svn] r43463 - pypy/extradoc/talk/ep2007 Message-ID: <20070517220413.7E80B8091@code0.codespeak.net> Author: fijal Date: Fri May 18 00:04:11 2007 New Revision: 43463 Modified: pypy/extradoc/talk/ep2007/abstract_interpreter.txt Log: Reword abstract, make a list of items we would like to talk Modified: pypy/extradoc/talk/ep2007/abstract_interpreter.txt ============================================================================== --- pypy/extradoc/talk/ep2007/abstract_interpreter.txt (original) +++ pypy/extradoc/talk/ep2007/abstract_interpreter.txt Fri May 18 00:04:11 2007 @@ -10,8 +10,14 @@ We would like to present features showcasing PyPy's flexible architecture, in other words, what makes the pypy interpreter unique. Showcases will -include PyPy's approach to distribution, security (ruby-like tainting), -lazy computations and others. Talk will be need-based, showing various -usecases of our flexible interpreter in use. +include, but not limited to: -XXX finish \ No newline at end of file +* multiple implementations of interpreter objects, allows more flexibility + regarding object usage, without user intervention (or even knowledge) +* transparent proxy, a way to have special objects of interpreter + types with completely controlled behaviour from application level. +* distribution, strange implementation of RMI which allows of remote + access to everything including frames, file objects etc. +* object tainting, security feature similiar to ruby's tainting. +* lazy computations +* ... and many more, as talk topics might appear just-in-time From antocuni at codespeak.net Fri May 18 11:21:31 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 18 May 2007 11:21:31 +0200 (CEST) Subject: [pypy-svn] r43464 - pypy/dist/pypy/translator/cli/test Message-ID: <20070518092131.0FA568092@code0.codespeak.net> Author: antocuni Date: Fri May 18 11:21:31 2007 New Revision: 43464 Modified: pypy/dist/pypy/translator/cli/test/test_string.py Log: skip these tests Modified: pypy/dist/pypy/translator/cli/test/test_string.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_string.py (original) +++ pypy/dist/pypy/translator/cli/test/test_string.py Fri May 18 11:21:31 2007 @@ -11,6 +11,8 @@ test_unichar_eq = test_unichar_const test_unichar_ord = test_unichar_const test_unichar_hash = test_unichar_const + test_char_unichar_eq = test_unichar_const + test_char_unichar_eq_2 = test_unichar_const def test_upper(self): py.test.skip("CLI doens't support backquotes inside string literals") From fijal at codespeak.net Fri May 18 11:26:35 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 18 May 2007 11:26:35 +0200 (CEST) Subject: [pypy-svn] r43465 - pypy/dist/pypy/translator/js/test Message-ID: <20070518092635.C35BC8092@code0.codespeak.net> Author: fijal Date: Fri May 18 11:26:35 2007 New Revision: 43465 Modified: pypy/dist/pypy/translator/js/test/runtest.py Log: Improve a bit testing machinery, more tests passes on 2.5 Modified: pypy/dist/pypy/translator/js/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/js/test/runtest.py (original) +++ pypy/dist/pypy/translator/js/test/runtest.py Fri May 18 11:26:35 2007 @@ -180,7 +180,7 @@ except JSException, e: s = e.args[0] assert s.startswith('uncaught exception:') - assert re.search(str(exception), s) + assert re.search(exception.__name__, s) else: raise AssertionError("Did not raise, returned %s" % res) #except ExceptionWrapper, ex: From fijal at codespeak.net Fri May 18 11:31:27 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 18 May 2007 11:31:27 +0200 (CEST) Subject: [pypy-svn] r43466 - pypy/dist/pypy/translator/js/test Message-ID: <20070518093127.6F2198092@code0.codespeak.net> Author: fijal Date: Fri May 18 11:31:27 2007 New Revision: 43466 Modified: pypy/dist/pypy/translator/js/test/test_str.py Log: Skip those tests Modified: pypy/dist/pypy/translator/js/test/test_str.py ============================================================================== --- pypy/dist/pypy/translator/js/test/test_str.py (original) +++ pypy/dist/pypy/translator/js/test/test_str.py Fri May 18 11:31:27 2007 @@ -3,7 +3,16 @@ from pypy.translator.js.test.runtest import JsTest import pypy.translator.oosupport.test_template.string as oostring + +# ====> ../../../rpython/test/test_str.py + class TestJsString(JsTest, oostring.BaseTestString): + def test_char_unichar_eq(self): + py.test.skip("Cannot test it yet") + + def test_char_unichar_eq_2(self): + py.test.skip("Cannot test it yet") + def test_unichar_const(self): py.test.skip("Cannot test it yet") From pedronis at codespeak.net Fri May 18 11:35:32 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Fri, 18 May 2007 11:35:32 +0200 (CEST) Subject: [pypy-svn] r43467 - pypy/extradoc/talk/ep2007 Message-ID: <20070518093532.D4CAC8092@code0.codespeak.net> Author: pedronis Date: Fri May 18 11:35:32 2007 New Revision: 43467 Added: pypy/extradoc/talk/ep2007/pypy_10_and_jit.txt Log: draft of an idea of an overview: status, jit, future talk Added: pypy/extradoc/talk/ep2007/pypy_10_and_jit.txt ============================================================================== --- (empty file) +++ pypy/extradoc/talk/ep2007/pypy_10_and_jit.txt Fri May 18 11:35:32 2007 @@ -0,0 +1,13 @@ +PyPy 1.0 and Beyond +-------------------- + +Speakers: Armin Rigo, Samuele Pedroni + +Abstract +---------- + +- PyPy intro and motivation +- Recapitulation of architecture +- The JIT and the ideas that made it possible (overview with smallish examples) +- Current status and the future + From niko at codespeak.net Fri May 18 12:59:28 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Fri, 18 May 2007 12:59:28 +0200 (CEST) Subject: [pypy-svn] r43471 - pypy/dist/pypy/translator/jvm/test Message-ID: <20070518105928.0ABDA8076@code0.codespeak.net> Author: niko Date: Fri May 18 12:59:28 2007 New Revision: 43471 Modified: pypy/dist/pypy/translator/jvm/test/test_string.py Log: skip the two failing tests because they require unicode passed on the command line Modified: pypy/dist/pypy/translator/jvm/test/test_string.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_string.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_string.py Fri May 18 12:59:28 2007 @@ -11,6 +11,8 @@ test_unichar_eq = test_unichar_const test_unichar_ord = test_unichar_const test_unichar_hash = test_unichar_const + test_char_unichar_eq = test_unichar_const + test_char_unichar_eq_2 = test_unichar_const def test_upper(self): py.test.skip("eval has trouble with evaluation of null literals") From niko at codespeak.net Fri May 18 13:26:08 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Fri, 18 May 2007 13:26:08 +0200 (CEST) Subject: [pypy-svn] r43472 - in pypy/dist/pypy/translator/jvm: . src/pypy Message-ID: <20070518112608.5E1D58074@code0.codespeak.net> Author: niko Date: Fri May 18 13:26:07 2007 New Revision: 43472 Removed: pypy/dist/pypy/translator/jvm/src/pypy/ExceptionWrapper.java Modified: pypy/dist/pypy/translator/jvm/database.py pypy/dist/pypy/translator/jvm/generator.py pypy/dist/pypy/translator/jvm/genjvm.py pypy/dist/pypy/translator/jvm/node.py pypy/dist/pypy/translator/jvm/typesystem.py Log: Remove the exception wrapper system completely. We now throw the exceptions natively; any attempt to throw an RPython object that does not descend from Exception will yield a ClassCastException Modified: pypy/dist/pypy/translator/jvm/database.py ============================================================================== --- pypy/dist/pypy/translator/jvm/database.py (original) +++ pypy/dist/pypy/translator/jvm/database.py Fri May 18 13:26:07 2007 @@ -5,7 +5,7 @@ from cStringIO import StringIO from pypy.rpython.lltypesystem import lltype -from pypy.rpython.ootypesystem import ootype +from pypy.rpython.ootypesystem import ootype, rclass from pypy.translator.jvm import typesystem as jvmtype from pypy.translator.jvm import node, methods from pypy.translator.jvm.option import getoption @@ -493,6 +493,15 @@ assert False, "Untranslatable type %s!" % OOT + def exception_root_object(self): + """ + Returns a JvmType representing the version of Object that + serves as the root of all exceptions. + """ + self.lltype_to_cts(rclass.OBJECT) + assert self._object_interf + return self._object_exc_impl + # _________________________________________________________________ # Uh.... # Modified: pypy/dist/pypy/translator/jvm/generator.py ============================================================================== --- pypy/dist/pypy/translator/jvm/generator.py (original) +++ pypy/dist/pypy/translator/jvm/generator.py Fri May 18 13:26:07 2007 @@ -286,6 +286,12 @@ class Method(object): + # Create a constructor: + def c(classty, argtypes): + return Method(classty.name, "", argtypes, jVoid, + opcode=INVOKESPECIAL) + c = staticmethod(c) + # Create a virtual method: def v(classty, methnm, argtypes, rettype): """ @@ -376,19 +382,18 @@ PYPYSTRTODOUBLE = Method.s(jPyPy, 'str_to_double', (jString,), jDouble) PYPYSTRTOCHAR = Method.s(jPyPy, 'str_to_char', (jString,), jChar) PYPYDUMP = Method.s(jPyPy, 'dump', (jString,), jVoid) +PYPYDUMPEXCWRAPPER = Method.s(jPyPy, 'dump_exc_wrapper', (jObject,), jVoid) PYPYSERIALIZEBOOLEAN = Method.s(jPyPy, 'serialize_boolean', (jBool,), jString) PYPYSERIALIZEUINT = Method.s(jPyPy, 'serialize_uint', (jInt,), jString) PYPYSERIALIZEVOID = Method.s(jPyPy, 'serialize_void', (), jString) PYPYESCAPEDCHAR = Method.s(jPyPy, 'escaped_char', (jChar,), jString) PYPYESCAPEDSTRING = Method.s(jPyPy, 'escaped_string', (jString,), jString) PYPYSERIALIZEOBJECT = Method.s(jPyPy, 'serializeObject', (jObject,), jString) -PYPYDUMPEXCWRAPPER = Method.s(jPyPy, 'dump_exc_wrapper', (jObject,), jVoid) PYPYRUNTIMENEW = Method.s(jPyPy, 'RuntimeNew', (jClass,), jObject) PYPYSTRING2BYTES = Method.s(jPyPy, 'string2bytes', (jString,), jByteArray) PYPYARRAYTOLIST = Method.s(jPyPy, 'array_to_list', (jObjectArray,), jArrayList) OBJECTGETCLASS = Method.v(jObject, 'getClass', (), jClass) CLASSGETNAME = Method.v(jClass, 'getName', (), jString) -EXCWRAPWRAP = Method.s(jPyPyExcWrap, 'wrap', (jObject,), jPyPyExcWrap) CUSTOMDICTMAKE = Method.s(jPyPyCustomDict, 'make', (jPyPyEquals, jPyPyHashCode), jPyPyCustomDict) @@ -452,8 +457,6 @@ PYPYINTERLINK= Field(jPyPy.name, 'interlink', jPyPyInterlink, True) -EXCWRAPOBJ = Field(jPyPyExcWrap.name, 'object', jObject, False) - # ___________________________________________________________________________ # Generator State @@ -772,6 +775,10 @@ jclasstype, jscalartype.unbox_method, (), jscalartype) self.emit(jmethod) + def swap(self): + """ Swaps the two words highest on the stack. """ + self.emit(SWAP) + # __________________________________________________________________ # Exception Handling @@ -789,32 +796,16 @@ """ self.endtrylbl = self.unique_label("end_try", mark=True) - def begin_catch(self, excclsty): + def begin_catch(self, jexcclsty): """ Begins a catch region corresponding to the last try; there can be more than one call to begin_catch, in which case the last try region is reused. - 'excclsty' --- a JvmType for the class of exception to be caught + 'jexcclsty' --- a JvmType for the class of exception to be caught """ catchlbl = self.unique_label("catch", mark=True) self.try_catch_region( - jPyPyExcWrap, self.begintrylbl, self.endtrylbl, catchlbl) - - # emit the code to unwrap the exception, check the type - # of the unwrapped object, and re-throw the exception - # if it not the right type - catch = self.unique_label('catch') - self.emit(DUP) - EXCWRAPOBJ.load(self) - self.emit(INSTANCEOF, excclsty) - self.emit(IFNE, catch) - self.emit(ATHROW) - - # If it IS the right type, just dereference and get the - # wrapped Python object - self.mark(catch) - EXCWRAPOBJ.load(self) - self.emit(CHECKCAST, excclsty) + jexcclsty, self.begintrylbl, self.endtrylbl, catchlbl) def end_catch(self): """ @@ -823,14 +814,14 @@ """ return - def try_catch_region(self, excclsty, trystartlbl, tryendlbl, catchlbl): + def try_catch_region(self, jexcclsty, trystartlbl, tryendlbl, catchlbl): """ Indicates a try/catch region. Either invoked directly, or from the begin_catch() routine: the latter is invoked by the oosupport code. - 'excclsty' --- a JvmType for the class of exception to be caught + 'jexcclsty' --- a JvmType for the class of exception to be caught 'trystartlbl', 'tryendlbl' --- labels marking the beginning and end of the try region 'catchlbl' --- label marking beginning of catch region @@ -952,7 +943,12 @@ jtype = self.db.lltype_to_cts(TYPE) self._instr(INSTANCEOF, jtype) + # included for compatibility with oosupport, but instanceof_jtype + # follows our naming convention better def isinstance(self, jtype): + return self.instanceof_jtype(jtype) + + def instanceof_jtype(self, jtype): self._instr(INSTANCEOF, jtype) def branch_unconditionally(self, target_label): @@ -1008,7 +1004,7 @@ self.new_with_jtype(jtype) def new_with_jtype(self, jtype): - ctor = Method(jtype.name, "", (), jVoid, opcode=INVOKESPECIAL) + ctor = Method.c(jtype, ()) self.emit(NEW, jtype) self.emit(DUP) self.emit(ctor) @@ -1023,6 +1019,9 @@ def dup(self, OOTYPE): jvmtype = self.db.lltype_to_cts(OOTYPE) + self.dup_jtype(jvmtype) + + def dup_jtype(self, jvmtype): if jvmtype.descriptor.type_width() == 1: self.emit(DUP) else: @@ -1092,12 +1091,6 @@ def throw(self): """ Throw the object from top of the stack as an exception """ - - # We have to deal with the problem that exceptions must - # derive from Throwable, but our exception hierarchy in OOTYPE - # does not (today). For now, we use a wrapper class, which is - # probably the worst answer of all, but an easy one. - self.emit(EXCWRAPWRAP) self._instr(ATHROW) def iabs(self): @@ -1301,9 +1294,9 @@ self.curclass.out(' %-60s\n' % (instr_text,)) self.curfunc.instr_counter+=1 - def try_catch_region(self, excclsty, trystartlbl, tryendlbl, catchlbl): + def try_catch_region(self, jexcclsty, trystartlbl, tryendlbl, catchlbl): self.curclass.out(' .catch %s from %s to %s using %s\n' % ( - excclsty.descriptor.int_class_name(), + jexcclsty.descriptor.int_class_name(), trystartlbl.jasmin_syntax(), tryendlbl.jasmin_syntax(), catchlbl.jasmin_syntax())) Modified: pypy/dist/pypy/translator/jvm/genjvm.py ============================================================================== --- pypy/dist/pypy/translator/jvm/genjvm.py (original) +++ pypy/dist/pypy/translator/jvm/genjvm.py Fri May 18 13:26:07 2007 @@ -132,7 +132,6 @@ 'CustomDict', 'DictItemsIterator', 'Equals', - 'ExceptionWrapper', 'Filter', 'FilterIterator', 'FilterSet', Modified: pypy/dist/pypy/translator/jvm/node.py ============================================================================== --- pypy/dist/pypy/translator/jvm/node.py (original) +++ pypy/dist/pypy/translator/jvm/node.py Fri May 18 13:26:07 2007 @@ -143,7 +143,8 @@ gen.goto(done_printing) gen.end_try() - gen.begin_catch(jObject) + jexc = self.db.exception_root_object() + gen.begin_catch(jexc) gen.emit(jvmgen.PYPYDUMPEXCWRAPPER) # dumps to stdout gen.end_catch() @@ -289,8 +290,9 @@ def begin_catch(self, llexitcase): ll_meta_exc = llexitcase ll_exc = ll_meta_exc._inst.class_._INSTANCE - cts_exc = self.cts.lltype_to_cts(ll_exc) - self.ilasm.begin_catch(cts_exc) + jtype = self.cts.lltype_to_cts(ll_exc) + assert jtype.throwable # SHOULD only try to catch subtypes of Exception + self.ilasm.begin_catch(jtype) def end_catch(self, exit_lbl): self.ilasm.goto(exit_lbl) @@ -324,6 +326,15 @@ def render_raise_block(self, block): exc = block.inputargs[1] self.ilasm.load(exc) + + # Check whether the static type is known to be throwable. + # If not, emit a CHECKCAST to the base exception type. + # According to Samuele, no non-Exceptions should be thrown, + # but this is not enforced by the RTyper or annotator. + jtype = self.db.lltype_to_cts(exc.concretetype) + if not jtype.throwable: + self.ilasm.downcast_jtype(self.db.exception_root_object()) + self.ilasm.throw() def _trace(self, str): @@ -625,13 +636,13 @@ "java.lang.String", supercls is a Class object """ JvmClassType.__init__(self, name) - self.super_class = supercls # JvmType; if None, must use set_super_class self.rendered = False # has rendering occurred? self.abstract = False # is this an abstract class? self.fields = {} # maps field name to jvmgen.Field object self.interfaces = [] # list of JvmTypes self.methods = {} # maps method name to a Function object* self.abstract_methods = {} # maps method name to jvmgen.Method object + self.set_super_class(supercls) # * --- actually maps to an object that defines the # attributes: name, method() and render(). Usually, this is a @@ -645,6 +656,10 @@ def set_super_class(self, supercls): self.super_class = supercls + # Throwability is inherited: + if self.super_class and self.super_class.throwable: + self.throwable = True + def add_field(self, fieldobj, fielddef): """ Creates a new field accessed via the jvmgen.Field descriptor 'fieldobj'. Must be called before render().""" @@ -739,4 +754,3 @@ gen.emit(self.helper) gen.return_val(jVoid) gen.end_function() - Modified: pypy/dist/pypy/translator/jvm/typesystem.py ============================================================================== --- pypy/dist/pypy/translator/jvm/typesystem.py (original) +++ pypy/dist/pypy/translator/jvm/typesystem.py Fri May 18 13:26:07 2007 @@ -136,9 +136,10 @@ only work to obtain the descriptor. We use it on occasion for classes like java.lang.Object etc. """ - def __init__(self, classnm): + def __init__(self, classnm, throwable=False): JvmType.__init__(self, desc_for_class(classnm)) - self.name = classnm # public String, like 'java.lang.Object' + self.name = classnm # public; String, like 'java.lang.Object' + self.throwable = throwable # public; boolean def lookup_field(self, fieldnm): raise KeyError(fieldnm) # we treat as opaque type def lookup_method(self, methodnm): @@ -153,7 +154,7 @@ jByteClass = JvmClassType('java.lang.Byte') jCharClass = JvmClassType('java.lang.Character') jBoolClass = JvmClassType('java.lang.Boolean') -jThrowable = JvmClassType('java.lang.Throwable') +jThrowable = JvmClassType('java.lang.Throwable', throwable=True) jObject = JvmClassType('java.lang.Object') jString = JvmClassType('java.lang.String') jCharSequence = JvmClassType('java.lang.CharSequence') @@ -175,7 +176,7 @@ jPyPyInterlink = JvmClassType('pypy.Interlink') jPyPyCustomDict = JvmClassType('pypy.CustomDict') -jArithmeticException = JvmClassType('java.lang.ArithmeticException') +jArithmeticException = JvmClassType('java.lang.ArithmeticException', throwable=True) class JvmScalarType(JvmType): """ From cfbolz at codespeak.net Fri May 18 14:06:42 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 18 May 2007 14:06:42 +0200 (CEST) Subject: [pypy-svn] r43473 - in pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator: . test Message-ID: <20070518120642.E50AD8074@code0.codespeak.net> Author: cfbolz Date: Fri May 18 14:06:41 2007 New Revision: 43473 Modified: pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator/bookkeeper.py pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator/test/test_annotator.py Log: Allow functions to be manually marked as "pure". Modified: pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator/bookkeeper.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator/bookkeeper.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator/bookkeeper.py Fri May 18 14:06:41 2007 @@ -125,6 +125,14 @@ ARGTYPES = [v.concretetype for v in op.args] return not operation.is_pure(*ARGTYPES) + def analyze_direct_call(self, graph, seen=None): + try: + func = graph.func + if getattr(func, "_pure_function_", False): + return False + except AttributeError: + pass + return graphanalyze.GraphAnalyzer.analyze_direct_call(self, graph, seen) class HintBookkeeper(object): Modified: pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator/test/test_annotator.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator/test/test_annotator.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator/test/test_annotator.py Fri May 18 14:06:41 2007 @@ -919,3 +919,27 @@ hs = hannotate(f, [], policy=P_NOVIRTUAL) assert not hs.is_green() + +def test_manual_marking_of_pure_functions(): + d = {} + def h1(s): + try: + return d[s] + except KeyError: + d[s] = r = hash(s) + return r + h1._pure_function_ = True + def f(n): + hint(n, concrete=True) + if n == 0: + s = "abc" + else: + s = "123" + a = h1(s) + return a + + P = StopAtXPolicy(h1) + P.oopspec = True + P.entrypoint_returns_red = False + hs = hannotate(f, [int], policy=P) + assert hs.is_green() From cfbolz at codespeak.net Fri May 18 14:09:52 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 18 May 2007 14:09:52 +0200 (CEST) Subject: [pypy-svn] r43474 - pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test Message-ID: <20070518120952.349588074@code0.codespeak.net> Author: cfbolz Date: Fri May 18 14:09:49 2007 New Revision: 43474 Modified: pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_timeshift.py Log: timeshifting test for manual marking of pure functions Modified: pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_timeshift.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_timeshift.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_timeshift.py Fri May 18 14:09:49 2007 @@ -1513,6 +1513,42 @@ assert res == f(4,113) self.check_insns({}) + def test_manual_marking_of_pure_functions(self): + class A(object): + pass + class B(object): + pass + a1 = A() + a2 = A() + d = {} + def h1(b, s): + try: + return d[s] + except KeyError: + d[s] = r = A() + return r + h1._pure_function_ = True + b = B() + def f(n): + hint(None, global_merge_point=True) + hint(n, concrete=True) + if n == 0: + s = "abc" + else: + s = "123" + a = h1(b, s) + return hint(n, variable=True) + + P = StopAtXPolicy(h1) + P.oopspec = True + res = self.timeshift(f, [0], [], policy=P) + assert res == 0 + self.check_insns({}) + res = self.timeshift(f, [4], [], policy=P) + assert res == 4 + self.check_insns({}) + + def test_red_int_add_ovf(self): def f(n, m): try: @@ -1697,3 +1733,5 @@ res = self.timeshift(f, [3, 6], policy=MyPolicy()) assert res == -3 + 600 self.check_insns({'int_neg': 1, 'int_add': 1}) + + From cfbolz at codespeak.net Fri May 18 14:21:22 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 18 May 2007 14:21:22 +0200 (CEST) Subject: [pypy-svn] r43475 - pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test Message-ID: <20070518122122.E6D118074@code0.codespeak.net> Author: cfbolz Date: Fri May 18 14:21:22 2007 New Revision: 43475 Modified: pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_timeshift.py Log: test that shows that hash(green_string) is not green. Modified: pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_timeshift.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_timeshift.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_timeshift.py Fri May 18 14:21:22 2007 @@ -1734,4 +1734,19 @@ assert res == -3 + 600 self.check_insns({'int_neg': 1, 'int_add': 1}) + def test_hash_of_green_string_is_green(self): + py.test.skip("unfortunately doesn't work") + def f(n): + if n == 0: + s = "abc" + elif n == 1: + s = "cde" + else: + s = "fgh" + return hash(s) + + res = self.timeshift(f, [0]) + self.check_insns({'int_eq': 2}) + assert res == f(0) + From hpk at codespeak.net Fri May 18 14:29:40 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Fri, 18 May 2007 14:29:40 +0200 (CEST) Subject: [pypy-svn] r43476 - pypy/extradoc/talk/ep2007 Message-ID: <20070518122940.9BD4B8071@code0.codespeak.net> Author: hpk Date: Fri May 18 14:29:37 2007 New Revision: 43476 Added: pypy/extradoc/talk/ep2007/howto-eu-project.txt (contents, props changed) Log: a talk proposal based on discussion/drafting with/from lene and bea Added: pypy/extradoc/talk/ep2007/howto-eu-project.txt ============================================================================== --- (empty file) +++ pypy/extradoc/talk/ep2007/howto-eu-project.txt Fri May 18 14:29:37 2007 @@ -0,0 +1,19 @@ +how to do an EU open source research project +========================================================== + +We'll look at how to do an EU research proposal, based on +experiences obtained through the PyPy project. We quickly +walk through the basics of the initial proposal, negotiations, +the EU funding contract, cost/funding models and how to modify and +amend the contract afterwards. The PyPy project has been one +of the first bigger open source research projects that +received funding from the EU - a liaison of diverging +cultures, as it turned out. We had to learn how to manage +this project: track/report results, budget, 'resources' to match +our contractual obligations, but also make the EU contract and +circumstances fit our needs and actual project developments. +We'll highlight how the finalisation process with the EU +(still pending for 31st May and following) worked out, summarize +our findings and give some recommendations maybe helpful for +other projects and organisations applying for EU Research funding. + From cfbolz at codespeak.net Fri May 18 14:35:13 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 18 May 2007 14:35:13 +0200 (CEST) Subject: [pypy-svn] r43477 - in pypy/branch/prolog-jit-experiments/pypy/lang/prolog: builtin interpreter interpreter/test Message-ID: <20070518123513.2D5698074@code0.codespeak.net> Author: cfbolz Date: Fri May 18 14:35:12 2007 New Revision: 43477 Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/__init__.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/register.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/portal.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/term.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/test/test_jit.py Log: assorted hints and hacks to make pyrolog-jit better. gives a speedup of around 2. Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/__init__.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/__init__.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/__init__.py Fri May 18 14:35:12 2007 @@ -1,5 +1,6 @@ # all builtins builtins = {} +builtins_list = [] # imports to register builtins import pypy.lang.prolog.builtin.allsolution Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/register.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/register.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/register.py Fri May 18 14:35:12 2007 @@ -2,17 +2,20 @@ from pypy.lang.prolog.interpreter import arithmetic from pypy.lang.prolog.interpreter.parsing import parse_file, TermBuilder from pypy.lang.prolog.interpreter import engine, helper, term, error -from pypy.lang.prolog.builtin import builtins +from pypy.lang.prolog.builtin import builtins, builtins_list from pypy.rlib.objectmodel import we_are_translated class Builtin(object): + _immutable_ = True def __init__(self, function): self.function = function def call(self, engine, query, continuation): return self.function(engine, query, continuation) + def _freeze_(self): + return True def expose_builtin(func, name, unwrap_spec=None, handles_continuation=False, translatable=True): @@ -80,5 +83,6 @@ exec py.code.Source("\n".join(code)).compile() in miniglobals for name in expose_as: signature = "%s/%s" % (name, len(unwrap_spec)) - builtins[signature] = Builtin(miniglobals[funcname]) - + b = Builtin(miniglobals[funcname]) + builtins[signature] = b + builtins_list.append((signature, b)) Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py Fri May 18 14:35:12 2007 @@ -3,7 +3,8 @@ from pypy.lang.prolog.interpreter.error import UnificationFailed, FunctionNotFound, \ CutException from pypy.lang.prolog.interpreter import error -from pypy.rlib.objectmodel import hint, specialize +from pypy.rlib.objectmodel import hint, specialize, _is_early_constant +from pypy.rlib.unroll import unrolling_iterable DEBUG = False @@ -95,6 +96,7 @@ return result class LinkedRules(object): + _immutable_ = True def __init__(self, rule, next=None): self.rule = rule self.next = next @@ -114,13 +116,19 @@ #import pdb;pdb.set_trace() while self: uh = self.rule.unify_hash - for j in range(len(uh)): + hint(uh, concrete=True) + uh = hint(uh, deepfreeze=True) + j = 0 + while j < len(uh): + hint(j, concrete=True) hash1 = uh[j] hash2 = query.unify_hash_of_child(j) if hash1 != 0 and hash2 != 0 and hash1 != hash2: break + j += 1 else: - return self + #XXX otherwise the result is green which seems wrong + return hint(self, variable=True) self = self.next return None @@ -128,14 +136,19 @@ return "LinkedRules(%r, %r)" % (self.rule, self.next) + class Function(object): - def __init__(self, firstrule): - self.rulechain = LinkedRules(firstrule) - self.last = self.rulechain + def __init__(self, firstrule=None): + if firstrule is None: + self.rulechain = self.last = None + else: + self.rulechain = LinkedRules(firstrule) + self.last = self.rulechain def add_rule(self, rule, end): - #import pdb; pdb.set_trace() - if end: + if self.rulechain is None: + self.rulechain = self.last = LinkedRules(rule) + elif end: self.rulechain, last = self.rulechain.copy() self.last = LinkedRules(rule) last.next = self.last @@ -146,13 +159,17 @@ self.rulechain, last = self.rulechain.copy(rulechain) last.next = rulechain.next + class Engine(object): def __init__(self): self.heap = Heap() self.signature2function = {} self.parser = None self.operations = None - + #XXX circular imports hack + from pypy.lang.prolog.builtin import builtins_list + globals()['unrolling_builtins'] = unrolling_iterable(builtins_list) + def add_rule(self, rule, end=True): from pypy.lang.prolog import builtin if DEBUG: @@ -204,29 +221,57 @@ def call(self, query, continuation=DONOTHING): assert isinstance(query, Callable) - from pypy.lang.prolog.builtin import builtins if DEBUG: debug_print("calling", query) signature = query.signature # check for builtins - builtins = hint(builtins, deepfreeze=True) + if _is_early_constant(signature): + for bsig, builtin in unrolling_builtins: + if signature == bsig: + #XXX should be: + #return builtin.call(self, query, continuation) + # but then the JIT explodes sometimes for funny reasons + return builtin.function(self, query, continuation) + # do a real call + return self.user_call(query, continuation) + else: + return self._opaque_call(signature, query, continuation) + + def _opaque_call(self, signature, query, continuation): + from pypy.lang.prolog.builtin import builtins + #builtins = hint(builtins, deepfreeze=True) builtin = builtins.get(signature, None) if builtin is not None: - return builtin.call(self, query, continuation) + #XXX should be: + #return builtin.call(self, query, continuation) + # but then the JIT explodes sometimes for funny reasons + return builtin.function(self, query, continuation) # do a real call return self.user_call(query, continuation) + def _jit_lookup(self, signature): + signature2function = self.signature2function + function = signature2function.get(signature, None) + if function is None: + signature2function[signature] = function = Function() + return function + _jit_lookup._pure_function_ = True + def user_call(self, query, continuation): #import pdb; pdb.set_trace() - signature = query.signature - function = self.signature2function.get(signature, None) - if function is None: + signature = hint(query.signature, promote=True) + function = self._jit_lookup(signature) + startrulechain = function.rulechain + startrulechain = hint(startrulechain, promote=True) + if startrulechain is None: error.throw_existence_error( "procedure", query.get_prolog_signature()) - rulechain = function.rulechain.find_applicable_rule(query) + + rulechain = startrulechain.find_applicable_rule(query) if rulechain is None: # none of the rules apply raise UnificationFailed() + rulechain = hint(rulechain, promote=True) rule = rulechain.rule rulechain = rulechain.next oldstate = self.heap.branch() @@ -235,6 +280,8 @@ if rulechain is None: self.heap.discard(oldstate) break + rulechain = hint(rulechain, promote=True) + hint(rule, concrete=True) if rule.contains_cut: continuation = LimitedScopeContinuation(continuation) try: @@ -257,6 +304,7 @@ self.heap.revert(oldstate) rule = rulechain.rule rulechain = rulechain.next + hint(rule, concrete=True) if rule.contains_cut: continuation = LimitedScopeContinuation(continuation) try: @@ -268,9 +316,12 @@ return self.try_rule(rule, query, continuation) def try_rule(self, rule, query, continuation=DONOTHING): + return self._portal_try_rule(rule, query, continuation) + + def _portal_try_rule(self, rule, query, continuation=DONOTHING): hint(None, global_merge_point=True) - rule = hint(rule, promote=True) rule = hint(rule, deepfreeze=True) + hint(self, concrete=True) if DEBUG: debug_print("trying rule", rule, query, self.heap.vars[:self.heap.needed_vars]) # standardizing apart @@ -284,7 +335,6 @@ continuation.call(self) return "yes" - def continue_after_cut(self, continuation, lsc=None): while 1: try: @@ -306,3 +356,7 @@ if self.operations is None: return default_operations return self.operations + + + + Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/portal.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/portal.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/portal.py Fri May 18 14:35:12 2007 @@ -11,7 +11,7 @@ 'pypy.lang.prolog.builtin.register': True } -PORTAL = engine.Engine.try_rule +PORTAL = engine.Engine._portal_try_rule.im_func class PyrologHintAnnotatorPolicy(HintAnnotatorPolicy): novirtualcontainer = True @@ -126,13 +126,21 @@ seegraph(cls.copy) seegraph(cls.__init__) seegraph(cls.copy_and_unify) + seegraph(cls.unify_hash_of_child) for cls in [term.Term, term.Number, term.Atom]: seegraph(cls.copy_and_basic_unify) + seegraph(cls.dereference) seegraph(cls.copy_and_basic_unify) + for cls in [term.Var, term.Term, term.Number, term.Atom]: + seegraph(cls.get_unify_hash) + for cls in [term.Callable, term.Atom, term.Term]: + seegraph(cls.get_prolog_signature) + seegraph(PORTAL) seegraph(pypy.lang.prolog.interpreter.engine.Heap.newvar) - seegraph(pypy.lang.prolog.interpreter.engine.Engine.try_rule) seegraph(pypy.lang.prolog.interpreter.term.Rule.clone_and_unify_head) seegraph(pypy.lang.prolog.interpreter.engine.Engine.call) + seegraph(pypy.lang.prolog.interpreter.engine.Engine.user_call) + seegraph(pypy.lang.prolog.interpreter.engine.LinkedRules.find_applicable_rule) return result_graphs def get_portal(drv): Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/term.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/term.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/term.py Fri May 18 14:35:12 2007 @@ -18,6 +18,10 @@ if DEBUG and not we_are_translated(): print " ".join([str(a) for a in args]) +def pure_hash_function(s): + return h(s) +pure_hash_function._pure_function_ = True + class PrologObject(object): __slots__ = () _immutable_ = True @@ -49,7 +53,7 @@ # they must not be unifiable raise NotImplementedError("abstract base class") - def unify_hash_of_child(self, i, heap=None): + def unify_hash_of_child(self, i): raise KeyError @specialize.arg(3) @@ -79,6 +83,7 @@ __slots__ = ('index', ) cache = {} + _immutable_ = True def __init__(self, index): self.index = index @@ -223,6 +228,7 @@ STANDARD_ORDER = 1 cache = {} + _immutable_ = True def __init__(self, name): self.name = name @@ -253,10 +259,11 @@ raise UnificationFailed def get_unify_hash(self): - return intmask(hash(self.name) << TAGBITS | self.TAG) + name = hint(self.name, promote=True) + return intmask(pure_hash_function(name) << TAGBITS | self.TAG) def get_prolog_signature(self): - return Term("/", [self, Number(0)]) + return Term("/", [self, NUMBER_0]) def newatom(name): result = Atom.cache.get(name, None) @@ -270,6 +277,7 @@ class Number(NonVar): TAG = tag() STANDARD_ORDER = 2 + _immutable_ = True def __init__(self, num): self.num = num @@ -298,10 +306,12 @@ def get_unify_hash(self): return intmask(self.num << TAGBITS | self.TAG) +NUMBER_0 = Number(0) class Float(NonVar): TAG = tag() STANDARD_ORDER = 2 + _immutable_ = True def __init__(self, num): self.num = num @@ -374,6 +384,7 @@ class Term(Callable): TAG = tag() STANDARD_ORDER = 3 + _immutable_ = True def __init__(self, name, args, signature=None): self.name = name self.args = args @@ -453,7 +464,8 @@ return self def get_unify_hash(self): - return intmask(hash(self.signature) << TAGBITS | self.TAG) + signature = hint(self.signature, promote=True) + return intmask(pure_hash_function(signature) << TAGBITS | self.TAG) def unify_hash_of_child(self, i): return self.args[i].get_unify_hash() @@ -506,6 +518,7 @@ def clone_and_unify_head(self, heap, head): memo = {} h2 = self.head + hint(h2, concrete=True) if isinstance(h2, Term): assert isinstance(head, Term) i = 0 @@ -516,6 +529,7 @@ arg2.copy_and_unify(arg1, heap, memo) i += 1 body = self.body + hint(body, concrete=True) if body is None: return None return body.copy(heap, memo) Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/test/test_jit.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/test/test_jit.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/test/test_jit.py Fri May 18 14:35:12 2007 @@ -1,5 +1,5 @@ import py -from pypy.jit.timeshifter.test.test_portal import PortalTest +from pypy.jit.timeshifter.test.test_portal import PortalTest, P_NOVIRTUAL from pypy.lang.prolog.interpreter import portal from pypy.lang.prolog.interpreter import engine, term from pypy.lang.prolog.interpreter.parsing import parse_query_term, get_engine @@ -37,11 +37,59 @@ assert res == True - res = self.timeshift_from_portal(main, engine.Engine.try_rule.im_func, + res = self.timeshift_from_portal(main, portal.PORTAL, [1], policy=POLICY) assert res == True - res = self.timeshift_from_portal(main, engine.Engine.try_rule.im_func, + res = self.timeshift_from_portal(main, portal.PORTAL, + [0], policy=POLICY) + assert res == True + + def test_and(self): + e = get_engine(""" + b(a). + a(a). + f(X) :- b(X), a(X). + """) + X = e.heap.newvar() + + def main(n): + e.heap.reset() + if n == 0: + e.call(term.Term("f", [X])) + return isinstance(X.dereference(e.heap), term.Atom) + else: + return False + + res = main(0) + assert res == True + + res = self.timeshift_from_portal(main, portal.PORTAL, + [0], policy=POLICY) + assert res == True + + def test_user_call(self): + e = get_engine(""" + h(X) :- f(X, b). + f(a, a). + f(X, b) :- g(X). + g(b). + """) + X = e.heap.newvar() + + def main(n): + e.heap.reset() + if n == 0: + e.call(term.Term("h", [X])) + return isinstance(X.dereference(e.heap), term.Atom) + else: + return False + + res = main(0) + assert res == True + + + res = self.timeshift_from_portal(main, portal.PORTAL, [0], policy=POLICY) assert res == True From hpk at codespeak.net Fri May 18 14:45:48 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Fri, 18 May 2007 14:45:48 +0200 (CEST) Subject: [pypy-svn] r43478 - pypy/extradoc/talk/ep2007 Message-ID: <20070518124548.EF7D78071@code0.codespeak.net> Author: hpk Date: Fri May 18 14:45:48 2007 New Revision: 43478 Modified: pypy/extradoc/talk/ep2007/abstract_interpreter.txt Log: suggested refinements to python interpreter talk Modified: pypy/extradoc/talk/ep2007/abstract_interpreter.txt ============================================================================== --- pypy/extradoc/talk/ep2007/abstract_interpreter.txt (original) +++ pypy/extradoc/talk/ep2007/abstract_interpreter.txt Fri May 18 14:45:48 2007 @@ -1,23 +1,23 @@ -Why use PyPy interpreter? +PyPy Python Interpreter(s) Features ===================================== -XXX Better title? - Speakers: Holger Krekel & Maciek Fijalkowski -Abstract ---------- +We quickly recap the basic architecture of PyPy Python interpreter(s) +and then demo and discuss the following unique features: + +* transparent proxy: a way to customize behaviour of builtin + objects, enabling new models of persistence and distribution +* distribution prototype: have objects from remote places + appear as local ones, including frames (PDB!), file objects etc. +* object tainting: automatically track and control propagation of + sensitive data through an application +* lazy computations: defer computation until result is needed +* generating from a single python interpreter source + code to C, LLVM, .NET, JVM, ... (IOW: implement features once, + have it available everywhere) +* ... and more, as talk topics/features might appear just-in-time :) -We would like to present features showcasing PyPy's flexible architecture, -in other words, what makes the pypy interpreter unique. Showcases will -include, but not limited to: +We'll also see to discuss future efforts and remaining obstacles +to reach wider adoption. -* multiple implementations of interpreter objects, allows more flexibility - regarding object usage, without user intervention (or even knowledge) -* transparent proxy, a way to have special objects of interpreter - types with completely controlled behaviour from application level. -* distribution, strange implementation of RMI which allows of remote - access to everything including frames, file objects etc. -* object tainting, security feature similiar to ruby's tainting. -* lazy computations -* ... and many more, as talk topics might appear just-in-time From lene at codespeak.net Fri May 18 15:22:14 2007 From: lene at codespeak.net (lene at codespeak.net) Date: Fri, 18 May 2007 15:22:14 +0200 (CEST) Subject: [pypy-svn] r43480 - pypy/extradoc/talk/ep2007 Message-ID: <20070518132214.73581807C@code0.codespeak.net> Author: lene Date: Fri May 18 15:22:14 2007 New Revision: 43480 Modified: pypy/extradoc/talk/ep2007/howto-eu-project.txt Log: we-ify a sentence Modified: pypy/extradoc/talk/ep2007/howto-eu-project.txt ============================================================================== --- pypy/extradoc/talk/ep2007/howto-eu-project.txt (original) +++ pypy/extradoc/talk/ep2007/howto-eu-project.txt Fri May 18 15:22:14 2007 @@ -4,8 +4,8 @@ We'll look at how to do an EU research proposal, based on experiences obtained through the PyPy project. We quickly walk through the basics of the initial proposal, negotiations, -the EU funding contract, cost/funding models and how to modify and -amend the contract afterwards. The PyPy project has been one +the EU funding contract, cost/funding models and how we modified and +amended the contract afterwards. The PyPy project has been one of the first bigger open source research projects that received funding from the EU - a liaison of diverging cultures, as it turned out. We had to learn how to manage From fijal at codespeak.net Fri May 18 15:52:13 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 18 May 2007 15:52:13 +0200 (CEST) Subject: [pypy-svn] r43483 - pypy/extradoc/talk/ep2007 Message-ID: <20070518135213.40EE98074@code0.codespeak.net> Author: fijal Date: Fri May 18 15:52:12 2007 New Revision: 43483 Modified: pypy/extradoc/talk/ep2007/abstract_interpreter.txt Log: (fijal, arigo) update Modified: pypy/extradoc/talk/ep2007/abstract_interpreter.txt ============================================================================== --- pypy/extradoc/talk/ep2007/abstract_interpreter.txt (original) +++ pypy/extradoc/talk/ep2007/abstract_interpreter.txt Fri May 18 15:52:12 2007 @@ -13,11 +13,11 @@ * object tainting: automatically track and control propagation of sensitive data through an application * lazy computations: defer computation until result is needed -* generating from a single python interpreter source - code to C, LLVM, .NET, JVM, ... (IOW: implement features once, - have it available everywhere) * ... and more, as talk topics/features might appear just-in-time :) +All above features might be translated to any supported backend, +which is by now C, LLVM and CLI, soon JVM. + We'll also see to discuss future efforts and remaining obstacles to reach wider adoption. From jlg at codespeak.net Fri May 18 16:10:34 2007 From: jlg at codespeak.net (jlg at codespeak.net) Date: Fri, 18 May 2007 16:10:34 +0200 (CEST) Subject: [pypy-svn] r43484 - pypy/dist/pypy/lang/scheme/test Message-ID: <20070518141034.2C2E68074@code0.codespeak.net> Author: jlg Date: Fri May 18 16:10:34 2007 New Revision: 43484 Added: pypy/dist/pypy/lang/scheme/test/test_object.py Log: pypy/lang/scheme parser + object added Added: pypy/dist/pypy/lang/scheme/test/test_object.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/scheme/test/test_object.py Fri May 18 16:10:34 2007 @@ -0,0 +1,12 @@ +from pypy.lang.scheme.object import * + +def test_false(): + w_false = W_False() + assert w_false.to_boolean() is False + assert isinstance(w_false, W_Boolean) + +def test_true(): + w_true = W_True() + assert w_true.to_boolean() is True + assert isinstance(w_true, W_Boolean) + From bea at codespeak.net Fri May 18 16:23:34 2007 From: bea at codespeak.net (bea at codespeak.net) Date: Fri, 18 May 2007 16:23:34 +0200 (CEST) Subject: [pypy-svn] r43485 - pypy/extradoc/talk/ep2007 Message-ID: <20070518142334.A60B9807E@code0.codespeak.net> Author: bea Date: Fri May 18 16:23:34 2007 New Revision: 43485 Modified: pypy/extradoc/talk/ep2007/planning.txt Log: talk between me and holger - summarizing here Modified: pypy/extradoc/talk/ep2007/planning.txt ============================================================================== --- pypy/extradoc/talk/ep2007/planning.txt (original) +++ pypy/extradoc/talk/ep2007/planning.txt Fri May 18 16:23:34 2007 @@ -1,8 +1,14 @@ I'm listing here some ideas gathered around on IRC: -1. "A usual pypy talk" - architecture, motivation etc. +1. "A usual pypy talk" - architecture, motivation etc. (samuele,armin) 2. An RPython talk: .NET, JVM, JS, extcompiler, possible uses, caveats, - probably some architecture + probably some architecture (anot,fijal) 3. Interpreters talk: taint, thunk, persistance, distribution, probably some stackless (?), interpreter for .NET (hpk, fijal) -4. Some agile talk (?) (Bea) +4. recommendations on "how to" OSS/EU-funding/research project (hpk,lene) +5. development process "how to" (bea, hpk maybe) + +These last two talks (4-5) should be scheduled together - let?s ask for it +when submitting. They are "final" recommendations and reflections from PyPy +regarding formal structures and informal structures (contract and development process). +(Phone discussion between hpk and bea 18/5) From bea at codespeak.net Fri May 18 16:23:47 2007 From: bea at codespeak.net (bea at codespeak.net) Date: Fri, 18 May 2007 16:23:47 +0200 (CEST) Subject: [pypy-svn] r43486 - pypy/extradoc/talk/ep2007 Message-ID: <20070518142347.964C3807E@code0.codespeak.net> Author: bea Date: Fri May 18 16:23:47 2007 New Revision: 43486 Modified: pypy/extradoc/talk/ep2007/howto-eu-project.txt Log: reading/typo Modified: pypy/extradoc/talk/ep2007/howto-eu-project.txt ============================================================================== --- pypy/extradoc/talk/ep2007/howto-eu-project.txt (original) +++ pypy/extradoc/talk/ep2007/howto-eu-project.txt Fri May 18 16:23:47 2007 @@ -1,4 +1,6 @@ -how to do an EU open source research project + + +How to do an EU open source research project ========================================================== We'll look at how to do an EU research proposal, based on From bea at codespeak.net Fri May 18 16:24:28 2007 From: bea at codespeak.net (bea at codespeak.net) Date: Fri, 18 May 2007 16:24:28 +0200 (CEST) Subject: [pypy-svn] r43487 - pypy/extradoc/talk/ep2007 Message-ID: <20070518142428.7F8BA807E@code0.codespeak.net> Author: bea Date: Fri May 18 16:24:28 2007 New Revision: 43487 Added: pypy/extradoc/talk/ep2007/development.txt Log: a draft for a talk to be done (hopefully) just after the how to talk Added: pypy/extradoc/talk/ep2007/development.txt ============================================================================== --- (empty file) +++ pypy/extradoc/talk/ep2007/development.txt Fri May 18 16:24:28 2007 @@ -0,0 +1,14 @@ +Post Mortem: development practices and infrastructure +in the EU open source research project PyPy + +How did the development process and infrastructure of the PyPy +project change during the period of EU-funding? We will talk about +our experiences regarding the evolving development environment (version +control, automated test-driven development, sprints, synchronization, +"conceptual integrity", communication channels). We will also share our +"learning by burnings" so that other projects can explore other +creative problems and mistakes than maybe repeating the ones we +identified during the last 3 years of PyPy development. And hopefully +we will be able to show that the OSS development style is compatible +with EU-funding. + From hpk at codespeak.net Fri May 18 16:41:40 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Fri, 18 May 2007 16:41:40 +0200 (CEST) Subject: [pypy-svn] r43488 - pypy/extradoc/talk/ep2007 Message-ID: <20070518144140.A5322806F@code0.codespeak.net> Author: hpk Date: Fri May 18 16:41:40 2007 New Revision: 43488 Modified: pypy/extradoc/talk/ep2007/development.txt Log: quick refactoring Modified: pypy/extradoc/talk/ep2007/development.txt ============================================================================== --- pypy/extradoc/talk/ep2007/development.txt (original) +++ pypy/extradoc/talk/ep2007/development.txt Fri May 18 16:41:40 2007 @@ -1,14 +1,18 @@ -Post Mortem: development practices and infrastructure -in the EU open source research project PyPy +PyPy: Why and how did it (not) work? +------------------------------------------------- -How did the development process and infrastructure of the PyPy -project change during the period of EU-funding? We will talk about -our experiences regarding the evolving development environment (version -control, automated test-driven development, sprints, synchronization, -"conceptual integrity", communication channels). We will also share our -"learning by burnings" so that other projects can explore other -creative problems and mistakes than maybe repeating the ones we -identified during the last 3 years of PyPy development. And hopefully -we will be able to show that the OSS development style is compatible -with EU-funding. +Some people have said that they have hardly seen a large +project succeed with delivering to the original goals as much +as the PyPy project did. Others consider it a failure because +it did not take over the world yet or is not usable for +mainstream purposes. Anyway, we'll talk about the mix of +development processes, methods and infrastructure of the PyPy +project. We will summarize our experiences regarding the +evolving development environment (version control, automated +test-driven development, sprints, synchronization, "conceptual +integrity", communication channels). We will also try to share +our "learning by burnings" so that other projects can explore +other creative problems and mistakes than maybe repeating the +ones we explored in depth already during the last 4 years of +PyPy development. From antocuni at codespeak.net Fri May 18 16:49:35 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 18 May 2007 16:49:35 +0200 (CEST) Subject: [pypy-svn] r43489 - pypy/extradoc/talk/ep2007 Message-ID: <20070518144935.EE2EE8071@code0.codespeak.net> Author: antocuni Date: Fri May 18 16:49:35 2007 New Revision: 43489 Modified: pypy/extradoc/talk/ep2007/ (props changed) pypy/extradoc/talk/ep2007/abstract_interpreter.txt (props changed) pypy/extradoc/talk/ep2007/development.txt (contents, props changed) pypy/extradoc/talk/ep2007/planning.txt (contents, props changed) pypy/extradoc/talk/ep2007/pypy_10_and_jit.txt (props changed) Log: typo and fixeol Modified: pypy/extradoc/talk/ep2007/development.txt ============================================================================== --- pypy/extradoc/talk/ep2007/development.txt (original) +++ pypy/extradoc/talk/ep2007/development.txt Fri May 18 16:49:35 2007 @@ -1,18 +1,18 @@ -PyPy: Why and how did it (not) work? -------------------------------------------------- - -Some people have said that they have hardly seen a large -project succeed with delivering to the original goals as much -as the PyPy project did. Others consider it a failure because -it did not take over the world yet or is not usable for -mainstream purposes. Anyway, we'll talk about the mix of -development processes, methods and infrastructure of the PyPy -project. We will summarize our experiences regarding the -evolving development environment (version control, automated -test-driven development, sprints, synchronization, "conceptual -integrity", communication channels). We will also try to share -our "learning by burnings" so that other projects can explore -other creative problems and mistakes than maybe repeating the -ones we explored in depth already during the last 4 years of -PyPy development. - +PyPy: Why and how did it (not) work? +------------------------------------------------- + +Some people have said that they have hardly seen a large +project succeed with delivering to the original goals as much +as the PyPy project did. Others consider it a failure because +it did not take over the world yet or is not usable for +mainstream purposes. Anyway, we'll talk about the mix of +development processes, methods and infrastructure of the PyPy +project. We will summarize our experiences regarding the +evolving development environment (version control, automated +test-driven development, sprints, synchronization, "conceptual +integrity", communication channels). We will also try to share +our "learning by burnings" so that other projects can explore +other creative problems and mistakes than maybe repeating the +ones we explored in depth already during the last 4 years of +PyPy development. + Modified: pypy/extradoc/talk/ep2007/planning.txt ============================================================================== --- pypy/extradoc/talk/ep2007/planning.txt (original) +++ pypy/extradoc/talk/ep2007/planning.txt Fri May 18 16:49:35 2007 @@ -2,13 +2,13 @@ 1. "A usual pypy talk" - architecture, motivation etc. (samuele,armin) 2. An RPython talk: .NET, JVM, JS, extcompiler, possible uses, caveats, - probably some architecture (anot,fijal) + probably some architecture (anto,fijal) 3. Interpreters talk: taint, thunk, persistance, distribution, probably some stackless (?), interpreter for .NET (hpk, fijal) -4. recommendations on "how to" OSS/EU-funding/research project (hpk,lene) -5. development process "how to" (bea, hpk maybe) - -These last two talks (4-5) should be scheduled together - let?s ask for it -when submitting. They are "final" recommendations and reflections from PyPy -regarding formal structures and informal structures (contract and development process). +4. recommendations on "how to" OSS/EU-funding/research project (hpk,lene) +5. development process "how to" (bea, hpk maybe) + +These last two talks (4-5) should be scheduled together - let?s ask for it +when submitting. They are "final" recommendations and reflections from PyPy +regarding formal structures and informal structures (contract and development process). (Phone discussion between hpk and bea 18/5) From pedronis at codespeak.net Fri May 18 16:54:51 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Fri, 18 May 2007 16:54:51 +0200 (CEST) Subject: [pypy-svn] r43490 - pypy/extradoc/talk/ep2007 Message-ID: <20070518145451.63307807C@code0.codespeak.net> Author: pedronis Date: Fri May 18 16:54:49 2007 New Revision: 43490 Modified: pypy/extradoc/talk/ep2007/pypy_10_and_jit.txt Log: text instead of bullet points Modified: pypy/extradoc/talk/ep2007/pypy_10_and_jit.txt ============================================================================== --- pypy/extradoc/talk/ep2007/pypy_10_and_jit.txt (original) +++ pypy/extradoc/talk/ep2007/pypy_10_and_jit.txt Fri May 18 16:54:49 2007 @@ -6,8 +6,17 @@ Abstract ---------- -- PyPy intro and motivation -- Recapitulation of architecture -- The JIT and the ideas that made it possible (overview with smallish examples) -- Current status and the future +PyPy released 1.0 in March of this year. PyPy contains a very +compliant Python interpreter, and with 1.0 the first +incarnation of a Just-In-Time compiler which is generated +from the interpreter automatically with novel techniques. +In this talk we are going to give a brief introduction +to PyPy and its motivation. After recapitulating PyPy +architecture we are going to give an overview of +how the JIT generation is achieved, through examples +and introducing the general ideas. + +In the last part we are going to reflect on the current +status of PyPy's Python interpreter and our thoughts +about its future and further progress on the JIT compiler. From cfbolz at codespeak.net Sat May 19 10:25:57 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 19 May 2007 10:25:57 +0200 (CEST) Subject: [pypy-svn] r43499 - pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter Message-ID: <20070519082557.CA260807E@code0.codespeak.net> Author: cfbolz Date: Sat May 19 10:25:57 2007 New Revision: 43499 Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/term.py Log: typo Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/term.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/term.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/term.py Sat May 19 10:25:57 2007 @@ -19,7 +19,7 @@ print " ".join([str(a) for a in args]) def pure_hash_function(s): - return h(s) + return hash(s) pure_hash_function._pure_function_ = True class PrologObject(object): From cfbolz at codespeak.net Sat May 19 10:38:05 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 19 May 2007 10:38:05 +0200 (CEST) Subject: [pypy-svn] r43500 - in pypy/branch/prolog-jit-experiments/pypy/lang/prolog: builtin interpreter interpreter/test Message-ID: <20070519083805.82503807E@code0.codespeak.net> Author: cfbolz Date: Sat May 19 10:38:05 2007 New Revision: 43500 Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/allsolution.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/arithmeticbuiltin.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/atomconstruction.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/control.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/exception.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/metacall.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/parseraccess.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/register.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/termconstruction.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/helper.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/interactive.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/test/tool.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/translatedmain.py Log: intermediate checkin towards using a more classical bytecode interpreter for Prolog. So far the bytecode is not actually compiled, but produced at runtime. Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/allsolution.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/allsolution.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/allsolution.py Sat May 19 10:38:05 2007 @@ -10,7 +10,7 @@ self.found = [] self.template = template - def call(self, engine): + def _call(self, engine): clone = self.template.getvalue(engine.heap) self.found.append(clone) raise error.UnificationFailed() Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/arithmeticbuiltin.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/arithmeticbuiltin.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/arithmeticbuiltin.py Sat May 19 10:38:05 2007 @@ -12,16 +12,16 @@ oldstate = engine.heap.branch() try: varorint.unify(term.Number(i), engine.heap) - return continuation.call(engine) + return continuation.call(engine, choice_point=True) except error.UnificationFailed: engine.heap.revert(oldstate) varorint.unify(term.Number(upper), engine.heap) - return continuation.call(engine) + return continuation.call(engine, choice_point=False) else: integer = helper.unwrap_int(varorint) if not (lower <= integer <= upper): raise error.UnificationFailed - return continuation.call(engine) + return continuation.call(engine, choice_point=False) expose_builtin(impl_between, "between", unwrap_spec=["int", "int", "obj"], handles_continuation=True) Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/atomconstruction.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/atomconstruction.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/atomconstruction.py Sat May 19 10:38:05 2007 @@ -15,7 +15,7 @@ try: a1.unify(term.Atom(r[:i]), engine.heap) a2.unify(term.Atom(r[i:]), engine.heap) - return continuation.call(engine) + return continuation.call(engine, choice_point=True) except error.UnificationFailed: engine.heap.revert(oldstate) raise error.UnificationFailed() @@ -39,7 +39,7 @@ else: s2 = helper.convert_to_str(a2) result.unify(term.Atom(s1 + s2), engine.heap) - return continuation.call(engine) + return continuation.call(engine, choice_point=False) expose_builtin(impl_atom_concat, "atom_concat", unwrap_spec=["obj", "obj", "obj"], handles_continuation=True) @@ -86,7 +86,7 @@ before.unify(term.Number(b), engine.heap) after.unify(term.Number(len(s) - len(s1) - b), engine.heap) length.unify(term.Number(len(s1)), engine.heap) - return continuation.call(engine) + return continuation.call(engine, choice_point=True) except: engine.heap.revert(oldstate) raise @@ -104,7 +104,7 @@ after.unify(term.Number(len(s) - l - b), engine.heap) length.unify(term.Number(l), engine.heap) sub.unify(term.Atom(s[b:b + l]), engine.heap) - return continuation.call(engine) + return continuation.call(engine, choice_point=True) except: engine.heap.revert(oldstate) raise @@ -123,7 +123,7 @@ after.unify(term.Number(a), engine.heap) length.unify(term.Number(l), engine.heap) sub.unify(term.Atom(s[b:b + l]), engine.heap) - return continuation.call(engine) + return continuation.call(engine, choice_point=True) return None except: engine.heap.revert(oldstate) Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/control.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/control.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/control.py Sat May 19 10:38:05 2007 @@ -11,13 +11,13 @@ expose_builtin(impl_fail, "fail", unwrap_spec=[]) def impl_true(engine): - pass + return expose_builtin(impl_true, "true", unwrap_spec=[]) def impl_repeat(engine, continuation): while 1: try: - return continuation.call(engine) + return continuation.call(engine, choice_point=True) except error.UnificationFailed: pass expose_builtin(impl_repeat, "repeat", unwrap_spec=[], handles_continuation=True) @@ -32,18 +32,16 @@ self.next_call = next_call self.continuation = continuation - def call(self, engine): + def _call(self, engine): next_call = self.next_call.dereference(engine.heap) - if isinstance(next_call, term.Var): - error.throw_instantiation_error() next_call = helper.ensure_callable(next_call) - return engine.call(next_call, self.continuation) + return engine.call(next_call, self.continuation, choice_point=False) def impl_and(engine, call1, call2, continuation): if not isinstance(call2, term.Var) and not isinstance(call2, term.Callable): error.throw_type_error('callable', call2) and_continuation = AndContinuation(call2, continuation) - return engine.call(call1, and_continuation) + return engine.call(call1, and_continuation, choice_point=False) expose_builtin(impl_and, ",", unwrap_spec=["callable", "raw"], handles_continuation=True) @@ -53,7 +51,7 @@ return engine.call(call1, continuation) except error.UnificationFailed: engine.heap.revert(oldstate) - return engine.call(call2, continuation) + return engine.call(call2, continuation, choice_point=False) expose_builtin(impl_or, ";", unwrap_spec=["callable", "callable"], handles_continuation=True) @@ -76,7 +74,8 @@ except error.UnificationFailed: engine.heap.revert(oldstate) raise - return engine.call(helper.ensure_callable(then_clause), continuation) + return engine.call(helper.ensure_callable(then_clause), continuation, + choice_point=False) expose_builtin(impl_if, "->", unwrap_spec=["callable", "raw"], handles_continuation=True) Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/exception.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/exception.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/exception.py Sat May 19 10:38:05 2007 @@ -31,7 +31,7 @@ raise error.UserError(exc_term) if isinstance(e, error.CatchableError): raise error.CatchableError(exc_term) - return engine.call(recover, continuation) + return engine.call(recover, continuation, choice_point=False) expose_builtin(impl_catch, "catch", unwrap_spec=["callable", "obj", "callable"], handles_continuation=True) Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/metacall.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/metacall.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/metacall.py Sat May 19 10:38:05 2007 @@ -9,13 +9,13 @@ try: return engine.call(call, continuation) except error.CutException, e: - return e.continuation.call(engine) + return e.continuation.call(engine, choice_point=False) expose_builtin(impl_call, "call", unwrap_spec=["callable"], handles_continuation=True) def impl_once(engine, clause, continuation): engine.call(clause) - return continuation.call(engine) + return continuation.call(engine, choice_point=False) expose_builtin(impl_once, "once", unwrap_spec=["callable"], handles_continuation=True) Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/parseraccess.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/parseraccess.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/parseraccess.py Sat May 19 10:38:05 2007 @@ -14,7 +14,7 @@ precedence.unify(term.Number(prec), engine.heap) typ.unify(term.Atom.newatom(form), engine.heap) name.unify(term.Atom(op), engine.heap) - return continuation.call(engine) + return continuation.call(engine, choice_point=True) except error.UnificationFailed: engine.heap.revert(oldstate) raise error.UnificationFailed() Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/register.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/register.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/register.py Sat May 19 10:38:05 2007 @@ -45,7 +45,7 @@ elif spec in ("concrete", "list"): code.append(" %s = query.args[%s].getvalue(engine.heap)" % (varname, i)) - if spec in ("callable", "int", "atom", "arithmetic", "list"): + if spec in ("int", "atom", "arithmetic", "list"): code.append( " if isinstance(%s, term.Var):" % (varname,)) code.append( @@ -77,7 +77,9 @@ call = " result = %s(%s)" % (func.func_name, ", ".join(subargs)) code.append(call) if not handles_continuation: - code.append(" return continuation.call(engine)") + code.append(" return continuation.call(engine, choice_point=False)") + else: + code.append(" return result") miniglobals = globals().copy() miniglobals[func.func_name] = func exec py.code.Source("\n".join(code)).compile() in miniglobals Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/termconstruction.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/termconstruction.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/termconstruction.py Sat May 19 10:38:05 2007 @@ -48,7 +48,7 @@ try: third.unify(arg, engine.heap) first.unify(term.Number(i + 1), engine.heap) - return continuation.call(engine) + return continuation.call(engine, choice_point=True) except error.UnificationFailed: engine.heap.revert(oldstate) raise error.UnificationFailed() @@ -64,7 +64,7 @@ third.unify(arg, engine.heap) else: error.throw_type_error("integer", first) - return continuation.call(engine) + return continuation.call(engine, choice_point=False) expose_builtin(impl_arg, "arg", unwrap_spec=["obj", "obj", "obj"], handles_continuation=True) Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py Sat May 19 10:38:05 2007 @@ -8,9 +8,22 @@ DEBUG = False +# bytecodes: +CALL = chr(0) +USER_CALL = chr(1) +TRY_RULE = chr(2) +CONTINUATION = chr(3) +DONE = chr(4) + + class Continuation(object): - def call(self, engine): - pass + def call(self, engine, choice_point=True): + if choice_point: + return engine.main_loop(CONTINUATION, None, self, None) + return (CONTINUATION, None, self, None) + + def _call(self, engine): + return (DONE, None, None, None) DONOTHING = Continuation() @@ -19,9 +32,9 @@ self.scope_active = True self.continuation = continuation - def call(self, engine): + def _call(self, engine): self.scope_active = False - return self.continuation.call(engine) + return self.continuation.call(engine, choice_point=False) START_NUMBER_OF_VARS = 4096 @@ -201,9 +214,9 @@ vars = query.get_max_var() + 1 self.heap.clear(vars) try: - return self.call(query, continuation) + return self.call(query, continuation, choice_point=True) except CutException, e: - self.continue_after_cut(e.continuation) + return self.continue_after_cut(e.continuation) def _build_and_run(self, tree): from pypy.lang.prolog.interpreter.parsing import TermBuilder @@ -219,35 +232,58 @@ from pypy.lang.prolog.interpreter.parsing import parse_file trees = parse_file(s, self.parser, Engine._build_and_run, self) - def call(self, query, continuation=DONOTHING): + def call(self, query, continuation=DONOTHING, choice_point=True, + inline=False): assert isinstance(query, Callable) - if DEBUG: - debug_print("calling", query) + if not choice_point: + return (CALL, query, continuation, None) + return self.main_loop(CALL, query, continuation) + + def _call(self, query, continuation): signature = query.signature # check for builtins - if _is_early_constant(signature): - for bsig, builtin in unrolling_builtins: - if signature == bsig: - #XXX should be: - #return builtin.call(self, query, continuation) - # but then the JIT explodes sometimes for funny reasons - return builtin.function(self, query, continuation) - # do a real call - return self.user_call(query, continuation) - else: - return self._opaque_call(signature, query, continuation) - - def _opaque_call(self, signature, query, continuation): +# if _is_early_constant(signature): +# for bsig, builtin in unrolling_builtins: +# if signature == bsig: +# #XXX should be: +# #return builtin.call(self, query, continuation) +# # but then the JIT explodes sometimes for funny reasons +# return builtin.function(self, query, continuation) +# # do a real call +# return self.user_call(query, continuation, tail_call=True) +# else: +# return self._opaque_call(signature, query, continuation) from pypy.lang.prolog.builtin import builtins #builtins = hint(builtins, deepfreeze=True) builtin = builtins.get(signature, None) if builtin is not None: - #XXX should be: - #return builtin.call(self, query, continuation) - # but then the JIT explodes sometimes for funny reasons - return builtin.function(self, query, continuation) + return builtin.call(self, query, continuation) # do a real call - return self.user_call(query, continuation) + return self.user_call(query, continuation, choice_point=False) + + def main_loop(self, where, query, continuation, rule=None): + from pypy.lang.prolog.builtin.control import AndContinuation + from pypy.lang.prolog.interpreter import helper + next = (DONE, None, None, None) + while 1: + #print " " * self.depth, where, query + if where == CALL: + next = self._call(query, continuation) + assert next is not None + elif where == TRY_RULE: + next = self._try_rule(rule, query, continuation) + assert next is not None + elif where == USER_CALL: + next = self._user_call(query, continuation) + assert next is not None + elif where == CONTINUATION: + next = continuation._call(self) + assert next is not None + elif where == DONE: + return next + else: + raise Exception("unknown bytecode") + where, query, continuation, rule = next def _jit_lookup(self, signature): signature2function = self.signature2function @@ -257,8 +293,12 @@ return function _jit_lookup._pure_function_ = True - def user_call(self, query, continuation): - #import pdb; pdb.set_trace() + def user_call(self, query, continuation, choice_point=True, inline=False): + if not choice_point: + return (USER_CALL, query, continuation, None) + return self.main_loop(USER_CALL, query, continuation) + + def _user_call(self, query, continuation): signature = hint(query.signature, promote=True) function = self._jit_lookup(signature) startrulechain = function.rulechain @@ -311,34 +351,30 @@ return self.try_rule(rule, query, continuation) except CutException, e: if continuation.scope_active: - self.continue_after_cut(e.continuation, continuation) + return self.continue_after_cut(e.continuation, continuation) raise - return self.try_rule(rule, query, continuation) + return self.try_rule(rule, query, continuation, choice_point=False) - def try_rule(self, rule, query, continuation=DONOTHING): - return self._portal_try_rule(rule, query, continuation) + def try_rule(self, rule, query, continuation=DONOTHING, choice_point=True, + inline=False): + if not choice_point: + return (TRY_RULE, query, continuation, rule) + return self.main_loop(TRY_RULE, query, continuation, rule) - def _portal_try_rule(self, rule, query, continuation=DONOTHING): - hint(None, global_merge_point=True) + def _try_rule(self, rule, query, continuation): rule = hint(rule, deepfreeze=True) hint(self, concrete=True) - if DEBUG: - debug_print("trying rule", rule, query, self.heap.vars[:self.heap.needed_vars]) # standardizing apart nextcall = rule.clone_and_unify_head(self.heap, query) - if DEBUG: - debug_print("worked", rule, query, self.heap.vars[:self.heap.needed_vars]) if nextcall is not None: - self.call(nextcall, continuation) - return "no" + return self.call(nextcall, continuation, choice_point=False) else: - continuation.call(self) - return "yes" + return continuation.call(self, choice_point=False) def continue_after_cut(self, continuation, lsc=None): while 1: try: - return continuation.call(self) + return continuation.call(self, choice_point=True) except CutException, e: if lsc is not None and not lsc.scope_active: raise Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/helper.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/helper.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/helper.py Sat May 19 10:38:05 2007 @@ -25,9 +25,12 @@ return isinstance(var, term.Callable) def ensure_callable(var): - if isinstance(var, term.Callable): + if isinstance(var, term.Var): + error.throw_instantiation_error() + elif isinstance(var, term.Callable): return var - error.throw_type_error("callable", var) + else: + error.throw_type_error("callable", var) def unwrap_int(obj): if isinstance(obj, term.Number): Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/interactive.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/interactive.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/interactive.py Sat May 19 10:38:05 2007 @@ -36,7 +36,7 @@ self.var_to_pos = var_to_pos self.write = write - def call(self, engine): + def _call(self, engine): self.write("yes\n") var_representation(self.var_to_pos, engine, self.write) while 1: Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/test/tool.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/test/tool.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/test/tool.py Sat May 19 10:38:05 2007 @@ -26,7 +26,7 @@ def __init__(self): self.heaps = [] - def call(self, engine): + def _call(self, engine): f = Heap() f.vars = engine.heap.vars[:] self.heaps.append(f) Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/translatedmain.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/translatedmain.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/translatedmain.py Sat May 19 10:38:05 2007 @@ -19,7 +19,7 @@ self.var_to_pos = var_to_pos self.write = write - def call(self, engine): + def _call(self, engine): self.write("yes\n") var_representation(self.var_to_pos, engine, self.write) while 1: From jlg at codespeak.net Sat May 19 13:52:51 2007 From: jlg at codespeak.net (jlg at codespeak.net) Date: Sat, 19 May 2007 13:52:51 +0200 (CEST) Subject: [pypy-svn] r43501 - in pypy/dist/pypy/lang/scheme: . test Message-ID: <20070519115251.033ED807E@code0.codespeak.net> Author: jlg Date: Sat May 19 13:52:51 2007 New Revision: 43501 Added: pypy/dist/pypy/lang/scheme/object.py pypy/dist/pypy/lang/scheme/ssparser.py pypy/dist/pypy/lang/scheme/test/test_parser.py Log: scheme parser with some tests Added: pypy/dist/pypy/lang/scheme/object.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/scheme/object.py Sat May 19 13:52:51 2007 @@ -0,0 +1,30 @@ + +class W_Root(object): + + def is_pair(self): + return False + + def to_string(self): + return '' + + def to_boolean(self): + return False + + def __str__(self): + return self.to_string() + +class W_Boolean(W_Root): + pass + +class W_False(W_Boolean): + + def to_string(self): + return "#f" + +class W_True(W_Boolean): + + def to_boolean(self): + return True + + def to_string(self): + return "#t" Added: pypy/dist/pypy/lang/scheme/ssparser.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/scheme/ssparser.py Sat May 19 13:52:51 2007 @@ -0,0 +1,29 @@ +import autopath +from pypy.rlib.parsing.ebnfparse import parse_ebnf, make_parse_function +from pypy.rlib.parsing.parsing import ParseError + +DEBUG = False + +grammar = r''' +STRING: "\"([^\\\"]|\\\"|\\\\)*\""; +IDENTIFIER: "[\+\-\*\^\?a-zA-Z0-9!<=>_~/$%&:]+"; +IGNORE: " |\n|\t|;[^\n]*"; +sexpr: ["("] sexpr* [")"] | | ; +''' +#IDENTIFIER: "[\+\-\*a-zA-Z0-9!<=>_~$%&/:?\^]+"; + +try: + regexs, rules, ToAST = parse_ebnf(grammar) +except ParseError, e: + #print e.nice_error_message() + raise + +parsef = make_parse_function(regexs, rules, eof=True) + +def parse(code): + t = parsef(code) + tree = t.visit(ToAST()) + if DEBUG: + ToAST().transform(t).view() + return tree + Added: pypy/dist/pypy/lang/scheme/test/test_parser.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/scheme/test/test_parser.py Sat May 19 13:52:51 2007 @@ -0,0 +1,36 @@ +from pypy.lang.scheme.ssparser import * +from pypy.rlib.parsing.parsing import Symbol, Nonterminal + +def test_simple_sexpr(): + #parse simple sexpr + t = parse(r'''(+ 1 2)''') + assert isinstance(t[0], Nonterminal) + assert len(t[0].children) == 3 + +def test_string(): + #parse string + t = parse(r'''"don't beleive \"them\""''') + assert isinstance(t[0], Symbol) + +def test_complex_sexpr(): + #parse more complex sexpr + t = parse(r''' + (define (fac n) ; comment + (if (< n 2) n + (* (fac (- n 1)) n))) + ''') + assert isinstance(t[0], Nonterminal) + assert len(t[0].children) == 3 + assert isinstance(t[0].children[0], Symbol) + assert isinstance(t[0].children[1], Nonterminal) + assert isinstance(t[0].children[2], Nonterminal) + +def test_ident_gen(): + ch_list = "+-*/azAZ09<=>-_~!$%&:?^" + for char in ch_list: + yield check_ident_ch, char + +def check_ident_ch(char): + t = parse("(" + char + ")") + assert isinstance(t[0], Nonterminal) + assert isinstance(t[0].children[0], Symbol) From jlg at codespeak.net Sat May 19 14:05:33 2007 From: jlg at codespeak.net (jlg at codespeak.net) Date: Sat, 19 May 2007 14:05:33 +0200 (CEST) Subject: [pypy-svn] r43502 - pypy/dist/pypy/lang/scheme Message-ID: <20070519120533.60828807E@code0.codespeak.net> Author: jlg Date: Sat May 19 14:05:32 2007 New Revision: 43502 Modified: pypy/dist/pypy/lang/scheme/ssparser.py Log: scheme parser with some tests Modified: pypy/dist/pypy/lang/scheme/ssparser.py ============================================================================== --- pypy/dist/pypy/lang/scheme/ssparser.py (original) +++ pypy/dist/pypy/lang/scheme/ssparser.py Sat May 19 14:05:32 2007 @@ -8,10 +8,9 @@ STRING: "\"([^\\\"]|\\\"|\\\\)*\""; IDENTIFIER: "[\+\-\*\^\?a-zA-Z0-9!<=>_~/$%&:]+"; IGNORE: " |\n|\t|;[^\n]*"; +progn: sexpr sexpr+ sexpr: ["("] sexpr* [")"] | | ; ''' -#IDENTIFIER: "[\+\-\*a-zA-Z0-9!<=>_~$%&/:?\^]+"; - try: regexs, rules, ToAST = parse_ebnf(grammar) except ParseError, e: From jlg at codespeak.net Sat May 19 14:36:38 2007 From: jlg at codespeak.net (jlg at codespeak.net) Date: Sat, 19 May 2007 14:36:38 +0200 (CEST) Subject: [pypy-svn] r43503 - pypy/dist/pypy/lang/scheme Message-ID: <20070519123638.5F2DB807E@code0.codespeak.net> Author: jlg Date: Sat May 19 14:36:37 2007 New Revision: 43503 Modified: pypy/dist/pypy/lang/scheme/ssparser.py Log: ssparser should work now Modified: pypy/dist/pypy/lang/scheme/ssparser.py ============================================================================== --- pypy/dist/pypy/lang/scheme/ssparser.py (original) +++ pypy/dist/pypy/lang/scheme/ssparser.py Sat May 19 14:36:37 2007 @@ -8,9 +8,9 @@ STRING: "\"([^\\\"]|\\\"|\\\\)*\""; IDENTIFIER: "[\+\-\*\^\?a-zA-Z0-9!<=>_~/$%&:]+"; IGNORE: " |\n|\t|;[^\n]*"; -progn: sexpr sexpr+ sexpr: ["("] sexpr* [")"] | | ; ''' + try: regexs, rules, ToAST = parse_ebnf(grammar) except ParseError, e: From cfbolz at codespeak.net Sat May 19 18:37:55 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 19 May 2007 18:37:55 +0200 (CEST) Subject: [pypy-svn] r43505 - in pypy/branch/prolog-jit-experiments/pypy/lang/prolog: builtin interpreter interpreter/test Message-ID: <20070519163755.70564808D@code0.codespeak.net> Author: cfbolz Date: Sat May 19 18:37:55 2007 New Revision: 43505 Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/register.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/portal.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/term.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/test/test_jit.py Log: add some more hints Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/register.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/register.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/builtin/register.py Sat May 19 18:37:55 2007 @@ -87,4 +87,7 @@ signature = "%s/%s" % (name, len(unwrap_spec)) b = Builtin(miniglobals[funcname]) builtins[signature] = b - builtins_list.append((signature, b)) + if signature == ",/2": + builtins_list.insert(0, (signature, b)) + else: + builtins_list.append((signature, b)) Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py Sat May 19 18:37:55 2007 @@ -4,6 +4,7 @@ CutException from pypy.lang.prolog.interpreter import error from pypy.rlib.objectmodel import hint, specialize, _is_early_constant +from pypy.rlib.objectmodel import we_are_jitted from pypy.rlib.unroll import unrolling_iterable DEBUG = False @@ -241,20 +242,20 @@ def _call(self, query, continuation): signature = query.signature - # check for builtins -# if _is_early_constant(signature): -# for bsig, builtin in unrolling_builtins: -# if signature == bsig: -# #XXX should be: -# #return builtin.call(self, query, continuation) -# # but then the JIT explodes sometimes for funny reasons -# return builtin.function(self, query, continuation) -# # do a real call -# return self.user_call(query, continuation, tail_call=True) -# else: -# return self._opaque_call(signature, query, continuation) from pypy.lang.prolog.builtin import builtins - #builtins = hint(builtins, deepfreeze=True) + builtins = hint(builtins, deepfreeze=True) + signature = hint(signature, promote=True) + for bsig, builtin in unrolling_builtins: + if signature == bsig: + #XXX should be: + #return builtin.call(self, query, continuation) + # but then the JIT explodes sometimes for funny reasons + return builtin.call(self, query, continuation) + return self.user_call(query, continuation, choice_point=False) + + def _opaque_call(self, query, continuation): + from pypy.lang.prolog.builtin import builtins + signature = query.signature builtin = builtins.get(signature, None) if builtin is not None: return builtin.call(self, query, continuation) @@ -265,25 +266,33 @@ from pypy.lang.prolog.builtin.control import AndContinuation from pypy.lang.prolog.interpreter import helper next = (DONE, None, None, None) + hint(where, concrete=True) + hint(rule, concrete=True) while 1: #print " " * self.depth, where, query if where == CALL: next = self._call(query, continuation) - assert next is not None elif where == TRY_RULE: + rule = hint(rule, promote=True) next = self._try_rule(rule, query, continuation) - assert next is not None elif where == USER_CALL: next = self._user_call(query, continuation) - assert next is not None elif where == CONTINUATION: + hint(continuation.__class__, promote=True) next = continuation._call(self) - assert next is not None elif where == DONE: return next else: raise Exception("unknown bytecode") where, query, continuation, rule = next + where = hint(where, promote=True) + + def _opaque_main_loop(self, where, query, continuation, rule=None): + return self.portal_main_loop(where, query, continuation, rule) + + def portal_main_loop(self, where, query, continuation, rule=None): + hint(None, global_merge_point=True) + return self.main_loop(where, query, continuation, rule) def _jit_lookup(self, signature): signature2function = self.signature2function @@ -359,7 +368,9 @@ inline=False): if not choice_point: return (TRY_RULE, query, continuation, rule) - return self.main_loop(TRY_RULE, query, continuation, rule) + if not we_are_jitted(): + return self.portal_main_loop(TRY_RULE, query, continuation, rule) + return self._opaque_main_loop(TRY_RULE, query, continuation, rule) def _try_rule(self, rule, query, continuation): rule = hint(rule, deepfreeze=True) Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/portal.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/portal.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/portal.py Sat May 19 18:37:55 2007 @@ -11,7 +11,7 @@ 'pypy.lang.prolog.builtin.register': True } -PORTAL = engine.Engine._portal_try_rule.im_func +PORTAL = engine.Engine.portal_main_loop.im_func class PyrologHintAnnotatorPolicy(HintAnnotatorPolicy): novirtualcontainer = True @@ -139,8 +139,17 @@ seegraph(pypy.lang.prolog.interpreter.engine.Heap.newvar) seegraph(pypy.lang.prolog.interpreter.term.Rule.clone_and_unify_head) seegraph(pypy.lang.prolog.interpreter.engine.Engine.call) + seegraph(pypy.lang.prolog.interpreter.engine.Engine._call) seegraph(pypy.lang.prolog.interpreter.engine.Engine.user_call) + seegraph(pypy.lang.prolog.interpreter.engine.Engine._user_call) + seegraph(pypy.lang.prolog.interpreter.engine.Engine.try_rule) + seegraph(pypy.lang.prolog.interpreter.engine.Engine._try_rule) + seegraph(pypy.lang.prolog.interpreter.engine.Engine.main_loop) seegraph(pypy.lang.prolog.interpreter.engine.LinkedRules.find_applicable_rule) + seegraph(pypy.lang.prolog.interpreter.engine.Continuation.call) + for cls in [engine.Continuation, engine.LimitedScopeContinuation, + pypy.lang.prolog.builtin.control.AndContinuation]: + seegraph(cls._call) return result_graphs def get_portal(drv): Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/term.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/term.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/term.py Sat May 19 18:37:55 2007 @@ -252,7 +252,7 @@ def copy_and_basic_unify(self, other, heap, memo): hint(self, concrete=True) - if isinstance(other, Atom) and (self is other or + if isinstance(other, Atom) and (hint(self is other, promote=True) or other.name == self.name): return self else: Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/test/test_jit.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/test/test_jit.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/test/test_jit.py Sat May 19 18:37:55 2007 @@ -38,11 +38,13 @@ res = self.timeshift_from_portal(main, portal.PORTAL, - [1], policy=POLICY) + [1], policy=POLICY, + backendoptimize=True) assert res == True res = self.timeshift_from_portal(main, portal.PORTAL, - [0], policy=POLICY) + [0], policy=POLICY, + backendoptimize=True) assert res == True def test_and(self): @@ -50,6 +52,7 @@ b(a). a(a). f(X) :- b(X), a(X). + f(X) :- fail. """) X = e.heap.newvar() @@ -65,7 +68,8 @@ assert res == True res = self.timeshift_from_portal(main, portal.PORTAL, - [0], policy=POLICY) + [0], policy=POLICY, + backendoptimize=True) assert res == True def test_user_call(self): @@ -90,6 +94,7 @@ res = self.timeshift_from_portal(main, portal.PORTAL, - [0], policy=POLICY) + [0], policy=POLICY, + backendoptimize=True) assert res == True From santagada at codespeak.net Sat May 19 19:01:57 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Sat, 19 May 2007 19:01:57 +0200 (CEST) Subject: [pypy-svn] r43506 - in pypy/dist/pypy/lang/js: . test Message-ID: <20070519170157.40168808D@code0.codespeak.net> Author: santagada Date: Sat May 19 19:01:56 2007 New Revision: 43506 Modified: pypy/dist/pypy/lang/js/astbuilder.py pypy/dist/pypy/lang/js/jsgrammar.txt pypy/dist/pypy/lang/js/jsparser.py pypy/dist/pypy/lang/js/operations.py pypy/dist/pypy/lang/js/test/test_interp.py pypy/dist/pypy/lang/js/test/test_parser.py Log: lots of changes, new operations, new way to call mathops, some grammar changes and some new tests. Modified: pypy/dist/pypy/lang/js/astbuilder.py ============================================================================== --- pypy/dist/pypy/lang/js/astbuilder.py (original) +++ pypy/dist/pypy/lang/js/astbuilder.py Sat May 19 19:01:56 2007 @@ -1,13 +1,42 @@ from pypy.rlib.parsing.tree import RPythonVisitor, Symbol from pypy.lang.js import operations +#this is a noop for now +def varfinder(opnode): + return [] + if isinstance(opnode, operations.Vars): + return [opnode,] + elif hasattr(opnode, "nodes"): + temp = [] + for op in opnode.nodes: + temp.extend(varfinder(op)) + return temp + elif hasattr(opnode, "body"): + return varfinder(opnode.body) + else: + return [] + +#this is a noop for now +def funcfinder(opnode): + return [] + if isinstance(opnode, operations.Function): + return [opnode,] + elif hasattr(opnode, "nodes"): + return [funcfinder(op) for op in opnode.nodes] + elif hasattr(opnode, "body"): + return funcfinder(opnode.body) + else: + return [] + + class ASTBuilder(RPythonVisitor): BINOP_TO_CLS = { '+': operations.Plus, '-': operations.Minus, '*': operations.Mult, - '/': operations.Div, + '/': operations.Division, '%': operations.Mod, + '.': operations.Member, } UNOP_TO_CLS = { '+': operations.UPlus, @@ -90,8 +119,7 @@ def visit_IDENTIFIERNAME(self, node): pos = self.get_pos(node) name = node.additional_info - initializer = operations.astundef #XXX this is uneded now - return operations.Identifier(pos, name, initializer) + return operations.Identifier(pos, name) def visit_program(self, node): pos = self.get_pos(node) @@ -100,11 +128,41 @@ def visit_sourceelements(self, node): pos = self.get_pos(node) - var_decl = None #XXX TODO - func_decl = None #XXX TODO nodes = [self.dispatch(child) for child in node.children] - return operations.Script(pos, var_decl, func_decl, nodes) + var_decl = [] + func_decl = [] + for node in nodes: + var_decl.extend(varfinder(node)) + func_decl.extend(funcfinder(node)) + + return operations.SourceElements(pos, var_decl, func_decl, nodes) def visit_expressionstatement(self, node): return self.dispatch(node.children[0]) - \ No newline at end of file + + def visit_variablestatement(self, node): + pos = self.get_pos(node) + body = self.dispatch(node.children[0]) + return operations.Variable(pos, body) + + def visit_variabledeclarationlist(self, node): + pos = self.get_pos(node) + + def visit_callexpression(self, node): + pos = self.get_pos(node) + left = self.dispatch(node.children[0]) + right = self.dispatch(node.children[1]) + return operations.Call(pos, left, right) + + def visit_argumentlist(self, node): + pos = self.get_pos(node) + nodes = [self.dispatch(child) for child in node.children] + return operations.ArgumentList(pos, nodes) + + def visit_assignmentexpression(self, node): + pos = self.get_pos(node) + left = self.dispatch(node.children[0]) + atype = node.children[1].additional_info + right = self.dispatch(node.children[2]) + return operations.Assignment(pos, left, right, atype) + \ No newline at end of file Modified: pypy/dist/pypy/lang/js/jsgrammar.txt ============================================================================== --- pypy/dist/pypy/lang/js/jsgrammar.txt (original) +++ pypy/dist/pypy/lang/js/jsgrammar.txt Sat May 19 19:01:56 2007 @@ -188,10 +188,10 @@ | ; -functiondeclaration : ["function"] identifier ["("] formalparameterlist? [")"] ["{"] >functionbody< ["}"] +functiondeclaration : ["function"] identifier ["("] formalparameterlist? [")"] ["{"] >functionbodyfunctionbody< ["}"] +functionexpression : ["function"] identifier? ["("] formalparameterlist? [")"] ["{"] >functionbodyformalparameterlist< @@ -201,15 +201,13 @@ functionbody : sourceelements ; -begmemberexpression : - | - ; - -memberexpression : begmemberexpression "[" expression "]" memberexpression* - | begmemberexpression "." identifier memberexpression* - | "new" memberexpression arguments - | - ; +memberexpression : "." identifier >memberexpression<* + | "[" expression "]" >memberexpression<* + | "new" memberexpression arguments + | >memberexpression<* + | >memberexpression<* + ; + newexpression : "new" newexpression | @@ -362,6 +360,6 @@ initialisernoin : "=" assignmentexpressionnoin ; -variabledeclarationlistnoin : variabledeclarationnoin "," variabledeclarationlistnoin +variabledeclarationlistnoin : variabledeclarationnoin [","] >variabledeclarationlistnoin< | variabledeclarationnoin ; Modified: pypy/dist/pypy/lang/js/jsparser.py ============================================================================== --- pypy/dist/pypy/lang/js/jsparser.py (original) +++ pypy/dist/pypy/lang/js/jsparser.py Sat May 19 19:01:56 2007 @@ -14,4 +14,4 @@ parsef = make_parse_function(regexs, rules, eof=True) def parse(code): - parsef(code).visit(ToAST()) + return parsef(code).visit(ToAST())[0] Modified: pypy/dist/pypy/lang/js/operations.py ============================================================================== --- pypy/dist/pypy/lang/js/operations.py (original) +++ pypy/dist/pypy/lang/js/operations.py Sat May 19 19:01:56 2007 @@ -46,7 +46,7 @@ raise NotImplementedError def __str__(self): - return ""%(self.opcode, self.value) + return "%s()"%(self.__class__) class Statement(Node): def __init__(self, pos): @@ -60,9 +60,9 @@ return self.eval(ctx) class ListOp(Expression): - def __init__(self, pos, nodelist): + def __init__(self, pos, nodes): self.pos = pos - self.nodelist = nodelist + self.nodes = nodes class UnaryOp(Expression): def __init__(self, pos, expr, postfix=False): @@ -100,30 +100,36 @@ class Array(ListOp): def eval(self, ctx): array = W_Array() - for i in range(len(self.nodelist)): - array.Put(str(i), self.nodelist[i].eval(ctx).GetValue()) + for i in range(len(self.nodes)): + array.Put(str(i), self.nodes[i].eval(ctx).GetValue()) return array -class Assign(BinaryOp): +class Assignment(Expression): + def __init__(self, pos, left, right, atype): + self.pos = pos + self.left = left + self.right = right + self.type = atype + def eval(self, ctx): v1 = self.left.eval(ctx) v3 = self.right.eval(ctx).GetValue() - op = self.value + op = self.type if op == "=": val = v3 - elif op == "*": + elif op == "*=": val = Mult().mathop(ctx, v1.GetValue(), v3) - elif op == "+": + elif op == "+=": val = Plus().mathop(ctx, v1.GetValue(), v3) - elif op == "/": + elif op == "/=": val = Div().mathop(ctx, v1.GetValue(), v3) - elif op == "%": + elif op == "%=": val = Mod().mathop(ctx, v1.GetValue(), v3) - elif op == "&": + elif op == "&=": val = BitwiseAnd().decision(ctx, v1.GetValue().ToInt32(), v3.ToInt32()) - elif op == "|": + elif op == "|=": val = BitwiseOR().decision(ctx, v1.GetValue().ToInt32(), v3.ToInt32()) - elif op == "^": + elif op == "^=": val = BitwiseXOR().decision(ctx, v1.GetValue().ToInt32(), v3.ToInt32()) else: print op @@ -131,11 +137,12 @@ v1.PutValue(val, ctx) return val + class Block(Statement): - def __init__(self, pos, t): - self.nodes = get_objects(t) - + def __init__(self, pos, nodes): + self.nodes = nodes + def execute(self, ctx): try: last = w_Undefined @@ -147,6 +154,7 @@ return e.value else: raise e + class BitwiseAnd(BinaryBitwiseOp): opcode = 'BITWISE_AND' @@ -193,9 +201,7 @@ def execute(self, ctx): raise ExecutionReturned('continue', None, None) -class Call(BinaryOp): - opcode = 'CALL' - +class Call(BinaryOp): def eval(self, ctx): r1 = self.left.eval(ctx) r2 = self.right.eval(ctx) @@ -217,6 +223,7 @@ return retval except ExecutionReturned, e: return e.value + class Comma(BinaryOp): opcode = 'COMMA' @@ -239,50 +246,38 @@ else: return self.falseop.eval(ctx).GetValue() -class Dot(BinaryOp): - opcode = 'DOT' - +class Member(BinaryOp): def eval(self, ctx): w_obj = self.left.eval(ctx).GetValue().ToObject() name = self.right.eval(ctx).GetPropertyName() return W_Reference(name, w_obj) + class Function(Expression): opcode = 'FUNCTION' - def __init__(self, pos, t): - self.name = get_string(t, 'name') - self.body = get_obj(t, 'body') - params = get_string(t, 'params') - if params == '': - self.params = [] - else: - self.params = params.split(',') + def __init__(self, pos, name, body, params): + self.name = name + self.body = body + self.params = params def eval(self, ctx): - # TODO: this is wrong, should clone the function prototype + #XXX this is wrong, should clone the function prototype w_obj = W_Object(ctx=ctx, callfunc = self) w_obj.Put('prototype', W_Object(ctx=ctx)) return w_obj class Identifier(Expression): - opcode = 'IDENTIFIER' - - def __init__(self, pos, name, initializer): + def __init__(self, pos, name): + self.pos = pos self.name = name - self.initializer = initializer - def __str__(self): - return ""%(str(self.name), str(self.initializer)) - def eval(self, ctx): - if not isinstance(self.initializer, Undefined): - ref = ctx.resolve_identifier(self.name) - ref.PutValue(self.initializer.eval(ctx).GetValue(), ctx) return ctx.resolve_identifier(self.name) def get_literal(self): return self.name + class This(Identifier): opcode = "THIS" @@ -335,64 +330,58 @@ return -1 class Or(BinaryOp): - def eval(self, ctx): s2 = self.left.eval(ctx).GetValue() if s2.ToBoolean(): return s2 s4 = self.right.eval(ctx).GetValue() return s4 + class And(BinaryOp): - opcode = 'AND' - def eval(self, ctx): s2 = self.left.eval(ctx).GetValue() if not s2.ToBoolean(): return s2 s4 = self.right.eval(ctx).GetValue() return s4 + class Ge(BinaryComparisonOp): - opcode = 'GE' - def decision(self, ctx, op1, op2): s5 = ARC(ctx, op1, op2) if s5 in (-1, 1): return W_Boolean(False) else: return W_Boolean(True) + class Gt(BinaryComparisonOp): - opcode = 'GT' - def decision(self, ctx, op1, op2): s5 = ARC(ctx, op2, op1) if s5 == -1: return W_Boolean(False) else: return W_Boolean(s5) + class Le(BinaryComparisonOp): - opcode = 'LE' - def decision(self, ctx, op1, op2): s5 = ARC(ctx, op2, op1) if s5 in (-1, 1): return W_Boolean(False) else: return W_Boolean(True) + class Lt(BinaryComparisonOp): - opcode = 'LT' - def decision(self, ctx, op1, op2): s5 = ARC(ctx, op1, op2) if s5 == -1: return W_Boolean(False) else: return W_Boolean(s5) - + ############################################################################## # @@ -617,11 +606,9 @@ name= self.right.eval(ctx).GetValue().ToString() return W_Reference(name, w_obj) -class List(ListOp): - opcode = 'LIST' - +class ArgumentList(ListOp): def eval(self, ctx): - return W_List([node.eval(ctx).GetValue() for node in self.list]) + return W_List([node.eval(ctx).GetValue() for node in self.nodes]) ############################################################################## @@ -637,47 +624,56 @@ result = self.mathop(ctx, nleft, nright) return result +def plus(ctx, nleft, nright): + if isinstance(nleft, W_String) or isinstance(nright, W_String): + sleft = nleft.ToString() + sright = nright.ToString() + return W_String(sleft + sright) + else: + fleft = nleft.ToNumber() + fright = nright.ToNumber() + return W_Number(fleft + fright) + +def mult(ctx, nleft, nright): + fleft = nleft.ToNumber() + fright = nright.ToNumber() + return W_Number(fleft * fright) + +def mod(ctx, nleft, nright): # XXX this one is really not following spec + ileft = nleft.ToInt32() + iright = nright.ToInt32() + return W_Number(ileft % iright) + +def division(ctx, nleft, nright): + fleft = nleft.ToNumber() + fright = nright.ToNumber() + return W_Number(fleft / fright) + +def minus(ctx, nleft, nright): + fleft = nleft.ToNumber() + fright = nright.ToNumber() + return W_Number(fleft - fright) + + class Plus(BinaryNumberOp): - @staticmethod - def mathop(ctx, nleft, nright): - if isinstance(nleft, W_String) or isinstance(nright, W_String): - sleft = nleft.ToString() - sright = nright.ToString() - return W_String(sleft + sright) - else: - fleft = nleft.ToNumber() - fright = nright.ToNumber() - return W_Number(fleft + fright) + mathop = staticmethod(plus) + class Mult(BinaryNumberOp): - def mathop(self, ctx, nleft, nright): - fleft = nleft.ToNumber() - fright = nright.ToNumber() - return W_Number(fleft * fright) + mathop = staticmethod(mult) + class Mod(BinaryNumberOp): - def mathop(self, ctx, nleft, nright): - fleft = nleft.ToInt32() - fright = nright.ToInt32() - return W_Number(fleft % fright) - -class Div(BinaryNumberOp): - opcode = 'DIV' - - def mathop(self, ctx, nleft, nright): - fleft = nleft.ToInt32() - fright = nright.ToInt32() - return W_Number(fleft / fright) - -class Minus(BinaryNumberOp): - opcode = 'MINUS' + mathop = staticmethod(mod) - def mathop(self, ctx, nleft, nright): - fleft = nleft.ToNumber() - fright = nright.ToNumber() - return W_Number(fleft - fright) +class Division(BinaryNumberOp): + mathop = staticmethod(division) + +class Minus(BinaryNumberOp): + mathop = staticmethod(minus) + class Null(Expression): opcode = 'NULL' @@ -768,7 +764,7 @@ def eval(self, ctx): w_obj = W_Object() - for prop in self.nodelist: + for prop in self.nodes: name = prop.left.name w_expr = prop.right.eval(ctx).GetValue() w_obj.Put(name, w_expr) @@ -780,9 +776,9 @@ # ############################################################################## -class Script(Statement): +class SourceElements(Statement): """ - Script nodes are found on each function declaration and in global code + SourceElements nodes are found on each function declaration and in global code """ def __init__(self, pos, var_decl, func_decl, nodes): self.var_decl = var_decl @@ -794,8 +790,6 @@ ctx.variable.Put(var.name, w_Undefined) for fun in self.func_decl: ctx.variable.Put(fun.name, fun.eval(ctx)) - - node = self try: @@ -808,7 +802,7 @@ raise else: # TODO: proper exception handling - print "exception in line: %s, on: %s"%(node.lineno, node.value) + print "exception in line: %s, on: %s"%(node.pos.lineno, node) raise class Program(Statement): @@ -817,7 +811,7 @@ self.body = body def execute(self, ctx): - return self.body.execute(self, ctx) + return self.body.execute(ctx) class Semicolon(Statement): opcode = 'SEMICOLON' @@ -830,6 +824,13 @@ return w_Undefined return self.expr.execute(ctx) +# class ExpressionStatement(Statement): +# def __init__(self, pos, exp): +# self.expr = expr +# +# def execute(self, ctx): +# return self.expr.eval(ctx) + class Return(Statement): opcode = 'RETURN' @@ -902,19 +903,36 @@ def execute(self, ctx): return None -class Vars(Statement): - opcode = 'VAR' - - def __init__(self, pos, t): - self.nodes = get_objects(t) +class VariableDeclaration(Node): + def __init__(self, pos, identifier, initializerexp): + self.pos = pos + self.identifier = identifier + self.initializerexp = initializerexp + + def eval(self, ctx): + if not isinstance(self.initializerexp, Undefined): + ref = self.identifier.eval() + ref.PutValue(self.initializerexp.eval(ctx).GetValue(), ctx) + return w_Undefined + - def execute(self, ctx): +class VariableDeclList(Expression): + def __init__(self, pos, nodes): + self.pos = pos + self.nodes = nodes + + def eval(self, ctx): for var in self.nodes: - var.execute(ctx) - return W_String(self.nodes[-1].get_literal()) + var.eval(ctx) + return w_Undefined + - def eval(self, ctx): - return self.execute(ctx) +class Variable(Statement): + def __init__(self, pos, body): + self.body = body + + def execute(self, ctx): + return self.body.eval() class Void(UnaryOp): opcode = 'VOID' @@ -1021,20 +1039,12 @@ continue class Boolean(Expression): - def __init__(self, pos, t): - if self.opcode == 'TRUE': - self.bool = True - else: - self.bool = False + def __init__(self, pos, boolval): + self.bool = boolval def eval(self, ctx): return W_Boolean(self.bool) - -class BTrue(Boolean): - pass - -class BFalse(Boolean): - pass + class Not(UnaryOp): def eval(self, ctx): @@ -1047,6 +1057,6 @@ class UPlus(UnaryOp): def eval(self, ctx): return W_Number(+self.expr.eval(ctx).GetValue().ToNumber()) - + astundef = Undefined(Position()) Modified: pypy/dist/pypy/lang/js/test/test_interp.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_interp.py (original) +++ pypy/dist/pypy/lang/js/test/test_interp.py Sat May 19 19:01:56 2007 @@ -15,18 +15,14 @@ py.test.skip("js binary not found") js_is_on_path() -py.test.skip("making the transition to the new parser") +py.test.skip("not ready yet") class TestInterp(object): def test_simple(self): - p = Plus() - n1 = Number() - n2 = Number() - n1.num = 2 - n2.num = 4 - p.left = n1 - p.right = n2 - assert p.eval(ExecutionContext()).GetValue().ToNumber() == 6 + n1 = Number(Position(), 2.0) + n2 = Number(Position(), 4.0) + p = Plus(Position(), n1, n2) + assert p.eval(ExecutionContext()).GetValue().ToNumber() == 6.0 l = [] interpreter.writer = l.append # Script([Semicolon(Call(Identifier('print', None), @@ -54,8 +50,8 @@ assert r.ToString() == result.ToString() def test_interp_parse(self): - self.assert_prints("print(1+1)", ["2"]) - self.assert_prints("print(1+2+3); print(1)", ["6", "1"]) + self.assert_prints("print(1+1);", ["2"]) + self.assert_prints("print(1+2+3); print(1);", ["6", "1"]) self.assert_prints("print(1,2,3);\n", ["1,2,3"]) def test_var_assign(self): @@ -63,7 +59,7 @@ self.assert_prints("x=3;y=4;print(x+y);", ["7"]) def test_minus(self): - self.assert_prints("print(2-1)", ["1"]) + self.assert_prints("print(2-1);", ["1"]) def test_string_var(self): self.assert_prints('print(\"sss\");', ["sss"]) @@ -90,7 +86,7 @@ def test_function_returns(self): self.assert_prints('x=function(){return 1;}; print(x()+x());', ["2"]) - self.assert_prints('function x() { return }', []) + self.assert_prints('function x() { return; };', []) def test_var_declaration(self): self.assert_prints('var x = 3; print(x);', ["3"]) @@ -167,13 +163,13 @@ """, [""]) def test_throw(self): - self.assert_prints("throw(3)", ["uncaught exception: 3"]) + self.assert_prints("throw(3);", ["uncaught exception: 3"]) def test_group(self): - self.assert_prints("print((2+1))", ["3"]) + self.assert_prints("print((2+1));", ["3"]) def test_comma(self): - self.assert_prints("print((500,3))", ["3"]) + self.assert_prints("print((500,3));", ["3"]) def test_try_catch(self): self.assert_prints(""" @@ -186,8 +182,8 @@ """, ["3"]) def test_block(self): - self.assert_result("{ 5}", W_Number(5)) - self.assert_result("{3; 5}", W_Number(5)) + self.assert_result("{ 5};", W_Number(5)) + self.assert_result("{3; 5};", W_Number(5)) def test_try_catch_finally(self): self.assert_prints(""" @@ -198,7 +194,7 @@ print(x); } finally { - print(5) + print(5); } """, ["3", "5"]) @@ -219,27 +215,27 @@ """, ["2"]) def test_compare(self): - self.assert_prints("print(1>0)",["true"]) - self.assert_prints("print(0>1)",["false"]) - self.assert_prints("print(0>0)",["false"]) - self.assert_prints("print(1<0)",["false"]) - self.assert_prints("print(0<1)",["true"]) - self.assert_prints("print(0<0)",["false"]) - self.assert_prints("print(1>=0)",["true"]) - self.assert_prints("print(1>=1)",["true"]) - self.assert_prints("print(1>=2)",["false"]) - self.assert_prints("print(0<=1)",["true"]) - self.assert_prints("print(1<=1)",["true"]) - self.assert_prints("print(1<=0)",["false"]) - self.assert_prints("print(0==0)",["true"]) - self.assert_prints("print(1==1)",["true"]) - self.assert_prints("print(0==1)",["false"]) - self.assert_prints("print(0!=1)",["true"]) - self.assert_prints("print(1!=1)",["false"]) + self.assert_prints("print(1>0);",["true"]) + self.assert_prints("print(0>1);",["false"]) + self.assert_prints("print(0>0);",["false"]) + self.assert_prints("print(1<0);",["false"]) + self.assert_prints("print(0<1);",["true"]) + self.assert_prints("print(0<0);",["false"]) + self.assert_prints("print(1>=0);",["true"]) + self.assert_prints("print(1>=1);",["true"]) + self.assert_prints("print(1>=2);",["false"]) + self.assert_prints("print(0<=1);",["true"]) + self.assert_prints("print(1<=1);",["true"]) + self.assert_prints("print(1<=0);",["false"]) + self.assert_prints("print(0==0);",["true"]) + self.assert_prints("print(1==1);",["true"]) + self.assert_prints("print(0==1);",["false"]) + self.assert_prints("print(0!=1);",["true"]) + self.assert_prints("print(1!=1);",["false"]) def test_binary_op(self): - self.assert_prints("print(0||0); print(1||0)",["0", "1"]) - self.assert_prints("print(0&&1); print(1&&1)",["0", "1"]) + self.assert_prints("print(0||0); print(1||0);",["0", "1"]) + self.assert_prints("print(0&&1); print(1&&1);",["0", "1"]) def test_while(self): self.assert_prints(""" @@ -264,7 +260,7 @@ print(z); } catch (e) { - print(e) + print(e); } """, ["ReferenceError: z is not defined"]) @@ -287,7 +283,7 @@ def test_vars(self): self.assert_prints(""" - var x;x=3; print(x)""", ["3"]) + var x;x=3; print(x);""", ["3"]) def test_minus(self): self.assert_prints(""" @@ -303,7 +299,7 @@ print(x); z = 2; """,""" - print(z) + print(z); """] ,["3", "2"]) @@ -324,7 +320,7 @@ def test_arrayobject(self): self.assert_prints("""var x = new Array(); - print(x.length == 0)""", ['true']) + print(x.length == 0);""", ['true']) def test_break(self): self.assert_prints(""" @@ -334,12 +330,12 @@ for(x=0;1==1;x++) { break; } - print('out')""", ["out"]) + print('out');""", ["out"]) def test_typeof(self): self.assert_result(""" var x = 3; - typeof x == 'number' + typeof x == 'number'; """, W_Boolean(True)) def test_semicolon(self): @@ -348,15 +344,15 @@ def test_newwithargs(self): self.assert_prints(""" var x = new Object(1,2,3,4); - print(x) + print(x); """, ["[object Object]"]) def test_increment(self): self.assert_prints(""" var x; - x = 1 - x++ - print(x)""", ["2"]) + x = 1; + x++; + print(x);""", ["2"]) def test_ternaryop(self): self.assert_prints([ @@ -368,53 +364,53 @@ self.assert_prints(""" var x = false; var y = true; - print(y) - print(x)""", ["true", "false"]) + print(y); + print(x);""", ["true", "false"]) def test_unarynot(self): self.assert_prints(""" var x = false; - print(!x) - print(!!x)""", ["true", "false"]) + print(!x); + print(!!x);""", ["true", "false"]) def test_equals(self): self.assert_prints(""" var x = 5; - y = z = x - print(y)""", ["5"]) + y = z = x; + print(y);""", ["5"]) def test_math_stuff(self): self.assert_prints(""" var x = 5; var z = 2; - print(x*z) - print(4/z) - print(isNaN(z)) - print(Math.abs(z-x)) - print(Number.NaN) - print(Number.POSITIVE_INFINITY) - print(Number.NEGATIVE_INFINITY) - print(Math.floor(3.2)) - print(null) - print(-z) + print(x*z); + print(4/z); + print(isNaN(z)); + print(Math.abs(z-x)); + print(Number.NaN); + print(Number.POSITIVE_INFINITY); + print(Number.NEGATIVE_INFINITY); + print(Math.floor(3.2)); + print(null); + print(-z); """, ['10', '2', 'false', '3', 'NaN', 'inf', '-inf', '3', '', '-2']) def test_globalproperties(self): self.assert_prints( """ - print(NaN) - print(Infinity) - print(undefined) + print(NaN); + print(Infinity); + print(undefined); """, ['NaN', 'inf', 'undefined']) def test_strangefunc(self): self.assert_prints("""function f1() { var z; var t;}""", []) - self.assert_prints(""" "'t'" """, []) + self.assert_prints(""" "'t'"; """, []) def test_null(self): - self.assert_result("null", w_Null) + self.assert_result("null;", w_Null) def test_void(self): - self.assert_prints("print(void print('hello'))", + self.assert_prints("print(void print('hello'));", ["hello", "undefined"]) def test_activationprob(self): @@ -424,17 +420,17 @@ return int1; } function x (v1){ - this.p1 = v1 - this.p2 = intern(this.p1) + this.p1 = v1; + this.p2 = intern(this.p1); } - var ins = new x(1) - print(ins.p1) - print(ins.p2) + var ins = new x(1); + print(ins.p1); + print(ins.p2); """, ['1','1', '1']) def test_array_acess(self): self.assert_prints(""" - var x = new Array() + var x = new Array(); x[0] = 1; x[x[0]] = 2; x[2] = x[0]+x[1]; @@ -447,73 +443,73 @@ self.assert_prints(""" var testcases = new Array(); var tc = testcases.length; - print('tc'+tc) + print('tc'+tc); """, ['tc0']) def test_mod_op(self): - self.assert_prints("print(2%2)", ['0']) + self.assert_prints("print(2%2);", ['0']) def test_unary_plus(self): - self.assert_prints("print(+1)", ['1']) + self.assert_prints("print(+1);", ['1']) def test_delete(self): self.assert_prints(""" - var x = {} + var x = {}; x.y = 1; - delete x.y - print(x.y) + delete x.y; + print(x.y); """, ['undefined']) def test_forin(self): self.assert_prints(""" - var x = {a:5} + var x = {a:5}; for(y in x){ - print(y) + print(y); } """, ['5',]) def test_stricteq(self): self.assert_prints(""" - print(2 === 2) - print(2 === 3) - print(2 !== 3) - print(2 !== 2) + print(2 === 2); + print(2 === 3); + print(2 !== 3); + print(2 !== 2); """, ['true', 'false', 'true', 'false']) def test_with(self): self.assert_prints(""" - var mock = {x:2} - var x=4 - print(x) + var mock = {x:2}; + var x=4; + print(x); try { with(mock) { - print(x) - throw 3 - print("not reacheable") + print(x); + throw 3; + print("not reacheable"); } } catch(y){ - print(y) + print(y); } - print(x) + print(x); """, ['4', '2', '3', '4']) def test_bitops(self): self.assert_prints(""" - print(2 ^ 2) - print(2 & 3) - print(2 | 3) + print(2 ^ 2); + print(2 & 3); + print(2 | 3); """, ['0', '2', '3']) def test_for_strange(self): self.assert_prints(""" - for (var arg = "", i = 0; i < 2; i++) { print(i)} + for (var arg = "", i = 0; i < 2; i++) { print(i);} """, ['0', '1']) def test_recursive_call(self): self.assert_prints(""" function fact(x) { if (x == 0) { return 1; } else { return fact(x-1)*x; }} - print(fact(3)) + print(fact(3)); """, ['6',]) def test_function_prototype(self): @@ -524,7 +520,8 @@ def test_function_this(self): self.assert_prints(""" - function foo() {print("debug");this.bar = function() {}}; + function foo() {print("debug");this.bar = function() {};}; var f = new foo(); f.bar(); """, ['debug',]) + Modified: pypy/dist/pypy/lang/js/test/test_parser.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_parser.py (original) +++ pypy/dist/pypy/lang/js/test/test_parser.py Sat May 19 19:01:56 2007 @@ -308,8 +308,8 @@ assert w_num.ToNumber() == 6 w_num = self.eval_expr('((((6))))') assert w_num.ToNumber() == 6 - w_array = self.eval_expr('[1,2,3]') - assert w_array.ToString() == '1,2,3' + #w_array = self.eval_expr('[1,2,3]') + #assert w_array.ToString() == '1,2,3' w_identifier = self.eval_expr('x') py.test.raises(ThrowException, w_identifier.GetValue) w_object = self.eval_expr('{x:1}') @@ -328,16 +328,3 @@ w_str = self.eval_expr('"hello "+\'world\'') assert w_str.ToString() == 'hello world' - -class TestToASTProgram(BaseGrammarTest): - def setup_class(cls): - cls.parse = parse_func() - - def to_ast(self, s): - ASTBuilder().dispatch(self.parse(s)) - - def test_simple(self): - self.to_ast("1;") - #self.to_ast("var x=1;") - #self.to_ast("print(1+1);") - From cfbolz at codespeak.net Sat May 19 19:10:22 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 19 May 2007 19:10:22 +0200 (CEST) Subject: [pypy-svn] r43507 - pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter Message-ID: <20070519171022.A1256808D@code0.codespeak.net> Author: cfbolz Date: Sat May 19 19:10:22 2007 New Revision: 43507 Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py Log: calculate unify_hash only once Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py Sat May 19 19:10:22 2007 @@ -126,7 +126,7 @@ curr = curr.next return first, copy - def find_applicable_rule(self, query): + def find_applicable_rule(self, uh2): #import pdb;pdb.set_trace() while self: uh = self.rule.unify_hash @@ -136,8 +136,8 @@ while j < len(uh): hint(j, concrete=True) hash1 = uh[j] - hash2 = query.unify_hash_of_child(j) - if hash1 != 0 and hash2 != 0 and hash1 != hash2: + hash2 = uh2[j] + if hash1 != 0 and hash2 != 0 and hash2 != hash1: break j += 1 else: @@ -316,7 +316,16 @@ error.throw_existence_error( "procedure", query.get_prolog_signature()) - rulechain = startrulechain.find_applicable_rule(query) + #XXX make a nice method + if isinstance(query, Term): + unify_hash = [] + i = 0 + while i < len(query.args): + unify_hash.append(query.unify_hash_of_child(i)) + i += 1 + else: + unify_hash = [] + rulechain = startrulechain.find_applicable_rule(unify_hash) if rulechain is None: # none of the rules apply raise UnificationFailed() @@ -325,7 +334,7 @@ rulechain = rulechain.next oldstate = self.heap.branch() while rulechain: - rulechain = rulechain.find_applicable_rule(query) + rulechain = rulechain.find_applicable_rule(unify_hash) if rulechain is None: self.heap.discard(oldstate) break @@ -366,8 +375,8 @@ def try_rule(self, rule, query, continuation=DONOTHING, choice_point=True, inline=False): - if not choice_point: - return (TRY_RULE, query, continuation, rule) + #if not choice_point and inline: + # return (TRY_RULE, query, continuation, rule) if not we_are_jitted(): return self.portal_main_loop(TRY_RULE, query, continuation, rule) return self._opaque_main_loop(TRY_RULE, query, continuation, rule) From santagada at codespeak.net Sat May 19 21:49:12 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Sat, 19 May 2007 21:49:12 +0200 (CEST) Subject: [pypy-svn] r43508 - in pypy/dist/pypy/lang/js: . test Message-ID: <20070519194912.3C8288090@code0.codespeak.net> Author: santagada Date: Sat May 19 21:49:11 2007 New Revision: 43508 Modified: pypy/dist/pypy/lang/js/jsgrammar.txt pypy/dist/pypy/lang/js/test/test_parser.py Log: tried to make the rule work, now it kind of works, but breaks rparsing... I need to talk to carl Modified: pypy/dist/pypy/lang/js/jsgrammar.txt ============================================================================== --- pypy/dist/pypy/lang/js/jsgrammar.txt (original) +++ pypy/dist/pypy/lang/js/jsgrammar.txt Sat May 19 21:49:11 2007 @@ -201,11 +201,13 @@ functionbody : sourceelements ; -memberexpression : "." identifier >memberexpression<* +memberexpression : primaryexpression >memberexpression< + | functionexpression >memberexpression< + | + | + | "." identifier >memberexpression<* | "[" expression "]" >memberexpression<* | "new" memberexpression arguments - | >memberexpression<* - | >memberexpression<* ; Modified: pypy/dist/pypy/lang/js/test/test_parser.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_parser.py (original) +++ pypy/dist/pypy/lang/js/test/test_parser.py Sat May 19 21:49:11 2007 @@ -308,8 +308,8 @@ assert w_num.ToNumber() == 6 w_num = self.eval_expr('((((6))))') assert w_num.ToNumber() == 6 - #w_array = self.eval_expr('[1,2,3]') - #assert w_array.ToString() == '1,2,3' + w_array = self.eval_expr('[1,2,3]') + assert w_array.ToString() == '1,2,3' w_identifier = self.eval_expr('x') py.test.raises(ThrowException, w_identifier.GetValue) w_object = self.eval_expr('{x:1}') From santagada at codespeak.net Sun May 20 17:57:32 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Sun, 20 May 2007 17:57:32 +0200 (CEST) Subject: [pypy-svn] r43511 - pypy/dist/pypy/lang/js/test Message-ID: <20070520155732.8854F8080@code0.codespeak.net> Author: santagada Date: Sun May 20 17:57:30 2007 New Revision: 43511 Added: pypy/dist/pypy/lang/js/test/test_astbuilder.py Log: new test suite, will contain the tests of the astbuilder Added: pypy/dist/pypy/lang/js/test/test_astbuilder.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/js/test/test_astbuilder.py Sun May 20 17:57:30 2007 @@ -0,0 +1,16 @@ +from pypy.lang.js.jsparser import parse +from pypy.lang.js.astbuilder import ASTBuilder +from pypy.lang.js import operations + +def to_ast(s): + ASTBuilder().dispatch(parse(s)) + +def test_simple(): + yield to_ast, "1;" + yield to_ast, "var x=1;" + yield to_ast, "print(1);" + #yield to_ast, "" + +def test_funcvarfinder(): + pos = operations.Position() + \ No newline at end of file From santagada at codespeak.net Sun May 20 18:01:45 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Sun, 20 May 2007 18:01:45 +0200 (CEST) Subject: [pypy-svn] r43512 - pypy/dist/pypy/lang/js/test Message-ID: <20070520160145.0256B8080@code0.codespeak.net> Author: santagada Date: Sun May 20 18:01:45 2007 New Revision: 43512 Modified: pypy/dist/pypy/lang/js/test/test_astbuilder.py Log: comment a failing test... Modified: pypy/dist/pypy/lang/js/test/test_astbuilder.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_astbuilder.py (original) +++ pypy/dist/pypy/lang/js/test/test_astbuilder.py Sun May 20 18:01:45 2007 @@ -8,7 +8,7 @@ def test_simple(): yield to_ast, "1;" yield to_ast, "var x=1;" - yield to_ast, "print(1);" + #yield to_ast, "print(1);" #yield to_ast, "" def test_funcvarfinder(): From cfbolz at codespeak.net Sun May 20 18:06:36 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 20 May 2007 18:06:36 +0200 (CEST) Subject: [pypy-svn] r43513 - in pypy/dist/pypy/lang/js: . test Message-ID: <20070520160636.C01B28080@code0.codespeak.net> Author: cfbolz Date: Sun May 20 18:06:36 2007 New Revision: 43513 Modified: pypy/dist/pypy/lang/js/jsgrammar.txt pypy/dist/pypy/lang/js/test/test_astbuilder.py Log: remove tabs, a newline at end of file Modified: pypy/dist/pypy/lang/js/jsgrammar.txt ============================================================================== --- pypy/dist/pypy/lang/js/jsgrammar.txt (original) +++ pypy/dist/pypy/lang/js/jsgrammar.txt Sun May 20 18:06:36 2007 @@ -201,15 +201,15 @@ functionbody : sourceelements ; -memberexpression : primaryexpression >memberexpression< - | functionexpression >memberexpression< - | - | - | "." identifier >memberexpression<* - | "[" expression "]" >memberexpression<* - | "new" memberexpression arguments - ; - +memberexpression: primaryexpression >memberexpression< + | functionexpression >memberexpression< + | + | + | "." identifier >memberexpression<* + | "[" expression "]" >memberexpression<* + | "new" memberexpression arguments + ; + newexpression : "new" newexpression | Modified: pypy/dist/pypy/lang/js/test/test_astbuilder.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_astbuilder.py (original) +++ pypy/dist/pypy/lang/js/test/test_astbuilder.py Sun May 20 18:06:36 2007 @@ -13,4 +13,4 @@ def test_funcvarfinder(): pos = operations.Position() - \ No newline at end of file + From cfbolz at codespeak.net Sun May 20 18:25:55 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 20 May 2007 18:25:55 +0200 (CEST) Subject: [pypy-svn] r43514 - pypy/dist/pypy/rlib/parsing/test Message-ID: <20070520162555.A710D8080@code0.codespeak.net> Author: cfbolz Date: Sun May 20 18:25:55 2007 New Revision: 43514 Modified: pypy/dist/pypy/rlib/parsing/test/test_ebnfparse.py Log: test that shows the problem leonardo found. Modified: pypy/dist/pypy/rlib/parsing/test/test_ebnfparse.py ============================================================================== --- pypy/dist/pypy/rlib/parsing/test/test_ebnfparse.py (original) +++ pypy/dist/pypy/rlib/parsing/test/test_ebnfparse.py Sun May 20 18:25:55 2007 @@ -436,3 +436,16 @@ t = parse("1 2 3 4 5") t = ToAST().transform(t) +def test_bug(): + # this could be seen as using the transformer in the wrong way + # but I have no clue how to detect this situation + py.test.skip("fix me somehow") + regexs, rules, ToAST = parse_ebnf(""" +IGNORE: " "; +DECIMAL: "0|[1-9][0-9]*"; +list: DECIMAL >list< | ; + """) + parse = make_parse_function(regexs, rules) + t = parse("1 2 3 4 5") + t = ToAST().transform(t) + From cfbolz at codespeak.net Sun May 20 18:26:24 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 20 May 2007 18:26:24 +0200 (CEST) Subject: [pypy-svn] r43515 - pypy/dist/pypy/lang/js Message-ID: <20070520162624.B7E688080@code0.codespeak.net> Author: cfbolz Date: Sun May 20 18:26:24 2007 New Revision: 43515 Modified: pypy/dist/pypy/lang/js/jsgrammar.txt Log: Fix the transformation annotations. I think this is what you want and it gives pretty nice trees. Modified: pypy/dist/pypy/lang/js/jsgrammar.txt ============================================================================== --- pypy/dist/pypy/lang/js/jsgrammar.txt (original) +++ pypy/dist/pypy/lang/js/jsgrammar.txt Sun May 20 18:26:24 2007 @@ -201,12 +201,12 @@ functionbody : sourceelements ; -memberexpression: primaryexpression >memberexpression< - | functionexpression >memberexpression< +memberexpression: primaryexpression memberexpression + | functionexpression memberexpression | | - | "." identifier >memberexpression<* - | "[" expression "]" >memberexpression<* + | "." identifier memberexpression* + | "[" expression "]" memberexpression* | "new" memberexpression arguments ; From cfbolz at codespeak.net Mon May 21 00:48:34 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 21 May 2007 00:48:34 +0200 (CEST) Subject: [pypy-svn] r43519 - pypy/dist/pypy/jit/timeshifter/test Message-ID: <20070520224834.8EC518084@code0.codespeak.net> Author: cfbolz Date: Mon May 21 00:48:33 2007 New Revision: 43519 Modified: pypy/dist/pypy/jit/timeshifter/test/test_portal.py Log: hm, it seems recursive portal calls work if you make sure yourself that the colors of the arguments are correct. thanks Armin. Modified: pypy/dist/pypy/jit/timeshifter/test/test_portal.py ============================================================================== --- pypy/dist/pypy/jit/timeshifter/test/test_portal.py (original) +++ pypy/dist/pypy/jit/timeshifter/test/test_portal.py Mon May 21 00:48:33 2007 @@ -558,10 +558,11 @@ res = self.timeshift_from_portal(f, g, [42], policy=P_NOVIRTUAL) def test_recursive_portal_call(self): - py.test.skip("recursive portal calls don't work right now") def indirection(green, red): - return portal((green + red) % 100, red + 1) + newgreen = hint((green + red) % 100, promote=True) + return portal(newgreen, red + 1) def portal(green, red): + hint(None, global_merge_point=True) green = abs(green) red = abs(red) hint(green, concrete=True) From santagada at codespeak.net Mon May 21 05:54:28 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Mon, 21 May 2007 05:54:28 +0200 (CEST) Subject: [pypy-svn] r43520 - in pypy/dist/pypy/lang/js: . test Message-ID: <20070521035428.B5709807E@code0.codespeak.net> Author: santagada Date: Mon May 21 05:54:27 2007 New Revision: 43520 Modified: pypy/dist/pypy/lang/js/astbuilder.py pypy/dist/pypy/lang/js/jsgrammar.txt pypy/dist/pypy/lang/js/jsobj.py pypy/dist/pypy/lang/js/operations.py pypy/dist/pypy/lang/js/test/test_astbuilder.py pypy/dist/pypy/lang/js/test/test_interp.py Log: now things are mostly working with call and member expression except for a new without arguments, eg. "new Array();" tomorrow the fixing will continue Modified: pypy/dist/pypy/lang/js/astbuilder.py ============================================================================== --- pypy/dist/pypy/lang/js/astbuilder.py (original) +++ pypy/dist/pypy/lang/js/astbuilder.py Mon May 21 05:54:27 2007 @@ -28,7 +28,6 @@ else: return [] - class ASTBuilder(RPythonVisitor): BINOP_TO_CLS = { '+': operations.Plus, @@ -36,7 +35,15 @@ '*': operations.Mult, '/': operations.Division, '%': operations.Mod, + '^': operations.BitwiseXor, + '|': operations.BitwiseOr, + '&': operations.BitwiseAnd, + '&&': operations.And, + '||': operations.Or, + '==': operations.Eq, + '!=': operations.Ne, '.': operations.Member, + '[': operations.Member, } UNOP_TO_CLS = { '+': operations.UPlus, @@ -48,7 +55,7 @@ '[': operations.Array, '{': operations.ObjectInit, } - + def get_pos(self, node): value = '' source_pos = None @@ -80,7 +87,6 @@ def string(self,node): pos = self.get_pos(node) return operations.String(pos, node.additional_info) - visit_DOUBLESTRING = string visit_SINGLESTRING = string @@ -91,10 +97,31 @@ pos = self.get_pos(op) right = self.dispatch(node.children[i * 2 + 2]) result = self.BINOP_TO_CLS[op.additional_info](pos, left, right) + print left, right left = result return left visit_additiveexpression = binaryop visit_multiplicativeexpression = binaryop + visit_memberexpression = binaryop + visit_bitwisexorexpression = binaryop + visit_bitwiseandexpression = binaryop + visit_bitwiseorexpression = binaryop + visit_equalityexpression = binaryop + visit_logicalorexpression = binaryop + visit_logicalandexpression = binaryop + + + def literalop(self, node): + pos = self.get_pos(node); + value = node.children[0].additional_info + if value == "true": + return operations.Boolean(pos, True) + elif value == "false": + return operations.Boolean(pos, False) + else: + return operations.Null(pos) + visit_nullliteral = literalop + visit_booleanliteral = literalop def visit_unaryexpression(self, node): op = node.children[0] @@ -107,7 +134,7 @@ pos = self.get_pos(op) l = [self.dispatch(child) for child in node.children[1:]] return self.LISTOP_TO_CLS[op.additional_info](pos, l) - visit_arrayliteral = listop + visit_arrayliteral = listop # XXX elision visit_objectliteral = listop def visit_propertynameandvalue(self, node): @@ -147,6 +174,17 @@ def visit_variabledeclarationlist(self, node): pos = self.get_pos(node) + nodes = [self.dispatch(child) for child in node.children] + return operations.VariableDeclList(pos, nodes) + + def visit_variabledeclaration(self, node): + pos = self.get_pos(node) + identifier = self.dispatch(node.children[0]) + if len(node.children) > 1: + expr = self.dispatch(node.children[1]) + else: + expr = None + return operations.VariableDeclaration(pos, identifier, expr) def visit_callexpression(self, node): pos = self.get_pos(node) @@ -154,9 +192,9 @@ right = self.dispatch(node.children[1]) return operations.Call(pos, left, right) - def visit_argumentlist(self, node): + def visit_arguments(self, node): pos = self.get_pos(node) - nodes = [self.dispatch(child) for child in node.children] + nodes = [self.dispatch(child) for child in node.children[1:]] return operations.ArgumentList(pos, nodes) def visit_assignmentexpression(self, node): @@ -165,4 +203,15 @@ atype = node.children[1].additional_info right = self.dispatch(node.children[2]) return operations.Assignment(pos, left, right, atype) - \ No newline at end of file + + def visit_functiondeclaration(self, node): + pos = self.get_pos(node) + + def visit_throwstatement(self, node): + pos = self.get_pos(node) + exp = self.dispatch(node.children[0]) + return operations.Throw(pos, exp) + + def visit_emptystatement(self, node): + return operations.astundef + Modified: pypy/dist/pypy/lang/js/jsgrammar.txt ============================================================================== --- pypy/dist/pypy/lang/js/jsgrammar.txt (original) +++ pypy/dist/pypy/lang/js/jsgrammar.txt Mon May 21 05:54:27 2007 @@ -160,10 +160,11 @@ | | | - | "\(" "\)" + | "(" ")" ; -arrayliteral : "[" >elementlist< ["]"] +arrayliteral : "[" [elision]? ["]"] # XXX should be treating elisions + | "[" >elementlist< ["]"] | "[" >elementlist< ["," "]"] ; @@ -188,43 +189,46 @@ | ; -functiondeclaration : ["function"] identifier ["("] formalparameterlist? [")"] ["{"] >functionbodyfunctionbodyformalparameterlist< | identifier ; -functionbody : sourceelements +functionbody : ; -memberexpression: primaryexpression memberexpression - | functionexpression memberexpression - | - | - | "." identifier memberexpression* - | "[" expression "]" memberexpression* - | "new" memberexpression arguments - ; +memberexpression : primaryexpression >memberexpressiona<+ + | functionexpression >memberexpressiona<+ + | "new" memberexpression arguments + | + | + ; +memberexpressiona : "." identifier + | "[" expression ["]"] + ; newexpression : "new" newexpression | ; -callexpression : memberexpression arguments arguments callexpression* - | memberexpression arguments "[" expression "]" callexpression* - | memberexpression arguments "." identifier callexpression* - | memberexpression arguments +callexpression : memberexpression arguments >callexpressiona<* ; -arguments : ["("] [")"] - | "(" ")" +callexpressiona : arguments + | "[" expression ["]"] + | "." identifier + ; + +arguments : "(" [")"] + | "(" >argumentlist< [")"] ; - + argumentlist : assignmentexpression [","] >argumentlist< | assignmentexpression; @@ -365,3 +369,4 @@ variabledeclarationlistnoin : variabledeclarationnoin [","] >variabledeclarationlistnoin< | variabledeclarationnoin ; + Modified: pypy/dist/pypy/lang/js/jsobj.py ============================================================================== --- pypy/dist/pypy/lang/js/jsobj.py (original) +++ pypy/dist/pypy/lang/js/jsobj.py Mon May 21 05:54:27 2007 @@ -378,6 +378,8 @@ def type(self): return 'string' + def GetPropertyName(self): + return self.ToString() class W_Number(W_Primitive): def __init__(self, floatval): @@ -428,6 +430,9 @@ return r_uint(self.floatval) + def GetPropertyName(self): + return self.ToString() + class W_List(W_Root): def __init__(self, list_w): self.list_w = list_w Modified: pypy/dist/pypy/lang/js/operations.py ============================================================================== --- pypy/dist/pypy/lang/js/operations.py (original) +++ pypy/dist/pypy/lang/js/operations.py Mon May 21 05:54:27 2007 @@ -170,15 +170,13 @@ return W_Number(~op1) -class BitwiseOR(BinaryBitwiseOp): +class BitwiseOr(BinaryBitwiseOp): opcode = 'BITWISE_OR' def decision(self, ctx, op1, op2): return W_Number(op1|op2) -class BitwiseXOR(BinaryBitwiseOp): - opcode = 'BITWISE_XOR' - +class BitwiseXor(BinaryBitwiseOp): def decision(self, ctx, op1, op2): return W_Number(op1^op2) @@ -251,33 +249,44 @@ w_obj = self.left.eval(ctx).GetValue().ToObject() name = self.right.eval(ctx).GetPropertyName() return W_Reference(name, w_obj) - - -class Function(Expression): - opcode = 'FUNCTION' +class FunctionStatement(Statement): def __init__(self, pos, name, body, params): self.name = name self.body = body self.params = params + + def execute(self, ctx): + pass + + def eval(self, ctx): + #XXX this is wrong, should clone the function prototype + w_obj = W_Object(ctx=ctx, callfunc = self) + w_obj.Put('prototype', W_Object(ctx=ctx)) + return w_obj + +class FunctionExpression(Expression): + def __init__(self, pos, name, body, params): + self.name = name + self.body = body + self.params = params + def eval(self, ctx): #XXX this is wrong, should clone the function prototype w_obj = W_Object(ctx=ctx, callfunc = self) w_obj.Put('prototype', W_Object(ctx=ctx)) return w_obj + class Identifier(Expression): def __init__(self, pos, name): self.pos = pos self.name = name - + def eval(self, ctx): return ctx.resolve_identifier(self.name) - def get_literal(self): - return self.name - class This(Identifier): opcode = "THIS" @@ -675,12 +684,7 @@ mathop = staticmethod(minus) -class Null(Expression): - opcode = 'NULL' - - def __init__(self, pos, t): - pass - +class Null(Expression): def eval(self, ctx): return w_Null @@ -720,22 +724,21 @@ return W_Number(self.num) class String(Expression): - opcode = 'STRING' - def __init__(self, pos, strval): + self.pos = pos self.strval = self.string_unquote(strval) - + def eval(self, ctx): return W_String(self.strval) def get_literal(self): return W_String(self.strval).ToString() - + def string_unquote(self, string): temp = [] stop = len(string)-1 last = "" - + #removing the begining quotes (" or \') if string.startswith('"'): singlequote = False @@ -746,7 +749,7 @@ print stop raise JsSyntaxError() internalstring = string[1:stop] - + for c in internalstring: if last == SLASH: unescapeseq = unescapedict[last+c] @@ -757,7 +760,7 @@ temp.append(c) last = c return ''.join(temp) - + class ObjectInit(ListOp): opcode = 'OBJECT_INIT' @@ -765,17 +768,11 @@ def eval(self, ctx): w_obj = W_Object() for prop in self.nodes: - name = prop.left.name + name = prop.left.eval(ctx).GetPropertyName() w_expr = prop.right.eval(ctx).GetValue() w_obj.Put(name, w_expr) return w_obj -############################################################################## -# -# Script and semicolon, the most important part of the interpreter probably -# -############################################################################## - class SourceElements(Statement): """ SourceElements nodes are found on each function declaration and in global code @@ -844,13 +841,12 @@ raise ExecutionReturned('return', self.expr.eval(ctx), None) class Throw(Statement): - opcode = 'THROW' + def __init__(self, pos, exp): + self.pos = pos + self.exp = exp - def __init__(self, pos, t): - self.exception = get_obj(t, 'exception') - def execute(self, ctx): - raise ThrowException(self.exception.eval(ctx)) + raise ThrowException(self.exp.eval(ctx).GetValue()) class Try(Statement): opcode = 'TRY' @@ -903,17 +899,17 @@ def execute(self, ctx): return None -class VariableDeclaration(Node): - def __init__(self, pos, identifier, initializerexp): +class VariableDeclaration(Expression): + def __init__(self, pos, identifier, expr=None): self.pos = pos self.identifier = identifier - self.initializerexp = initializerexp + self.expr = expr def eval(self, ctx): - if not isinstance(self.initializerexp, Undefined): - ref = self.identifier.eval() - ref.PutValue(self.initializerexp.eval(ctx).GetValue(), ctx) - return w_Undefined + # XXX take a look at the specs + v = self.identifier.eval(ctx) + if self.expr is not None: + v.PutValue(self.expr.eval(ctx).GetValue(), ctx) class VariableDeclList(Expression): @@ -926,13 +922,13 @@ var.eval(ctx) return w_Undefined - class Variable(Statement): def __init__(self, pos, body): + self.pos = pos self.body = body def execute(self, ctx): - return self.body.eval() + return self.body.eval(ctx) class Void(UnaryOp): opcode = 'VOID' Modified: pypy/dist/pypy/lang/js/test/test_astbuilder.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_astbuilder.py (original) +++ pypy/dist/pypy/lang/js/test/test_astbuilder.py Mon May 21 05:54:27 2007 @@ -8,7 +8,12 @@ def test_simple(): yield to_ast, "1;" yield to_ast, "var x=1;" - #yield to_ast, "print(1);" + yield to_ast, "print(1);" + yield to_ast, "x.y;" + yield to_ast, "x[1];" + yield to_ast, "true;" + yield to_ast, "false;" + yield to_ast, "null;" #yield to_ast, "" def test_funcvarfinder(): Modified: pypy/dist/pypy/lang/js/test/test_interp.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_interp.py (original) +++ pypy/dist/pypy/lang/js/test/test_interp.py Mon May 21 05:54:27 2007 @@ -15,7 +15,6 @@ py.test.skip("js binary not found") js_is_on_path() -py.test.skip("not ready yet") class TestInterp(object): def test_simple(self): @@ -82,9 +81,11 @@ self.assert_prints('x={d:"x"}; print(x["d"]);', ["x"]) def test_function_prints(self): + py.test.skip("not ready yet") self.assert_prints('x=function(){print(3);}; x();', ["3"]) def test_function_returns(self): + py.test.skip("not ready yet") self.assert_prints('x=function(){return 1;}; print(x()+x());', ["2"]) self.assert_prints('function x() { return; };', []) @@ -93,6 +94,7 @@ self.assert_prints('var x = 3; print(x+x);', ["6"]) def test_var_scoping(self): + py.test.skip("not ready yet") self.assert_prints(""" var y; var p; @@ -107,6 +109,7 @@ """, ["5,3,0"]) def test_function_args(self): + py.test.skip("not ready yet") self.assert_prints(""" x = function (t,r) { return t+r; @@ -115,6 +118,7 @@ """, ["5"]) def test_function_less_args(self): + py.test.skip("not ready yet") self.assert_prints(""" x = function (t, r) { return t + r; @@ -123,6 +127,7 @@ """, ["NaN"]) def test_function_more_args(self): + py.test.skip("not ready yet") self.assert_prints(""" x = function (t, r) { return t + r; @@ -131,6 +136,7 @@ """, ["5"]) def test_function_has_var(self): + py.test.skip("not ready yet") self.assert_prints(""" x = function () { var t = 'test'; @@ -140,6 +146,7 @@ """, ["test"]) def test_function_arguments(self): + py.test.skip("not ready yet") self.assert_prints(""" x = function () { r = arguments[0]; @@ -169,9 +176,11 @@ self.assert_prints("print((2+1));", ["3"]) def test_comma(self): + py.test.skip("not ready yet") self.assert_prints("print((500,3));", ["3"]) def test_try_catch(self): + py.test.skip("not ready yet") self.assert_prints(""" try { throw(3); @@ -182,10 +191,12 @@ """, ["3"]) def test_block(self): + py.test.skip("not ready yet") self.assert_result("{ 5};", W_Number(5)) self.assert_result("{3; 5};", W_Number(5)) def test_try_catch_finally(self): + py.test.skip("not ready yet") self.assert_prints(""" try { throw(3); @@ -199,6 +210,7 @@ """, ["3", "5"]) def test_if_then(self): + py.test.skip("not ready yet") self.assert_prints(""" if (1) { print(1); @@ -206,6 +218,7 @@ """, ["1"]) def test_if_then_else(self): + py.test.skip("not ready yet") self.assert_prints(""" if (0) { print(1); @@ -215,6 +228,7 @@ """, ["2"]) def test_compare(self): + py.test.skip("not ready yet") self.assert_prints("print(1>0);",["true"]) self.assert_prints("print(0>1);",["false"]) self.assert_prints("print(0>0);",["false"]) @@ -238,6 +252,7 @@ self.assert_prints("print(0&&1); print(1&&1);",["0", "1"]) def test_while(self): + py.test.skip("not ready yet") self.assert_prints(""" i = 0; while (i<3) { @@ -248,12 +263,14 @@ """, ["0","1","2","3"]) def test_object_creation(self): + py.test.skip("not ready yet") self.assert_prints(""" o = new Object(); print(o); """, ["[object Object]"]) def test_var_decl(self): + py.test.skip("not ready yet") self.assert_prints("print(x); var x;", ["undefined"]) self.assert_prints(""" try { @@ -265,6 +282,7 @@ """, ["ReferenceError: z is not defined"]) def test_function_name(self): + py.test.skip("not ready yet") self.assert_prints(""" function x() { print("my name is x"); @@ -273,6 +291,7 @@ """, ["my name is x"]) def test_new_with_function(self): + py.test.skip("not ready yet") c= """ x = function() {this.info = 'hello';}; o = new x(); @@ -286,6 +305,7 @@ var x;x=3; print(x);""", ["3"]) def test_minus(self): + py.test.skip("not ready yet") self.assert_prints(""" x = {y:3}; print("y" in x); @@ -304,6 +324,7 @@ ,["3", "2"]) def test_for(self): + py.test.skip("not ready yet") self.assert_prints(""" for (i=0; i<3; i++) { print(i); @@ -312,6 +333,7 @@ """, ["0","1","2","3"]) def test_eval(self): + py.test.skip("not ready yet") self.assert_prints(""" var x = 2; eval('x=x+1; print(x); z=2'); @@ -319,10 +341,12 @@ """, ["3","2"]) def test_arrayobject(self): + py.test.skip("not ready yet") self.assert_prints("""var x = new Array(); print(x.length == 0);""", ['true']) def test_break(self): + py.test.skip("not ready yet") self.assert_prints(""" while(1){ break; @@ -333,21 +357,25 @@ print('out');""", ["out"]) def test_typeof(self): + py.test.skip("not ready yet") self.assert_result(""" var x = 3; typeof x == 'number'; """, W_Boolean(True)) def test_semicolon(self): + py.test.skip("not ready yet") self.assert_prints(';', []) def test_newwithargs(self): + py.test.skip("not ready yet") self.assert_prints(""" var x = new Object(1,2,3,4); print(x); """, ["[object Object]"]) def test_increment(self): + py.test.skip("not ready yet") self.assert_prints(""" var x; x = 1; @@ -355,6 +383,7 @@ print(x);""", ["2"]) def test_ternaryop(self): + py.test.skip("not ready yet") self.assert_prints([ "( 1 == 1 ) ? print('yep') : print('nope');", "( 1 == 0 ) ? print('yep') : print('nope');"], @@ -368,6 +397,7 @@ print(x);""", ["true", "false"]) def test_unarynot(self): + py.test.skip("not ready yet") self.assert_prints(""" var x = false; print(!x); @@ -403,6 +433,7 @@ """, ['NaN', 'inf', 'undefined']) def test_strangefunc(self): + py.test.skip("not ready yet") self.assert_prints("""function f1() { var z; var t;}""", []) self.assert_prints(""" "'t'"; """, []) @@ -410,10 +441,12 @@ self.assert_result("null;", w_Null) def test_void(self): + py.test.skip("not ready yet") self.assert_prints("print(void print('hello'));", ["hello", "undefined"]) def test_activationprob(self): + py.test.skip("not ready yet") self.assert_prints( """ function intern (int1){ print(int1); @@ -429,6 +462,7 @@ """, ['1','1', '1']) def test_array_acess(self): + py.test.skip("not ready yet") self.assert_prints(""" var x = new Array(); x[0] = 1; @@ -440,6 +474,7 @@ """, ['1', '2', '3']) def test_array_length(self): + py.test.skip("not ready yet") self.assert_prints(""" var testcases = new Array(); var tc = testcases.length; @@ -453,6 +488,7 @@ self.assert_prints("print(+1);", ['1']) def test_delete(self): + py.test.skip("not ready yet") self.assert_prints(""" var x = {}; x.y = 1; @@ -461,6 +497,7 @@ """, ['undefined']) def test_forin(self): + py.test.skip("not ready yet") self.assert_prints(""" var x = {a:5}; for(y in x){ @@ -469,6 +506,7 @@ """, ['5',]) def test_stricteq(self): + py.test.skip("not ready yet") self.assert_prints(""" print(2 === 2); print(2 === 3); @@ -477,6 +515,7 @@ """, ['true', 'false', 'true', 'false']) def test_with(self): + py.test.skip("not ready yet") self.assert_prints(""" var mock = {x:2}; var x=4; @@ -502,23 +541,27 @@ """, ['0', '2', '3']) def test_for_strange(self): + py.test.skip("not ready yet") self.assert_prints(""" for (var arg = "", i = 0; i < 2; i++) { print(i);} """, ['0', '1']) def test_recursive_call(self): + py.test.skip("not ready yet") self.assert_prints(""" function fact(x) { if (x == 0) { return 1; } else { return fact(x-1)*x; }} print(fact(3)); """, ['6',]) def test_function_prototype(self): + py.test.skip("not ready yet") self.assert_prints(""" - function foo() {}; foo.prototype.bar = function() {} + function foo() {}; foo.prototype.bar = function() {}; """, []) def test_function_this(self): + py.test.skip("not ready yet") self.assert_prints(""" function foo() {print("debug");this.bar = function() {};}; var f = new foo(); From cfbolz at codespeak.net Mon May 21 13:00:54 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 21 May 2007 13:00:54 +0200 (CEST) Subject: [pypy-svn] r43523 - in pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter: . test Message-ID: <20070521110054.B37F9808F@code0.codespeak.net> Author: cfbolz Date: Mon May 21 13:00:53 2007 New Revision: 43523 Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/portal.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/test/test_jit.py Log: more changes Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py Mon May 21 13:00:53 2007 @@ -137,7 +137,7 @@ hint(j, concrete=True) hash1 = uh[j] hash2 = uh2[j] - if hash1 != 0 and hash2 != 0 and hash2 != hash1: + if hash1 != 0 and hash2 * (hash2 - hash1) != 0: break j += 1 else: @@ -269,6 +269,7 @@ hint(where, concrete=True) hint(rule, concrete=True) while 1: + hint(None, global_merge_point=True) #print " " * self.depth, where, query if where == CALL: next = self._call(query, continuation) @@ -287,13 +288,6 @@ where, query, continuation, rule = next where = hint(where, promote=True) - def _opaque_main_loop(self, where, query, continuation, rule=None): - return self.portal_main_loop(where, query, continuation, rule) - - def portal_main_loop(self, where, query, continuation, rule=None): - hint(None, global_merge_point=True) - return self.main_loop(where, query, continuation, rule) - def _jit_lookup(self, signature): signature2function = self.signature2function function = signature2function.get(signature, None) @@ -375,11 +369,24 @@ def try_rule(self, rule, query, continuation=DONOTHING, choice_point=True, inline=False): - #if not choice_point and inline: - # return (TRY_RULE, query, continuation, rule) if not we_are_jitted(): - return self.portal_main_loop(TRY_RULE, query, continuation, rule) - return self._opaque_main_loop(TRY_RULE, query, continuation, rule) + if not choice_point: + return (TRY_RULE, query, continuation, rule) + return self.portal_try_rule(rule, query, continuation, choice_point) + if _is_early_constant(rule): + rule = hint(rule, promote=True) + return self.portal_try_rule(rule, query, continuation, choice_point) + return self._opaque_try_rule(rule, query, continuation, choice_point) + + def _opaque_try_rule(self, rule, query, continuation, choice_point): + return self.portal_try_rule(rule, query, continuation, choice_point) + + def portal_try_rule(self, rule, query, continuation, choice_point): + hint(None, global_merge_point=True) + #hint(choice_point, concrete=True) + #if not choice_point: + # return self._try_rule(rule, query, continuation) + return self.main_loop(TRY_RULE, query, continuation, rule) def _try_rule(self, rule, query, continuation): rule = hint(rule, deepfreeze=True) Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/portal.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/portal.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/portal.py Mon May 21 13:00:53 2007 @@ -11,7 +11,7 @@ 'pypy.lang.prolog.builtin.register': True } -PORTAL = engine.Engine.portal_main_loop.im_func +PORTAL = engine.Engine.portal_try_rule.im_func class PyrologHintAnnotatorPolicy(HintAnnotatorPolicy): novirtualcontainer = True @@ -126,7 +126,6 @@ seegraph(cls.copy) seegraph(cls.__init__) seegraph(cls.copy_and_unify) - seegraph(cls.unify_hash_of_child) for cls in [term.Term, term.Number, term.Atom]: seegraph(cls.copy_and_basic_unify) seegraph(cls.dereference) @@ -147,6 +146,7 @@ seegraph(pypy.lang.prolog.interpreter.engine.Engine.main_loop) seegraph(pypy.lang.prolog.interpreter.engine.LinkedRules.find_applicable_rule) seegraph(pypy.lang.prolog.interpreter.engine.Continuation.call) + seegraph(term.Term.unify_hash_of_child) for cls in [engine.Continuation, engine.LimitedScopeContinuation, pypy.lang.prolog.builtin.control.AndContinuation]: seegraph(cls._call) Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/test/test_jit.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/test/test_jit.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/test/test_jit.py Mon May 21 13:00:53 2007 @@ -98,3 +98,24 @@ backendoptimize=True) assert res == True + def test_loop(self): + e = get_engine(""" + f(X) :- h(X). + f(a). + h(X) :- h(X). + """) + X = e.heap.newvar() + + def main(n): + e.heap.reset() + if n == 0: + e.call(term.Term("f", [X])) + return isinstance(X.dereference(e.heap), term.Atom) + else: + return False + + res = self.timeshift_from_portal(main, portal.PORTAL, + [0], policy=POLICY, + backendoptimize=True) + assert res == True + From cfbolz at codespeak.net Mon May 21 13:43:14 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 21 May 2007 13:43:14 +0200 (CEST) Subject: [pypy-svn] r43525 - pypy/branch/prolog-jit-experiments/pypy/translator/goal Message-ID: <20070521114314.493D2808F@code0.codespeak.net> Author: cfbolz Date: Mon May 21 13:43:13 2007 New Revision: 43525 Modified: pypy/branch/prolog-jit-experiments/pypy/translator/goal/targetprologstandalone.py Log: tell the jit the name of the executable Modified: pypy/branch/prolog-jit-experiments/pypy/translator/goal/targetprologstandalone.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/translator/goal/targetprologstandalone.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/translator/goal/targetprologstandalone.py Mon May 21 13:43:13 2007 @@ -14,6 +14,9 @@ term.DEBUG = False def entry_point(argv): + from pypy.jit.codegen.hlinfo import highleveljitinfo + if highleveljitinfo.sys_executable is None: + highleveljitinfo.sys_executable = argv[0] if len(argv) == 2: execute(e, argv[1]) try: From arigo at codespeak.net Mon May 21 13:53:15 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 21 May 2007 13:53:15 +0200 (CEST) Subject: [pypy-svn] r43526 - in pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter: . test Message-ID: <20070521115315.C7F17808F@code0.codespeak.net> Author: arigo Date: Mon May 21 13:53:13 2007 New Revision: 43526 Modified: pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/hrtyper.py pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_timeshift.py Log: Check with an assert that global merge points only occur in the portal. Modified: pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/hrtyper.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/hrtyper.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/hrtyper.py Mon May 21 13:53:13 2007 @@ -414,9 +414,12 @@ transformer.transform() flowmodel.checkgraph(graph) # for now self.contains_promotion |= transformer.contains_promotion + global_mp = transformer.mergepointfamily.has_global_mergepoints() if is_portal: - self.portal_contains_global_mp = ( - transformer.mergepointfamily.has_global_mergepoints()) + self.portal_contains_global_mp = global_mp + else: + assert not global_mp, ("global_merge_point only allowed in portal" + " - found one in %s" % (graph,)) return transformer.tsgraphs_seen def timeshift_ops(self, graph): Modified: pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_timeshift.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_timeshift.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_timeshift.py Mon May 21 13:53:13 2007 @@ -1749,4 +1749,11 @@ self.check_insns({'int_eq': 2}) assert res == f(0) - + def test_misplaced_global_merge_point(self): + def g(n): + hint(None, global_merge_point=True) + return n+1 + def f(n): + hint(None, global_merge_point=True) + return g(n) + py.test.raises(AssertionError, self.timeshift, f, [7], []) From santagada at codespeak.net Mon May 21 17:34:53 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Mon, 21 May 2007 17:34:53 +0200 (CEST) Subject: [pypy-svn] r43528 - in pypy/dist/pypy/lang/js: . test Message-ID: <20070521153453.C94608093@code0.codespeak.net> Author: santagada Date: Mon May 21 17:34:52 2007 New Revision: 43528 Modified: pypy/dist/pypy/lang/js/operations.py pypy/dist/pypy/lang/js/test/test_interp.py Log: restructured test_interp Modified: pypy/dist/pypy/lang/js/operations.py ============================================================================== --- pypy/dist/pypy/lang/js/operations.py (original) +++ pypy/dist/pypy/lang/js/operations.py Mon May 21 17:34:52 2007 @@ -53,9 +53,6 @@ self.pos = pos class Expression(Statement): - def eval(self, ctx): - return W_Root() - def execute(self, ctx): return self.eval(ctx) @@ -809,24 +806,7 @@ def execute(self, ctx): return self.body.execute(ctx) - -class Semicolon(Statement): - opcode = 'SEMICOLON' - - def __init__(self, pos, t): - self.expr = get_obj(t, 'expression') - def execute(self, ctx): - if self.expr is None: - return w_Undefined - return self.expr.execute(ctx) - -# class ExpressionStatement(Statement): -# def __init__(self, pos, exp): -# self.expr = expr -# -# def execute(self, ctx): -# return self.expr.eval(ctx) class Return(Statement): opcode = 'RETURN' Modified: pypy/dist/pypy/lang/js/test/test_interp.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_interp.py (original) +++ pypy/dist/pypy/lang/js/test/test_interp.py Mon May 21 17:34:52 2007 @@ -2,569 +2,586 @@ import sys from StringIO import StringIO -import py.test +import py from pypy.lang.js import interpreter from pypy.lang.js.jsparser import parse from pypy.lang.js.interpreter import * from pypy.lang.js.jsobj import W_Number, W_Object, ExecutionContext - -def js_is_on_path(): - if py.path.local.sysfind("js") is None: - py.test.skip("js binary not found") - -js_is_on_path() - -class TestInterp(object): - def test_simple(self): - n1 = Number(Position(), 2.0) - n2 = Number(Position(), 4.0) - p = Plus(Position(), n1, n2) - assert p.eval(ExecutionContext()).GetValue().ToNumber() == 6.0 - l = [] - interpreter.writer = l.append - # Script([Semicolon(Call(Identifier('print', None), - # List([Number(1), Number(2)])))],[],[]).execute(ExecutionContext()) - # assert l == ['1,2'] - - def assert_prints(self, code, assval): - l = [] - interpreter.writer = l.append - js_int = interpreter.Interpreter() - try: - if isinstance(code, str): - js_int.run(load_source(code)) - else: - for codepiece in code: - js_int.run(load_source(codepiece)) - except ThrowException, excpt: - l.append("uncaught exception: "+str(excpt.exception.ToString())) - print l, assval - assert l == assval - - def assert_result(self, code, result): - inter = interpreter.Interpreter() - r = inter.run(load_source(code)) - assert r.ToString() == result.ToString() - - def test_interp_parse(self): - self.assert_prints("print(1+1);", ["2"]) - self.assert_prints("print(1+2+3); print(1);", ["6", "1"]) - self.assert_prints("print(1,2,3);\n", ["1,2,3"]) - - def test_var_assign(self): - self.assert_prints("x=3;print(x);", ["3"]) - self.assert_prints("x=3;y=4;print(x+y);", ["7"]) - - def test_minus(self): - self.assert_prints("print(2-1);", ["1"]) - - def test_string_var(self): - self.assert_prints('print(\"sss\");', ["sss"]) - - def test_string_concat(self): - self.assert_prints('x="xxx"; y="yyy"; print(x+y);', ["xxxyyy"]) - - def test_string_num_concat(self): - self.assert_prints('x=4; y="x"; print(x+y, y+x);', ["4x,x4"]) - - def test_to_string(self): - self.assert_prints("x={}; print(x);", ["[object Object]"]) - - def test_object_access(self): - self.assert_prints("x={d:3}; print(x.d);", ["3"]) - self.assert_prints("x={d:3}; print(x.d.d);", ["undefined"]) - self.assert_prints("x={d:3, z:4}; print(x.d+x.z);", ["7"]) - - def test_object_access_index(self): - self.assert_prints('x={d:"x"}; print(x["d"]);', ["x"]) - - def test_function_prints(self): - py.test.skip("not ready yet") - self.assert_prints('x=function(){print(3);}; x();', ["3"]) - - def test_function_returns(self): - py.test.skip("not ready yet") - self.assert_prints('x=function(){return 1;}; print(x()+x());', ["2"]) - self.assert_prints('function x() { return; };', []) - - def test_var_declaration(self): - self.assert_prints('var x = 3; print(x);', ["3"]) - self.assert_prints('var x = 3; print(x+x);', ["6"]) - - def test_var_scoping(self): - py.test.skip("not ready yet") - self.assert_prints(""" - var y; +def test_simple(): + n1 = Number(Position(), 2.0) + n2 = Number(Position(), 4.0) + p = Plus(Position(), n1, n2) + assert p.eval(ExecutionContext()).GetValue().ToNumber() == 6.0 + l = [] + interpreter.writer = l.append + +def assert_prints(code, assval): + l = [] + interpreter.writer = l.append + js_int = interpreter.Interpreter() + try: + if isinstance(code, str): + js_int.run(load_source(code)) + else: + for codepiece in code: + js_int.run(load_source(codepiece)) + except ThrowException, excpt: + l.append("uncaught exception: "+str(excpt.exception.ToString())) + print l, assval + assert l == assval + +def assertp(code, prints): + l = [] + interpreter.writer = l.append + jsint = interpreter.Interpreter() + try: + jsint.run(load_source(code)) + except ThrowException, excpt: + l.append("uncaught exception: "+str(excpt.exception.ToString())) + print l, prints + if isinstance(prints, list): + assert l == prints + else: + assert l[0] == prints + +def assertv(code, value): + jsint = interpreter.Interpreter() + try: + code_val = jsint.run(load_source(code)).GetValue() + except ThrowException, excpt: + code_val = excpt + print code_val, value + if isinstance(value, int): + assert code_val.ToInt32() == value + elif isinstance(value, float): + assert code_val.ToNumber() == value + else: + assert code_val.ToString() == value + +def assert_result(code, result): + inter = interpreter.Interpreter() + r = inter.run(load_source(code)) + assert r.ToString() == result.ToString() + +def test_interp_parse(): + yield assertv, "1+1;", 2 + yield assertp, "print(1+2+3); print(1);", ["6", "1"] + yield assertp, "print(1,2,3);\n", "1,2,3" + +def test_var_assign(): + yield assertv, "x=3;x;", 3 + yield assertv, "x=3;y=4;x+y;", 7 + +def test_minus(): + assert_prints("print(2-1);", ["1"]) + +def test_string_var(): + assert_prints('print(\"sss\");', ["sss"]) + +def test_string_concat(): + assert_prints('x="xxx"; y="yyy"; print(x+y);', ["xxxyyy"]) + +def test_string_num_concat(): + assert_prints('x=4; y="x"; print(x+y, y+x);', ["4x,x4"]) + +def test_to_string(): + assert_prints("x={}; print(x);", ["[object Object]"]) + +def test_object_access(): + assert_prints("x={d:3}; print(x.d);", ["3"]) + assert_prints("x={d:3}; print(x.d.d);", ["undefined"]) + assert_prints("x={d:3, z:4}; print(x.d+x.z);", ["7"]) + +def test_object_access_index(): + assert_prints('x={d:"x"}; print(x["d"]);', ["x"]) + +def test_function_prints(): + py.test.skip("not ready yet") + assert_prints('x=function(){print(3);}; x();', ["3"]) + +def test_function_returns(): + py.test.skip("not ready yet") + assert_prints('x=function(){return 1;}; print(x()+x());', ["2"]) + assert_prints('function x() { return; };', []) + +def test_var_declaration(): + assert_prints('var x = 3; print(x);', ["3"]) + assert_prints('var x = 3; print(x+x);', ["6"]) + +def test_var_scoping(): + py.test.skip("not ready yet") + assert_prints(""" + var y; + var p; + p = 0; + x = function() { var p; - p = 0; - x = function() { - var p; - p = 1; - y = 3; return y + z; - }; - var z = 2; - print(x(), y, p); - """, ["5,3,0"]) - - def test_function_args(self): - py.test.skip("not ready yet") - self.assert_prints(""" - x = function (t,r) { - return t+r; - }; - print(x(2,3)); - """, ["5"]) - - def test_function_less_args(self): - py.test.skip("not ready yet") - self.assert_prints(""" - x = function (t, r) { - return t + r; - }; - print(x(2)); - """, ["NaN"]) - - def test_function_more_args(self): - py.test.skip("not ready yet") - self.assert_prints(""" - x = function (t, r) { - return t + r; - }; - print(x(2,3,4)); - """, ["5"]) - - def test_function_has_var(self): - py.test.skip("not ready yet") - self.assert_prints(""" - x = function () { - var t = 'test'; - return t; - }; - print(x()); - """, ["test"]) - - def test_function_arguments(self): - py.test.skip("not ready yet") - self.assert_prints(""" - x = function () { - r = arguments[0]; - t = arguments[1]; - return t + r; - }; - print(x(2,3)); - """, ["5"]) - - - def test_index(self): - self.assert_prints(""" - x = {1:"test"}; - print(x[1]); - """, ["test"]) - - def test_array_initializer(self): - self.assert_prints(""" - x = []; + p = 1; + y = 3; return y + z; + }; + var z = 2; + print(x(), y, p); + """, ["5,3,0"]) + +def test_function_args(): + py.test.skip("not ready yet") + assert_prints(""" + x = function (t,r) { + return t+r; + }; + print(x(2,3)); + """, ["5"]) + +def test_function_less_args(): + py.test.skip("not ready yet") + assert_prints(""" + x = function (t, r) { + return t + r; + }; + print(x(2)); + """, ["NaN"]) + +def test_function_more_args(): + py.test.skip("not ready yet") + assert_prints(""" + x = function (t, r) { + return t + r; + }; + print(x(2,3,4)); + """, ["5"]) + +def test_function_has_var(): + py.test.skip("not ready yet") + assert_prints(""" + x = function () { + var t = 'test'; + return t; + }; + print(x()); + """, ["test"]) + +def test_function_arguments(): + py.test.skip("not ready yet") + assert_prints(""" + x = function () { + r = arguments[0]; + t = arguments[1]; + return t + r; + }; + print(x(2,3)); + """, ["5"]) + + +def test_index(): + yield assertv, """ + x = {1:"test"}; + x[1]; + """, 'test' + +def test_array_initializer(): + assert_prints(""" + x = []; + print(x); + """, [""]) + +def test_throw(): + assert_prints("throw(3);", ["uncaught exception: 3"]) + +def test_group(): + assert_prints("print((2+1));", ["3"]) + +def test_comma(): + py.test.skip("not ready yet") + assert_prints("print((500,3));", ["3"]) + +def test_try_catch(): + py.test.skip("not ready yet") + assert_prints(""" + try { + throw(3); + } + catch (x) { print(x); - """, [""]) - - def test_throw(self): - self.assert_prints("throw(3);", ["uncaught exception: 3"]) - - def test_group(self): - self.assert_prints("print((2+1));", ["3"]) - - def test_comma(self): - py.test.skip("not ready yet") - self.assert_prints("print((500,3));", ["3"]) - - def test_try_catch(self): - py.test.skip("not ready yet") - self.assert_prints(""" - try { - throw(3); - } - catch (x) { - print(x); - } - """, ["3"]) - - def test_block(self): - py.test.skip("not ready yet") - self.assert_result("{ 5};", W_Number(5)) - self.assert_result("{3; 5};", W_Number(5)) - - def test_try_catch_finally(self): - py.test.skip("not ready yet") - self.assert_prints(""" - try { - throw(3); - } - catch (x) { - print(x); - } - finally { - print(5); - } - """, ["3", "5"]) - - def test_if_then(self): - py.test.skip("not ready yet") - self.assert_prints(""" - if (1) { - print(1); - } - """, ["1"]) - - def test_if_then_else(self): - py.test.skip("not ready yet") - self.assert_prints(""" - if (0) { - print(1); - } else { - print(2); - } - """, ["2"]) - - def test_compare(self): - py.test.skip("not ready yet") - self.assert_prints("print(1>0);",["true"]) - self.assert_prints("print(0>1);",["false"]) - self.assert_prints("print(0>0);",["false"]) - self.assert_prints("print(1<0);",["false"]) - self.assert_prints("print(0<1);",["true"]) - self.assert_prints("print(0<0);",["false"]) - self.assert_prints("print(1>=0);",["true"]) - self.assert_prints("print(1>=1);",["true"]) - self.assert_prints("print(1>=2);",["false"]) - self.assert_prints("print(0<=1);",["true"]) - self.assert_prints("print(1<=1);",["true"]) - self.assert_prints("print(1<=0);",["false"]) - self.assert_prints("print(0==0);",["true"]) - self.assert_prints("print(1==1);",["true"]) - self.assert_prints("print(0==1);",["false"]) - self.assert_prints("print(0!=1);",["true"]) - self.assert_prints("print(1!=1);",["false"]) - - def test_binary_op(self): - self.assert_prints("print(0||0); print(1||0);",["0", "1"]) - self.assert_prints("print(0&&1); print(1&&1);",["0", "1"]) - - def test_while(self): - py.test.skip("not ready yet") - self.assert_prints(""" - i = 0; - while (i<3) { - print(i); - i = i+1; - } - print(i); - """, ["0","1","2","3"]) - - def test_object_creation(self): - py.test.skip("not ready yet") - self.assert_prints(""" - o = new Object(); - print(o); - """, ["[object Object]"]) - - def test_var_decl(self): - py.test.skip("not ready yet") - self.assert_prints("print(x); var x;", ["undefined"]) - self.assert_prints(""" - try { - print(z); - } - catch (e) { - print(e); - } - """, ["ReferenceError: z is not defined"]) + } + """, ["3"]) - def test_function_name(self): - py.test.skip("not ready yet") - self.assert_prints(""" - function x() { - print("my name is x"); - } - x(); - """, ["my name is x"]) - - def test_new_with_function(self): - py.test.skip("not ready yet") - c= """ - x = function() {this.info = 'hello';}; - o = new x(); - print(o.info); - """ - print c - self.assert_prints(c, ["hello"]) - - def test_vars(self): - self.assert_prints(""" - var x;x=3; print(x);""", ["3"]) - - def test_minus(self): - py.test.skip("not ready yet") - self.assert_prints(""" - x = {y:3}; - print("y" in x); - print("z" in x); - """, ["true", "false"]) - - def test_append_code(self): - self.assert_prints([""" - var x; x=3; - """, """ +def test_block(): + py.test.skip("not ready yet") + assert_result("{ 5};", W_Number(5)) + assert_result("{3; 5};", W_Number(5)) + +def test_try_catch_finally(): + py.test.skip("not ready yet") + assert_prints(""" + try { + throw(3); + } + catch (x) { print(x); - z = 2; - """,""" - print(z); - """] - ,["3", "2"]) - - def test_for(self): - py.test.skip("not ready yet") - self.assert_prints(""" - for (i=0; i<3; i++) { - print(i); - } + } + finally { + print(5); + } + """, ["3", "5"]) + +def test_if_then(): + py.test.skip("not ready yet") + assert_prints(""" + if (1) { + print(1); + } + """, ["1"]) + +def test_if_then_else(): + py.test.skip("not ready yet") + assert_prints(""" + if (0) { + print(1); + } else { + print(2); + } + """, ["2"]) + +def test_compare(): + py.test.skip("not ready yet") + assert_prints("print(1>0);",["true"]) + assert_prints("print(0>1);",["false"]) + assert_prints("print(0>0);",["false"]) + assert_prints("print(1<0);",["false"]) + assert_prints("print(0<1);",["true"]) + assert_prints("print(0<0);",["false"]) + assert_prints("print(1>=0);",["true"]) + assert_prints("print(1>=1);",["true"]) + assert_prints("print(1>=2);",["false"]) + assert_prints("print(0<=1);",["true"]) + assert_prints("print(1<=1);",["true"]) + assert_prints("print(1<=0);",["false"]) + assert_prints("print(0==0);",["true"]) + assert_prints("print(1==1);",["true"]) + assert_prints("print(0==1);",["false"]) + assert_prints("print(0!=1);",["true"]) + assert_prints("print(1!=1);",["false"]) + +def test_binary_op(): + assert_prints("print(0||0); print(1||0);",["0", "1"]) + assert_prints("print(0&&1); print(1&&1);",["0", "1"]) + +def test_while(): + py.test.skip("not ready yet") + assert_prints(""" + i = 0; + while (i<3) { print(i); - """, ["0","1","2","3"]) - - def test_eval(self): - py.test.skip("not ready yet") - self.assert_prints(""" - var x = 2; - eval('x=x+1; print(x); z=2'); + i = i+1; + } + print(i); + """, ["0","1","2","3"]) + +def test_object_creation(): + py.test.skip("not ready yet") + assert_prints(""" + o = new Object(); + print(o); + """, ["[object Object]"]) + +def test_var_decl(): + py.test.skip("not ready yet") + assert_prints("print(x); var x;", ["undefined"]) + assert_prints(""" + try { print(z); - """, ["3","2"]) - - def test_arrayobject(self): - py.test.skip("not ready yet") - self.assert_prints("""var x = new Array(); - print(x.length == 0);""", ['true']) - - def test_break(self): - py.test.skip("not ready yet") - self.assert_prints(""" - while(1){ - break; - } - for(x=0;1==1;x++) { - break; - } - print('out');""", ["out"]) - - def test_typeof(self): - py.test.skip("not ready yet") - self.assert_result(""" - var x = 3; - typeof x == 'number'; - """, W_Boolean(True)) + } + catch (e) { + print(e); + } + """, ["ReferenceError: z is not defined"]) + +def test_function_name(): + py.test.skip("not ready yet") + assert_prints(""" + function x() { + print("my name is x"); + } + x(); + """, ["my name is x"]) - def test_semicolon(self): - py.test.skip("not ready yet") - self.assert_prints(';', []) - - def test_newwithargs(self): - py.test.skip("not ready yet") - self.assert_prints(""" - var x = new Object(1,2,3,4); - print(x); - """, ["[object Object]"]) - - def test_increment(self): - py.test.skip("not ready yet") - self.assert_prints(""" - var x; - x = 1; - x++; - print(x);""", ["2"]) - - def test_ternaryop(self): - py.test.skip("not ready yet") - self.assert_prints([ - "( 1 == 1 ) ? print('yep') : print('nope');", - "( 1 == 0 ) ? print('yep') : print('nope');"], - ["yep","nope"]) - - def test_booleanliterals(self): - self.assert_prints(""" - var x = false; - var y = true; +def test_new_with_function(): + py.test.skip("not ready yet") + c= """ + x = function() {this.info = 'hello';}; + o = new x(); + print(o.info); + """ + print c + assert_prints(c, ["hello"]) + +def test_vars(): + assert_prints(""" + var x;x=3; print(x);""", ["3"]) + +def test_minus(): + py.test.skip("not ready yet") + assert_prints(""" + x = {y:3}; + print("y" in x); + print("z" in x); + """, ["true", "false"]) + +def test_append_code(): + assert_prints([""" + var x; x=3; + """, """ + print(x); + z = 2; + """,""" + print(z); + """] + ,["3", "2"]) + +def test_for(): + py.test.skip("not ready yet") + assert_prints(""" + for (i=0; i<3; i++) { + print(i); + } + print(i); + """, ["0","1","2","3"]) + +def test_eval(): + py.test.skip("not ready yet") + assert_prints(""" + var x = 2; + eval('x=x+1; print(x); z=2'); + print(z); + """, ["3","2"]) + +def test_arrayobject(): + py.test.skip("not ready yet") + assert_prints("""var x = new Array(); + print(x.length == 0);""", ['true']) + +def test_break(): + py.test.skip("not ready yet") + assert_prints(""" + while(1){ + break; + } + for(x=0;1==1;x++) { + break; + } + print('out');""", ["out"]) + +def test_typeof(): + py.test.skip("not ready yet") + assert_result(""" + var x = 3; + typeof x == 'number'; + """, W_Boolean(True)) + +def test_semicolon(): + py.test.skip("not ready yet") + assert_prints(';', []) + +def test_newwithargs(): + py.test.skip("not ready yet") + assert_prints(""" + var x = new Object(1,2,3,4); + print(x); + """, ["[object Object]"]) + +def test_increment(): + py.test.skip("not ready yet") + assert_prints(""" + var x; + x = 1; + x++; + print(x);""", ["2"]) + +def test_ternaryop(): + py.test.skip("not ready yet") + assert_prints([ + "( 1 == 1 ) ? print('yep') : print('nope');", + "( 1 == 0 ) ? print('yep') : print('nope');"], + ["yep","nope"]) + +def test_booleanliterals(): + assert_prints(""" + var x = false; + var y = true; + print(y); + print(x);""", ["true", "false"]) + +def test_unarynot(): + py.test.skip("not ready yet") + assert_prints(""" + var x = false; + print(!x); + print(!!x);""", ["true", "false"]) + +def test_equals(): + assert_prints(""" + var x = 5; + y = z = x; + print(y);""", ["5"]) + +def test_math_stuff(): + assert_prints(""" + var x = 5; + var z = 2; + print(x*z); + print(4/z); + print(isNaN(z)); + print(Math.abs(z-x)); + print(Number.NaN); + print(Number.POSITIVE_INFINITY); + print(Number.NEGATIVE_INFINITY); + print(Math.floor(3.2)); + print(null); + print(-z); + """, ['10', '2', 'false', '3', 'NaN', 'inf', '-inf', '3', '', '-2']) + +def test_globalproperties(): + assert_prints( """ + print(NaN); + print(Infinity); + print(undefined); + """, ['NaN', 'inf', 'undefined']) + +def test_strangefunc(): + py.test.skip("not ready yet") + assert_prints("""function f1() { var z; var t;}""", []) + assert_prints(""" "'t'"; """, []) + +def test_null(): + assert_result("null;", w_Null) + +def test_void(): + py.test.skip("not ready yet") + assert_prints("print(void print('hello'));", + ["hello", "undefined"]) + +def test_activationprob(): + py.test.skip("not ready yet") + assert_prints( """ + function intern (int1){ + print(int1); + return int1; + } + function x (v1){ + this.p1 = v1; + this.p2 = intern(this.p1); + } + var ins = new x(1); + print(ins.p1); + print(ins.p2); + """, ['1','1', '1']) + +def test_array_acess(): + py.test.skip("not ready yet") + assert_prints(""" + var x = new Array(); + x[0] = 1; + x[x[0]] = 2; + x[2] = x[0]+x[1]; + for(i=0; i<3; i++){ + print(x[i]); + } + """, ['1', '2', '3']) + +def test_array_length(): + py.test.skip("not ready yet") + assert_prints(""" + var testcases = new Array(); + var tc = testcases.length; + print('tc'+tc); + """, ['tc0']) + +def test_mod_op(): + assert_prints("print(2%2);", ['0']) + +def test_unary_plus(): + assert_prints("print(+1);", ['1']) + +def test_delete(): + py.test.skip("not ready yet") + assert_prints(""" + var x = {}; + x.y = 1; + delete x.y; + print(x.y); + """, ['undefined']) + +def test_forin(): + py.test.skip("not ready yet") + assert_prints(""" + var x = {a:5}; + for(y in x){ print(y); - print(x);""", ["true", "false"]) - - def test_unarynot(self): - py.test.skip("not ready yet") - self.assert_prints(""" - var x = false; - print(!x); - print(!!x);""", ["true", "false"]) - - def test_equals(self): - self.assert_prints(""" - var x = 5; - y = z = x; - print(y);""", ["5"]) - - def test_math_stuff(self): - self.assert_prints(""" - var x = 5; - var z = 2; - print(x*z); - print(4/z); - print(isNaN(z)); - print(Math.abs(z-x)); - print(Number.NaN); - print(Number.POSITIVE_INFINITY); - print(Number.NEGATIVE_INFINITY); - print(Math.floor(3.2)); - print(null); - print(-z); - """, ['10', '2', 'false', '3', 'NaN', 'inf', '-inf', '3', '', '-2']) - - def test_globalproperties(self): - self.assert_prints( """ - print(NaN); - print(Infinity); - print(undefined); - """, ['NaN', 'inf', 'undefined']) - - def test_strangefunc(self): - py.test.skip("not ready yet") - self.assert_prints("""function f1() { var z; var t;}""", []) - self.assert_prints(""" "'t'"; """, []) - - def test_null(self): - self.assert_result("null;", w_Null) + } + """, ['5',]) - def test_void(self): - py.test.skip("not ready yet") - self.assert_prints("print(void print('hello'));", - ["hello", "undefined"]) - - def test_activationprob(self): - py.test.skip("not ready yet") - self.assert_prints( """ - function intern (int1){ - print(int1); - return int1; - } - function x (v1){ - this.p1 = v1; - this.p2 = intern(this.p1); - } - var ins = new x(1); - print(ins.p1); - print(ins.p2); - """, ['1','1', '1']) - - def test_array_acess(self): - py.test.skip("not ready yet") - self.assert_prints(""" - var x = new Array(); - x[0] = 1; - x[x[0]] = 2; - x[2] = x[0]+x[1]; - for(i=0; i<3; i++){ - print(x[i]); - } - """, ['1', '2', '3']) - - def test_array_length(self): - py.test.skip("not ready yet") - self.assert_prints(""" - var testcases = new Array(); - var tc = testcases.length; - print('tc'+tc); - """, ['tc0']) - - def test_mod_op(self): - self.assert_prints("print(2%2);", ['0']) - - def test_unary_plus(self): - self.assert_prints("print(+1);", ['1']) - - def test_delete(self): - py.test.skip("not ready yet") - self.assert_prints(""" - var x = {}; - x.y = 1; - delete x.y; - print(x.y); - """, ['undefined']) - - def test_forin(self): - py.test.skip("not ready yet") - self.assert_prints(""" - var x = {a:5}; - for(y in x){ - print(y); - } - """, ['5',]) - - def test_stricteq(self): - py.test.skip("not ready yet") - self.assert_prints(""" - print(2 === 2); - print(2 === 3); - print(2 !== 3); - print(2 !== 2); - """, ['true', 'false', 'true', 'false']) - - def test_with(self): - py.test.skip("not ready yet") - self.assert_prints(""" - var mock = {x:2}; - var x=4; - print(x); - try { - with(mock) { - print(x); - throw 3; - print("not reacheable"); - } - } - catch(y){ - print(y); +def test_stricteq(): + py.test.skip("not ready yet") + assert_prints(""" + print(2 === 2); + print(2 === 3); + print(2 !== 3); + print(2 !== 2); + """, ['true', 'false', 'true', 'false']) + +def test_with(): + py.test.skip("not ready yet") + assert_prints(""" + var mock = {x:2}; + var x=4; + print(x); + try { + with(mock) { + print(x); + throw 3; + print("not reacheable"); } - print(x); - """, ['4', '2', '3', '4']) - - def test_bitops(self): - self.assert_prints(""" - print(2 ^ 2); - print(2 & 3); - print(2 | 3); - """, ['0', '2', '3']) - - def test_for_strange(self): - py.test.skip("not ready yet") - self.assert_prints(""" - for (var arg = "", i = 0; i < 2; i++) { print(i);} - """, ['0', '1']) - - def test_recursive_call(self): - py.test.skip("not ready yet") - self.assert_prints(""" - function fact(x) { if (x == 0) { return 1; } else { return fact(x-1)*x; }} - print(fact(3)); - """, ['6',]) - - def test_function_prototype(self): - py.test.skip("not ready yet") - self.assert_prints(""" - function foo() {}; foo.prototype.bar = function() {}; - """, []) - - - def test_function_this(self): - py.test.skip("not ready yet") - self.assert_prints(""" - function foo() {print("debug");this.bar = function() {};}; - var f = new foo(); - f.bar(); - """, ['debug',]) + } + catch(y){ + print(y); + } + print(x); + """, ['4', '2', '3', '4']) + +def test_bitops(): + assert_prints(""" + print(2 ^ 2); + print(2 & 3); + print(2 | 3); + """, ['0', '2', '3']) + +def test_for_strange(): + py.test.skip("not ready yet") + assert_prints(""" + for (var arg = "", i = 0; i < 2; i++) { print(i);} + """, ['0', '1']) + +def test_recursive_call(): + py.test.skip("not ready yet") + assert_prints(""" + function fact(x) { if (x == 0) { return 1; } else { return fact(x-1)*x; }} + print(fact(3)); + """, ['6',]) + +def test_function_prototype(): + py.test.skip("not ready yet") + assert_prints(""" + function foo() {}; foo.prototype.bar = function() {}; + """, []) + + +def test_function_this(): + py.test.skip("not ready yet") + assert_prints(""" + function foo() {print("debug");this.bar = function() {};}; + var f = new foo(); + f.bar(); + """, ['debug',]) From santagada at codespeak.net Mon May 21 18:05:58 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Mon, 21 May 2007 18:05:58 +0200 (CEST) Subject: [pypy-svn] r43536 - in pypy/dist/pypy/lang/js: . test Message-ID: <20070521160558.1EC1D808F@code0.codespeak.net> Author: santagada Date: Mon May 21 18:05:57 2007 New Revision: 43536 Modified: pypy/dist/pypy/lang/js/astbuilder.py pypy/dist/pypy/lang/js/jsgrammar.txt pypy/dist/pypy/lang/js/operations.py pypy/dist/pypy/lang/js/test/test_astbuilder.py Log: "new" preliminar support Modified: pypy/dist/pypy/lang/js/astbuilder.py ============================================================================== --- pypy/dist/pypy/lang/js/astbuilder.py (original) +++ pypy/dist/pypy/lang/js/astbuilder.py Mon May 21 18:05:57 2007 @@ -102,7 +102,6 @@ return left visit_additiveexpression = binaryop visit_multiplicativeexpression = binaryop - visit_memberexpression = binaryop visit_bitwisexorexpression = binaryop visit_bitwiseandexpression = binaryop visit_bitwiseorexpression = binaryop @@ -110,6 +109,18 @@ visit_logicalorexpression = binaryop visit_logicalandexpression = binaryop + def visit_memberexpression(self, node): + if isinstance(node.children[0], Symbol) and \ + node.children[0].additional_info == 'new': # XXX could be a identifier? + # "new case" + pos = self.get_pos(node) + left = self.dispatch(node.children[1]) + right = self.dispatch(node.children[2]) + exp = operations.Call(pos, left, right) + return operations.New(pos, exp) + else: + return self.binaryop(node) + def literalop(self, node): pos = self.get_pos(node); @@ -215,3 +226,11 @@ def visit_emptystatement(self, node): return operations.astundef + def visit_newexpression(self, node): + if len(node.children) == 1: + return self.dispatch(node.children[0]) + else: + pos = self.get_pos(node) + val = self.dispatch(node.children[0]) + return operations.New(pos, val) + Modified: pypy/dist/pypy/lang/js/jsgrammar.txt ============================================================================== --- pypy/dist/pypy/lang/js/jsgrammar.txt (original) +++ pypy/dist/pypy/lang/js/jsgrammar.txt Mon May 21 18:05:57 2007 @@ -213,8 +213,8 @@ | "[" expression ["]"] ; -newexpression : "new" newexpression - | +newexpression : + | "new" newexpression ; callexpression : memberexpression arguments >callexpressiona<* Modified: pypy/dist/pypy/lang/js/operations.py ============================================================================== --- pypy/dist/pypy/lang/js/operations.py (original) +++ pypy/dist/pypy/lang/js/operations.py Mon May 21 18:05:57 2007 @@ -693,14 +693,13 @@ ############################################################################## class New(UnaryOp): - opcode = 'NEW' - def eval(self, ctx): x = self.expr.eval(ctx).GetValue() if not isinstance(x, W_PrimitiveObject): raise TypeError() return x.Construct(ctx=ctx) + class NewWithArgs(BinaryOp): opcode = 'NEW_WITH_ARGS' Modified: pypy/dist/pypy/lang/js/test/test_astbuilder.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_astbuilder.py (original) +++ pypy/dist/pypy/lang/js/test/test_astbuilder.py Mon May 21 18:05:57 2007 @@ -3,7 +3,10 @@ from pypy.lang.js import operations def to_ast(s): - ASTBuilder().dispatch(parse(s)) + print s + tp = parse(s) + print tp + ASTBuilder().dispatch(tp) def test_simple(): yield to_ast, "1;" @@ -13,7 +16,9 @@ yield to_ast, "x[1];" yield to_ast, "true;" yield to_ast, "false;" - yield to_ast, "null;" + yield to_ast, "null;" + yield to_ast, "f();" + yield to_ast, "new f();" #yield to_ast, "" def test_funcvarfinder(): From cfbolz at codespeak.net Mon May 21 18:13:45 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 21 May 2007 18:13:45 +0200 (CEST) Subject: [pypy-svn] r43537 - in pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter: . test Message-ID: <20070521161345.AA3ED8082@code0.codespeak.net> Author: cfbolz Date: Mon May 21 18:13:44 2007 New Revision: 43537 Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/portal.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/test/test_jit.py Log: pyrolog-jit now produce somewhat sensible code again! :-) Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py Mon May 21 18:13:44 2007 @@ -141,8 +141,7 @@ break j += 1 else: - #XXX otherwise the result is green which seems wrong - return hint(self, variable=True) + return self self = self.next return None @@ -247,9 +246,6 @@ signature = hint(signature, promote=True) for bsig, builtin in unrolling_builtins: if signature == bsig: - #XXX should be: - #return builtin.call(self, query, continuation) - # but then the JIT explodes sometimes for funny reasons return builtin.call(self, query, continuation) return self.user_call(query, continuation, choice_point=False) @@ -263,31 +259,34 @@ return self.user_call(query, continuation, choice_point=False) def main_loop(self, where, query, continuation, rule=None): - from pypy.lang.prolog.builtin.control import AndContinuation - from pypy.lang.prolog.interpreter import helper next = (DONE, None, None, None) hint(where, concrete=True) hint(rule, concrete=True) while 1: - hint(None, global_merge_point=True) + #hint(None, global_merge_point=True) #print " " * self.depth, where, query - if where == CALL: - next = self._call(query, continuation) - elif where == TRY_RULE: - rule = hint(rule, promote=True) - next = self._try_rule(rule, query, continuation) - elif where == USER_CALL: - next = self._user_call(query, continuation) - elif where == CONTINUATION: - hint(continuation.__class__, promote=True) - next = continuation._call(self) - elif where == DONE: + if where == DONE: return next - else: - raise Exception("unknown bytecode") + next = self.dispatch_bytecode(where, query, continuation, rule) where, query, continuation, rule = next where = hint(where, promote=True) + + def dispatch_bytecode(self, where, query, continuation, rule): + if where == CALL: + next = self._call(query, continuation) + elif where == TRY_RULE: + rule = hint(rule, promote=True) + next = self._try_rule(rule, query, continuation) + elif where == USER_CALL: + next = self._user_call(query, continuation) + elif where == CONTINUATION: + hint(continuation.__class__, promote=True) + next = continuation._call(self) + else: + raise Exception("unknown bytecode") + return next + def _jit_lookup(self, signature): signature2function = self.signature2function function = signature2function.get(signature, None) @@ -296,7 +295,7 @@ return function _jit_lookup._pure_function_ = True - def user_call(self, query, continuation, choice_point=True, inline=False): + def user_call(self, query, continuation, choice_point=True): if not choice_point: return (USER_CALL, query, continuation, None) return self.main_loop(USER_CALL, query, continuation) @@ -323,7 +322,6 @@ if rulechain is None: # none of the rules apply raise UnificationFailed() - rulechain = hint(rulechain, promote=True) rule = rulechain.rule rulechain = rulechain.next oldstate = self.heap.branch() @@ -332,7 +330,6 @@ if rulechain is None: self.heap.discard(oldstate) break - rulechain = hint(rulechain, promote=True) hint(rule, concrete=True) if rule.contains_cut: continuation = LimitedScopeContinuation(continuation) @@ -370,12 +367,12 @@ def try_rule(self, rule, query, continuation=DONOTHING, choice_point=True, inline=False): if not we_are_jitted(): - if not choice_point: - return (TRY_RULE, query, continuation, rule) - return self.portal_try_rule(rule, query, continuation, choice_point) - if _is_early_constant(rule): - rule = hint(rule, promote=True) return self.portal_try_rule(rule, query, continuation, choice_point) + if not choice_point: + return (TRY_RULE, query, continuation, rule) + #if _is_early_constant(rule): + # rule = hint(rule, promote=True) + # return self.portal_try_rule(rule, query, continuation, choice_point) return self._opaque_try_rule(rule, query, continuation, choice_point) def _opaque_try_rule(self, rule, query, continuation, choice_point): @@ -386,7 +383,18 @@ #hint(choice_point, concrete=True) #if not choice_point: # return self._try_rule(rule, query, continuation) - return self.main_loop(TRY_RULE, query, continuation, rule) + where = TRY_RULE + next = (DONE, None, None, None) + hint(where, concrete=True) + hint(rule, concrete=True) + while 1: + hint(None, global_merge_point=True) + rule = hint(rule, promote=True) + if where == DONE: + return next + next = self.dispatch_bytecode(where, query, continuation, rule) + where, query, continuation, rule = next + where = hint(where, promote=True) def _try_rule(self, rule, query, continuation): rule = hint(rule, deepfreeze=True) Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/portal.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/portal.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/portal.py Mon May 21 18:13:44 2007 @@ -40,15 +40,6 @@ return False return True -def jitme(func): - func._look_inside_me_ = True - return func - -def dontjitme(func): - func._look_inside_me_ = False - return func - - def enumerate_reachable_graphs(translator, startgraph): from pypy.translator.backendopt.support import find_calls_from pending = [(startgraph, None)] @@ -144,6 +135,7 @@ seegraph(pypy.lang.prolog.interpreter.engine.Engine.try_rule) seegraph(pypy.lang.prolog.interpreter.engine.Engine._try_rule) seegraph(pypy.lang.prolog.interpreter.engine.Engine.main_loop) + seegraph(pypy.lang.prolog.interpreter.engine.Engine.dispatch_bytecode) seegraph(pypy.lang.prolog.interpreter.engine.LinkedRules.find_applicable_rule) seegraph(pypy.lang.prolog.interpreter.engine.Continuation.call) seegraph(term.Term.unify_hash_of_child) Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/test/test_jit.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/test/test_jit.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/test/test_jit.py Mon May 21 18:13:44 2007 @@ -72,6 +72,31 @@ backendoptimize=True) assert res == True + def test_append(self): + e = get_engine(""" + append([], L, L). + append([X|Y], L, [X|Z]) :- append(Y, L, Z). + """) + t = parse_query_term("append([a, b, c], [d, f, g], X).") + X = e.heap.newvar() + + def main(n): + if n == 0: + e.call(t) + return isinstance(X.dereference(e.heap), term.Term) + else: + return False + + res = main(0) + assert res == True + + e.heap.reset() + res = self.timeshift_from_portal(main, portal.PORTAL, + [0], policy=POLICY, + backendoptimize=True) + assert res == True + + def test_user_call(self): e = get_engine(""" h(X) :- f(X, b). From cfbolz at codespeak.net Mon May 21 18:14:20 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 21 May 2007 18:14:20 +0200 (CEST) Subject: [pypy-svn] r43538 - pypy/branch/prolog-jit-experiments/pypy/rpython Message-ID: <20070521161420.E11718082@code0.codespeak.net> Author: cfbolz Date: Mon May 21 18:14:20 2007 New Revision: 43538 Modified: pypy/branch/prolog-jit-experiments/pypy/rpython/llinterp.py Log: comment out a warning Modified: pypy/branch/prolog-jit-experiments/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/rpython/llinterp.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/rpython/llinterp.py Mon May 21 18:14:20 2007 @@ -594,7 +594,8 @@ if hasattr(obj, 'graph'): assert obj.graph in graphs else: - log.warn("op_indirect_call with graphs=None:", f) + pass + #log.warn("op_indirect_call with graphs=None:", f) return self.op_direct_call(f, *args) def op_adr_call(self, TGT, f, *inargs): From cfbolz at codespeak.net Mon May 21 18:15:52 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 21 May 2007 18:15:52 +0200 (CEST) Subject: [pypy-svn] r43539 - pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test Message-ID: <20070521161552.CF038808F@code0.codespeak.net> Author: cfbolz Date: Mon May 21 18:15:52 2007 New Revision: 43539 Modified: pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_timeshift.py Log: use the inline argument too Modified: pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_timeshift.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_timeshift.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_timeshift.py Mon May 21 18:15:52 2007 @@ -58,7 +58,7 @@ auto_inlining(t, threshold=inline) if backendoptimize: from pypy.translator.backendopt.all import backend_optimizations - backend_optimizations(t) + backend_optimizations(t, inline_threshold=inline) if portal is None: portal = func if hasattr(policy, "seetranslator"): From cfbolz at codespeak.net Mon May 21 18:37:24 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 21 May 2007 18:37:24 +0200 (CEST) Subject: [pypy-svn] r43540 - in pypy/branch/prolog-jit-experiments/pypy: annotation/test interpreter jit/timeshifter/test jit/tl lang/automata lang/prolog/interpreter module/pypyjit objspace/std rlib rpython/lltypesystem/test translator/goal Message-ID: <20070521163724.C22CA806E@code0.codespeak.net> Author: cfbolz Date: Mon May 21 18:37:23 2007 New Revision: 43540 Added: pypy/branch/prolog-jit-experiments/pypy/rlib/jit.py (contents, props changed) Modified: pypy/branch/prolog-jit-experiments/pypy/annotation/test/test_annrpython.py pypy/branch/prolog-jit-experiments/pypy/interpreter/baseobjspace.py pypy/branch/prolog-jit-experiments/pypy/interpreter/gateway.py pypy/branch/prolog-jit-experiments/pypy/interpreter/pyframe.py pypy/branch/prolog-jit-experiments/pypy/interpreter/pyopcode.py pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_frontend.py pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_portal.py pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_promotion.py pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_timeshift.py pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_vdict.py pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_virtualizable.py pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_vlist.py pypy/branch/prolog-jit-experiments/pypy/jit/tl/tiny1.py pypy/branch/prolog-jit-experiments/pypy/jit/tl/tiny2.py pypy/branch/prolog-jit-experiments/pypy/jit/tl/tl.py pypy/branch/prolog-jit-experiments/pypy/jit/tl/tlc.py pypy/branch/prolog-jit-experiments/pypy/jit/tl/tlr.py pypy/branch/prolog-jit-experiments/pypy/lang/automata/dfa.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/term.py pypy/branch/prolog-jit-experiments/pypy/module/pypyjit/interp_jit.py pypy/branch/prolog-jit-experiments/pypy/objspace/std/multimethod.py pypy/branch/prolog-jit-experiments/pypy/objspace/std/objspace.py pypy/branch/prolog-jit-experiments/pypy/objspace/std/typeobject.py pypy/branch/prolog-jit-experiments/pypy/rlib/objectmodel.py pypy/branch/prolog-jit-experiments/pypy/rpython/lltypesystem/test/test_rvirtualizable.py pypy/branch/prolog-jit-experiments/pypy/translator/goal/ann_override.py pypy/branch/prolog-jit-experiments/pypy/translator/goal/targetportal.py Log: move jit-related stuff to its own file in rlib.jit Modified: pypy/branch/prolog-jit-experiments/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/annotation/test/test_annrpython.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/annotation/test/test_annrpython.py Mon May 21 18:37:23 2007 @@ -2706,7 +2706,7 @@ assert isinstance(s, annmodel.SomeExternalBuiltin) def test_instance_with_flags(self): - from pypy.rlib.objectmodel import hint + from pypy.rlib.jit import hint class A: _virtualizable_ = True Modified: pypy/branch/prolog-jit-experiments/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/interpreter/baseobjspace.py Mon May 21 18:37:23 2007 @@ -3,7 +3,7 @@ from pypy.interpreter.argument import Arguments, ArgumentsFromValuestack from pypy.interpreter.pycompiler import CPythonCompiler, PythonAstCompiler from pypy.interpreter.miscutils import ThreadLocals -from pypy.rlib.objectmodel import hint +from pypy.rlib.jit import hint from pypy.tool.cache import Cache from pypy.tool.uid import HUGEVAL_BYTES import os, sys Modified: pypy/branch/prolog-jit-experiments/pypy/interpreter/gateway.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/interpreter/gateway.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/interpreter/gateway.py Mon May 21 18:37:23 2007 @@ -19,7 +19,7 @@ from pypy.interpreter.baseobjspace import Wrappable, SpaceCache, DescrMismatch from pypy.interpreter.argument import Arguments, AbstractArguments from pypy.tool.sourcetools import NiceCompile, compile2 -from pypy.rlib.objectmodel import hint +from pypy.rlib.jit import hint # internal non-translatable parts: import py Modified: pypy/branch/prolog-jit-experiments/pypy/interpreter/pyframe.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/interpreter/pyframe.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/interpreter/pyframe.py Mon May 21 18:37:23 2007 @@ -8,7 +8,7 @@ from pypy.interpreter import pytraceback import opcode from pypy.rlib.objectmodel import we_are_translated, instantiate -from pypy.rlib.objectmodel import we_are_jitted, hint +from pypy.rlib.jit import we_are_jitted, hint from pypy.rlib import rstack # for resume points Modified: pypy/branch/prolog-jit-experiments/pypy/interpreter/pyopcode.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/interpreter/pyopcode.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/interpreter/pyopcode.py Mon May 21 18:37:23 2007 @@ -12,7 +12,8 @@ from pypy.interpreter.argument import Arguments from pypy.interpreter.pycode import PyCode from pypy.tool.sourcetools import func_with_new_name -from pypy.rlib.objectmodel import we_are_translated, hint, we_are_jitted +from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.jit import hint, we_are_jitted from pypy.rlib.rarithmetic import r_uint, intmask from pypy.tool.stdlib_opcode import opcodedesc, HAVE_ARGUMENT from pypy.tool.stdlib_opcode import unrolling_opcode_descs Modified: pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_frontend.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_frontend.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_frontend.py Mon May 21 18:37:23 2007 @@ -1,4 +1,4 @@ -from pypy.rlib.objectmodel import we_are_jitted, _is_early_constant, hint +from pypy.rlib.jit import we_are_jitted, _is_early_constant, hint from pypy.rpython.test.test_llinterp import interpret from pypy.jit.timeshifter.test.test_timeshift import TimeshiftingTests Modified: pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_portal.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_portal.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_portal.py Mon May 21 18:37:23 2007 @@ -7,7 +7,7 @@ from pypy.rpython.llinterp import LLInterpreter from pypy.rpython.lltypesystem import lltype from pypy.objspace.flow.model import summary -from pypy.rlib.objectmodel import hint +from pypy.rlib.jit import hint from pypy.jit.codegen.llgraph.rgenop import RGenOp as LLRGenOp import py.test Modified: pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_promotion.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_promotion.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_promotion.py Mon May 21 18:37:23 2007 @@ -4,7 +4,7 @@ from pypy.jit.timeshifter.test.test_timeshift import StopAtXPolicy from pypy.jit.timeshifter.test.test_timeshift import P_NOVIRTUAL from pypy.jit.timeshifter.test.test_vlist import P_OOPSPEC -from pypy.rlib.objectmodel import hint +from pypy.rlib.jit import hint from pypy.rpython.module.support import LLSupport class TestPromotion(TimeshiftingTests): Modified: pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_timeshift.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_timeshift.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_timeshift.py Mon May 21 18:37:23 2007 @@ -9,7 +9,8 @@ from pypy.jit.timeshifter import rtimeshift, rvalue from pypy.objspace.flow.model import summary, Variable from pypy.rpython.lltypesystem import lltype, llmemory, rstr -from pypy.rlib.objectmodel import hint, keepalive_until_here, debug_assert +from pypy.rlib.jit import hint +from pypy.rlib.objectmodel import keepalive_until_here, debug_assert from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.rarithmetic import ovfcheck from pypy.rpython.annlowlevel import PseudoHighLevelCallable, cachedtype Modified: pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_vdict.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_vdict.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_vdict.py Mon May 21 18:37:23 2007 @@ -1,6 +1,6 @@ from pypy.jit.hintannotator.annotator import HintAnnotatorPolicy from pypy.jit.timeshifter.test.test_timeshift import TimeshiftingTests -from pypy.rlib.objectmodel import hint +from pypy.rlib.jit import hint P_OOPSPEC = HintAnnotatorPolicy(novirtualcontainer = True, oopspec = True) Modified: pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_virtualizable.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_virtualizable.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_virtualizable.py Mon May 21 18:37:23 2007 @@ -2,7 +2,7 @@ from pypy.jit.timeshifter.test.test_timeshift import StopAtXPolicy from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.lltypesystem.rvirtualizable import VABLERTIPTR -from pypy.rlib.objectmodel import hint +from pypy.rlib.jit import hint import py S = lltype.GcStruct('s', ('a', lltype.Signed), ('b', lltype.Signed)) Modified: pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_vlist.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_vlist.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_vlist.py Mon May 21 18:37:23 2007 @@ -1,6 +1,6 @@ from pypy.jit.hintannotator.annotator import HintAnnotatorPolicy from pypy.jit.timeshifter.test.test_timeshift import TimeshiftingTests -from pypy.rlib.objectmodel import hint +from pypy.rlib.jit import hint P_OOPSPEC = HintAnnotatorPolicy(novirtualcontainer=True, oopspec=True) Modified: pypy/branch/prolog-jit-experiments/pypy/jit/tl/tiny1.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/jit/tl/tiny1.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/jit/tl/tiny1.py Mon May 21 18:37:23 2007 @@ -1,4 +1,4 @@ -from pypy.rlib.objectmodel import hint +from pypy.rlib.jit import hint def ll_plus_minus(s, x, y): Modified: pypy/branch/prolog-jit-experiments/pypy/jit/tl/tiny2.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/jit/tl/tiny2.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/jit/tl/tiny2.py Mon May 21 18:37:23 2007 @@ -27,7 +27,7 @@ { #1 #1 1 SUB ->#1 #1 } => when called with 5, gives '5 4 3 2 1' """ -from pypy.rlib.objectmodel import hint, _is_early_constant +from pypy.rlib.jit import hint, _is_early_constant # # See pypy/doc/jit.txt for a higher-level overview of the JIT techniques Modified: pypy/branch/prolog-jit-experiments/pypy/jit/tl/tl.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/jit/tl/tl.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/jit/tl/tl.py Mon May 21 18:37:23 2007 @@ -2,7 +2,7 @@ import py from pypy.jit.tl.opcode import * -from pypy.rlib.objectmodel import hint +from pypy.rlib.jit import hint def char2int(c): t = ord(c) Modified: pypy/branch/prolog-jit-experiments/pypy/jit/tl/tlc.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/jit/tl/tlc.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/jit/tl/tlc.py Mon May 21 18:37:23 2007 @@ -3,7 +3,7 @@ import py from pypy.jit.tl.opcode import * from pypy.jit.tl import opcode as tlopcode -from pypy.rlib.objectmodel import hint +from pypy.rlib.jit import hint class Obj(object): Modified: pypy/branch/prolog-jit-experiments/pypy/jit/tl/tlr.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/jit/tl/tlr.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/jit/tl/tlr.py Mon May 21 18:37:23 2007 @@ -1,4 +1,4 @@ -from pypy.rlib.objectmodel import hint +from pypy.rlib.jit import hint MOV_A_R = 1 Modified: pypy/branch/prolog-jit-experiments/pypy/lang/automata/dfa.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/automata/dfa.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/automata/dfa.py Mon May 21 18:37:23 2007 @@ -1,6 +1,6 @@ " a very stripped down version of cfbolz's algorithm/automaton module " -from pypy.rlib.objectmodel import hint +from pypy.rlib.jit import hint from pypy.rpython.lltypesystem.lltype import GcArray, Signed, malloc class DFA(object): Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py Mon May 21 18:37:23 2007 @@ -3,8 +3,8 @@ from pypy.lang.prolog.interpreter.error import UnificationFailed, FunctionNotFound, \ CutException from pypy.lang.prolog.interpreter import error -from pypy.rlib.objectmodel import hint, specialize, _is_early_constant -from pypy.rlib.objectmodel import we_are_jitted +from pypy.rlib.jit import hint, we_are_jitted, _is_early_constant +from pypy.rlib.objectmodel import specialize from pypy.rlib.unroll import unrolling_iterable DEBUG = False Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/term.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/term.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/term.py Mon May 21 18:37:23 2007 @@ -2,7 +2,8 @@ from pypy.rlib.objectmodel import we_are_translated, UnboxedValue from pypy.rlib.rarithmetic import intmask from pypy.lang.prolog.interpreter.error import UnificationFailed, UncatchableError -from pypy.rlib.objectmodel import hint, specialize +from pypy.rlib.jit import hint +from pypy.rlib.objectmodel import specialize DEBUG = False Modified: pypy/branch/prolog-jit-experiments/pypy/module/pypyjit/interp_jit.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/module/pypyjit/interp_jit.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/module/pypyjit/interp_jit.py Mon May 21 18:37:23 2007 @@ -10,7 +10,7 @@ import sys from pypy.tool.pairtype import extendabletype from pypy.rlib.rarithmetic import r_uint, intmask -from pypy.rlib.objectmodel import hint, _is_early_constant +from pypy.rlib.jit import hint, _is_early_constant import pypy.interpreter.pyopcode # for side-effects from pypy.interpreter.eval import Frame from pypy.interpreter.pycode import PyCode, CO_VARARGS, CO_VARKEYWORDS Modified: pypy/branch/prolog-jit-experiments/pypy/objspace/std/multimethod.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/objspace/std/multimethod.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/objspace/std/multimethod.py Mon May 21 18:37:23 2007 @@ -794,7 +794,7 @@ bodylines.append(' pass') bodylines.append('return %s' % expr(calllist[-1])) - from pypy.rlib.objectmodel import hint + from pypy.rlib.jit import hint miniglobals['hint'] = hint entry = FuncEntry(bodylines, miniglobals, fallback) key = entry.key() Modified: pypy/branch/prolog-jit-experiments/pypy/objspace/std/objspace.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/objspace/std/objspace.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/objspace/std/objspace.py Mon May 21 18:37:23 2007 @@ -14,7 +14,8 @@ from pypy.objspace.descroperation import DescrOperation from pypy.objspace.std import stdtypedef from pypy.rlib.rarithmetic import base_int -from pypy.rlib.objectmodel import we_are_translated, hint, we_are_jitted +from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.jit import hint, we_are_jitted import sys import os import __builtin__ Modified: pypy/branch/prolog-jit-experiments/pypy/objspace/std/typeobject.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/objspace/std/typeobject.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/objspace/std/typeobject.py Mon May 21 18:37:23 2007 @@ -6,7 +6,8 @@ from pypy.objspace.std.stdtypedef import std_dict_descr, issubtypedef, Member from pypy.objspace.std.objecttype import object_typedef from pypy.objspace.std.dictproxyobject import W_DictProxyObject -from pypy.rlib.objectmodel import we_are_translated, hint +from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.jit import hint from pypy.rlib.rarithmetic import intmask, r_uint from copy_reg import _HEAPTYPE Added: pypy/branch/prolog-jit-experiments/pypy/rlib/jit.py ============================================================================== --- (empty file) +++ pypy/branch/prolog-jit-experiments/pypy/rlib/jit.py Mon May 21 18:37:23 2007 @@ -0,0 +1,82 @@ +from pypy.rpython.extregistry import ExtRegistryEntry + +def hint(x, **kwds): + return x + +class Entry(ExtRegistryEntry): + _about_ = hint + + def compute_result_annotation(self, s_x, **kwds_s): + from pypy.annotation import model as annmodel + s_x = annmodel.not_const(s_x) + if 's_access_directly' in kwds_s: + if isinstance(s_x, annmodel.SomeInstance): + from pypy.objspace.flow.model import Constant + classdesc = s_x.classdef.classdesc + virtualizable = classdesc.read_attribute('_virtualizable_', + Constant(False)).value + if virtualizable: + flags = s_x.flags.copy() + flags['access_directly'] = True + s_x = annmodel.SomeInstance(s_x.classdef, + s_x.can_be_None, + flags) + return s_x + + def specialize_call(self, hop, **kwds_i): + from pypy.rpython.lltypesystem import lltype + hints = {} + for key, index in kwds_i.items(): + s_value = hop.args_s[index] + if not s_value.is_constant(): + from pypy.rpython.error import TyperError + raise TyperError("hint %r is not constant" % (key,)) + assert key.startswith('i_') + hints[key[2:]] = s_value.const + v = hop.inputarg(hop.args_r[0], arg=0) + c_hint = hop.inputconst(lltype.Void, hints) + hop.exception_cannot_occur() + return hop.genop('hint', [v, c_hint], resulttype=v.concretetype) + + +def we_are_jitted(): + return False +# timeshifts to True + +_we_are_jitted = CDefinedIntSymbolic('0 /* we are not jitted here */', + default=0) + +class Entry(ExtRegistryEntry): + _about_ = we_are_jitted + + def compute_result_annotation(self): + from pypy.annotation import model as annmodel + return annmodel.SomeInteger(nonneg=True) + + def specialize_call(self, hop): + from pypy.rpython.lltypesystem import lltype + return hop.inputconst(lltype.Signed, _we_are_jitted) + +def _is_early_constant(x): + return False + +class Entry(ExtRegistryEntry): + _about_ = _is_early_constant + + def compute_result_annotation(self, s_value): + from pypy.annotation import model as annmodel + s = annmodel.SomeBool() + if s_value.is_constant(): + s.const = True + return s + + def specialize_call(self, hop): + from pypy.rpython.lltypesystem import lltype + if hop.s_result.is_constant(): + assert hop.s_result.const + return hop.inputconst(lltype.Bool, True) + v, = hop.inputargs(hop.args_r[0]) + return hop.genop('is_early_constant', [v], resulttype=lltype.Bool) + + + Modified: pypy/branch/prolog-jit-experiments/pypy/rlib/objectmodel.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/rlib/objectmodel.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/rlib/objectmodel.py Mon May 21 18:37:23 2007 @@ -189,85 +189,6 @@ # ____________________________________________________________ -def hint(x, **kwds): - return x - -class Entry(ExtRegistryEntry): - _about_ = hint - - def compute_result_annotation(self, s_x, **kwds_s): - from pypy.annotation import model as annmodel - s_x = annmodel.not_const(s_x) - if 's_access_directly' in kwds_s: - if isinstance(s_x, annmodel.SomeInstance): - from pypy.objspace.flow.model import Constant - classdesc = s_x.classdef.classdesc - virtualizable = classdesc.read_attribute('_virtualizable_', - Constant(False)).value - if virtualizable: - flags = s_x.flags.copy() - flags['access_directly'] = True - s_x = annmodel.SomeInstance(s_x.classdef, - s_x.can_be_None, - flags) - return s_x - - def specialize_call(self, hop, **kwds_i): - from pypy.rpython.lltypesystem import lltype - hints = {} - for key, index in kwds_i.items(): - s_value = hop.args_s[index] - if not s_value.is_constant(): - from pypy.rpython.error import TyperError - raise TyperError("hint %r is not constant" % (key,)) - assert key.startswith('i_') - hints[key[2:]] = s_value.const - v = hop.inputarg(hop.args_r[0], arg=0) - c_hint = hop.inputconst(lltype.Void, hints) - hop.exception_cannot_occur() - return hop.genop('hint', [v, c_hint], resulttype=v.concretetype) - - -def we_are_jitted(): - return False -# timeshifts to True - -_we_are_jitted = CDefinedIntSymbolic('0 /* we are not jitted here */', - default=0) - -class Entry(ExtRegistryEntry): - _about_ = we_are_jitted - - def compute_result_annotation(self): - from pypy.annotation import model as annmodel - return annmodel.SomeInteger(nonneg=True) - - def specialize_call(self, hop): - from pypy.rpython.lltypesystem import lltype - return hop.inputconst(lltype.Signed, _we_are_jitted) - -def _is_early_constant(x): - return False - -class Entry(ExtRegistryEntry): - _about_ = _is_early_constant - - def compute_result_annotation(self, s_value): - from pypy.annotation import model as annmodel - s = annmodel.SomeBool() - if s_value.is_constant(): - s.const = True - return s - - def specialize_call(self, hop): - from pypy.rpython.lltypesystem import lltype - if hop.s_result.is_constant(): - assert hop.s_result.const - return hop.inputconst(lltype.Bool, True) - v, = hop.inputargs(hop.args_r[0]) - return hop.genop('is_early_constant', [v], resulttype=lltype.Bool) - - def debug_assert(x, msg): """After translation to C, this becomes an RPyAssert.""" assert x, msg Modified: pypy/branch/prolog-jit-experiments/pypy/rpython/lltypesystem/test/test_rvirtualizable.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/rpython/lltypesystem/test/test_rvirtualizable.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/rpython/lltypesystem/test/test_rvirtualizable.py Mon May 21 18:37:23 2007 @@ -1,6 +1,6 @@ import py from pypy.objspace.flow.model import summary -from pypy.rlib.objectmodel import hint +from pypy.rlib.jit import hint from pypy.rpython.lltypesystem import lltype, rclass from pypy.rpython.test.test_llinterp import interpret, get_interpreter from pypy.rpython.annlowlevel import cast_instance_to_base_ptr Modified: pypy/branch/prolog-jit-experiments/pypy/translator/goal/ann_override.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/translator/goal/ann_override.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/translator/goal/ann_override.py Mon May 21 18:37:23 2007 @@ -167,7 +167,7 @@ return CACHED_LOOKUP = """ -from pypy.rlib.objectmodel import hint +from pypy.rlib.jit import hint def lookup_%(attr)s(space, w_obj, name): w_type = space.type(w_obj) if not w_type.is_heaptype(): @@ -177,7 +177,7 @@ """ CACHED_LOOKUP_IN_TYPE_WHERE = """ -from pypy.rlib.objectmodel import hint +from pypy.rlib.jit import hint def lookup_in_type_where_%(attr)s(space, w_type, name): if not w_type.is_heaptype(): w_type = hint(w_type, deepfreeze=True) Modified: pypy/branch/prolog-jit-experiments/pypy/translator/goal/targetportal.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/translator/goal/targetportal.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/translator/goal/targetportal.py Mon May 21 18:37:23 2007 @@ -1,4 +1,4 @@ -from pypy.rlib.objectmodel import hint, we_are_jitted +from pypy.rlib.jit import hint, we_are_jitted def jitted(): print "jitted" From cfbolz at codespeak.net Mon May 21 18:45:49 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 21 May 2007 18:45:49 +0200 (CEST) Subject: [pypy-svn] r43541 - pypy/branch/prolog-jit-experiments/pypy/rpython/lltypesystem Message-ID: <20070521164549.35BBC807B@code0.codespeak.net> Author: cfbolz Date: Mon May 21 18:45:48 2007 New Revision: 43541 Modified: pypy/branch/prolog-jit-experiments/pypy/rpython/lltypesystem/rstr.py Log: add a "pure" hint to hash(string) Modified: pypy/branch/prolog-jit-experiments/pypy/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/rpython/lltypesystem/rstr.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/rpython/lltypesystem/rstr.py Mon May 21 18:45:48 2007 @@ -181,6 +181,7 @@ x = _hash_string(s.chars) s.hash = x return x + ll_strhash._pure_function_ = True # it's pure but it does not look like it def ll_strfasthash(s): return s.hash # assumes that the hash is already computed From cfbolz at codespeak.net Mon May 21 18:52:28 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 21 May 2007 18:52:28 +0200 (CEST) Subject: [pypy-svn] r43544 - in pypy/branch/prolog-jit-experiments/pypy: lang/prolog/interpreter rlib Message-ID: <20070521165228.D7AF3807B@code0.codespeak.net> Author: cfbolz Date: Mon May 21 18:52:27 2007 New Revision: 43544 Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/term.py pypy/branch/prolog-jit-experiments/pypy/rlib/jit.py Log: introduce a decorator to say that something is pure and use it Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py Mon May 21 18:52:27 2007 @@ -3,7 +3,7 @@ from pypy.lang.prolog.interpreter.error import UnificationFailed, FunctionNotFound, \ CutException from pypy.lang.prolog.interpreter import error -from pypy.rlib.jit import hint, we_are_jitted, _is_early_constant +from pypy.rlib.jit import hint, we_are_jitted, _is_early_constant, purefunction from pypy.rlib.objectmodel import specialize from pypy.rlib.unroll import unrolling_iterable @@ -287,13 +287,13 @@ raise Exception("unknown bytecode") return next + @purefunction def _jit_lookup(self, signature): signature2function = self.signature2function function = signature2function.get(signature, None) if function is None: signature2function[signature] = function = Function() return function - _jit_lookup._pure_function_ = True def user_call(self, query, continuation, choice_point=True): if not choice_point: Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/term.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/term.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/term.py Mon May 21 18:52:27 2007 @@ -19,9 +19,6 @@ if DEBUG and not we_are_translated(): print " ".join([str(a) for a in args]) -def pure_hash_function(s): - return hash(s) -pure_hash_function._pure_function_ = True class PrologObject(object): __slots__ = () @@ -261,7 +258,7 @@ def get_unify_hash(self): name = hint(self.name, promote=True) - return intmask(pure_hash_function(name) << TAGBITS | self.TAG) + return intmask(hash(name) << TAGBITS | self.TAG) def get_prolog_signature(self): return Term("/", [self, NUMBER_0]) @@ -466,7 +463,7 @@ def get_unify_hash(self): signature = hint(self.signature, promote=True) - return intmask(pure_hash_function(signature) << TAGBITS | self.TAG) + return intmask(hash(signature) << TAGBITS | self.TAG) def unify_hash_of_child(self, i): return self.args[i].get_unify_hash() Modified: pypy/branch/prolog-jit-experiments/pypy/rlib/jit.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/rlib/jit.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/rlib/jit.py Mon May 21 18:52:27 2007 @@ -1,5 +1,9 @@ from pypy.rpython.extregistry import ExtRegistryEntry +def purefunction(func): + func._pure_function_ = True + return func + def hint(x, **kwds): return x From cfbolz at codespeak.net Mon May 21 18:52:50 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 21 May 2007 18:52:50 +0200 (CEST) Subject: [pypy-svn] r43545 - pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter Message-ID: <20070521165250.3F7F4807B@code0.codespeak.net> Author: cfbolz Date: Mon May 21 18:52:49 2007 New Revision: 43545 Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/portal.py Log: use the impurity analyzer in the policy to not look into pure functions. Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/portal.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/portal.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/portal.py Mon May 21 18:52:49 2007 @@ -18,6 +18,8 @@ oopspec = True def seetranslator(self, t): + from pypy.jit.hintannotator.bookkeeper import ImpurityAnalyzer + self.analyzer = ImpurityAnalyzer(t) portal = getattr(PORTAL, 'im_func', PORTAL) portal_graph = graphof(t, portal) self.timeshift_graphs = timeshift_graphs(t, portal_graph) @@ -25,12 +27,18 @@ def look_inside_graph(self, graph): if graph in self.timeshift_graphs: return self.timeshift_graphs[graph] + # don't look into pure functions + if not self.analyzer.analyze_direct_call(graph): + return False + try: func = graph.func except AttributeError: return True if hasattr(func, '_look_inside_me_'): return func._look_inside_me_ + if getattr(func, '_pure_function_', False): + return False mod = func.__module__ or '?' if mod in forbidden_modules: return False From santagada at codespeak.net Mon May 21 18:53:29 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Mon, 21 May 2007 18:53:29 +0200 (CEST) Subject: [pypy-svn] r43546 - in pypy/dist/pypy/lang/js: . test Message-ID: <20070521165329.6432E807B@code0.codespeak.net> Author: santagada Date: Mon May 21 18:53:27 2007 New Revision: 43546 Modified: pypy/dist/pypy/lang/js/astbuilder.py pypy/dist/pypy/lang/js/operations.py pypy/dist/pypy/lang/js/test/test_interp.py Log: more tests passing Modified: pypy/dist/pypy/lang/js/astbuilder.py ============================================================================== --- pypy/dist/pypy/lang/js/astbuilder.py (original) +++ pypy/dist/pypy/lang/js/astbuilder.py Mon May 21 18:53:27 2007 @@ -42,10 +42,14 @@ '||': operations.Or, '==': operations.Eq, '!=': operations.Ne, + '!==': operations.StrictNe, + '===': operations.StrictEq, '.': operations.Member, '[': operations.Member, } UNOP_TO_CLS = { + '~': operations.BitwiseNot, + '!': operations.Not, '+': operations.UPlus, '-': operations.UMinus, '++': operations.Increment, @@ -74,6 +78,8 @@ source_pos = curr.token.source_pos # XXX some of the source positions are not perfect + if source_pos is None: + return operations.Position() return operations.Position( source_pos.lineno, source_pos.columnno, @@ -112,12 +118,10 @@ def visit_memberexpression(self, node): if isinstance(node.children[0], Symbol) and \ node.children[0].additional_info == 'new': # XXX could be a identifier? - # "new case" pos = self.get_pos(node) left = self.dispatch(node.children[1]) right = self.dispatch(node.children[2]) - exp = operations.Call(pos, left, right) - return operations.New(pos, exp) + return operations.NewWithArgs(pos, left, right) else: return self.binaryop(node) @@ -139,6 +143,13 @@ pos = self.get_pos(op) child = self.dispatch(node.children[1]) return self.UNOP_TO_CLS[op.additional_info](pos, child) + + def visit_postfixexpression(self, node): + op = node.children[1] + pos = self.get_pos(op) + child = self.dispatch(node.children[0]) + return self.UNOP_TO_CLS[op.additional_info](pos, child, postfix=True) + def listop(self, node): op = node.children[0] Modified: pypy/dist/pypy/lang/js/operations.py ============================================================================== --- pypy/dist/pypy/lang/js/operations.py (original) +++ pypy/dist/pypy/lang/js/operations.py Mon May 21 18:53:27 2007 @@ -154,28 +154,25 @@ class BitwiseAnd(BinaryBitwiseOp): - opcode = 'BITWISE_AND' - def decision(self, ctx, op1, op2): return W_Number(op1&op2) + class BitwiseNot(UnaryOp): - opcode = 'BITWISE_NOT' - def eval(self, ctx): op1 = self.expr.eval(ctx).GetValue().ToInt32() return W_Number(~op1) - + class BitwiseOr(BinaryBitwiseOp): - opcode = 'BITWISE_OR' - def decision(self, ctx, op1, op2): return W_Number(op1|op2) + class BitwiseXor(BinaryBitwiseOp): def decision(self, ctx, op1, op2): return W_Number(op1^op2) + class Unconditional(Statement): def __init__(self, pos, t): @@ -702,14 +699,13 @@ class NewWithArgs(BinaryOp): - opcode = 'NEW_WITH_ARGS' - def eval(self, ctx): x = self.left.eval(ctx).GetValue() if not isinstance(x, W_PrimitiveObject): raise TypeError() args = self.right.eval(ctx).get_args() return x.Construct(ctx=ctx, args=args) + class Number(Expression): def __init__(self, pos, num): @@ -1024,6 +1020,7 @@ class Not(UnaryOp): def eval(self, ctx): return W_Boolean(not self.expr.eval(ctx).GetValue().ToBoolean()) + class UMinus(UnaryOp): def eval(self, ctx): Modified: pypy/dist/pypy/lang/js/test/test_interp.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_interp.py (original) +++ pypy/dist/pypy/lang/js/test/test_interp.py Mon May 21 18:53:27 2007 @@ -53,7 +53,9 @@ except ThrowException, excpt: code_val = excpt print code_val, value - if isinstance(value, int): + if isinstance(value, bool): + assert code_val.ToBoolean() == value + elif isinstance(value, int): assert code_val.ToInt32() == value elif isinstance(value, float): assert code_val.ToNumber() == value @@ -75,10 +77,10 @@ yield assertv, "x=3;y=4;x+y;", 7 def test_minus(): - assert_prints("print(2-1);", ["1"]) + assertv("2-1;", 1) def test_string_var(): - assert_prints('print(\"sss\");', ["sss"]) + assertv('\"sss\";', 'sss') def test_string_concat(): assert_prints('x="xxx"; y="yyy"; print(x+y);', ["xxxyyy"]) @@ -280,11 +282,10 @@ """, ["0","1","2","3"]) def test_object_creation(): - py.test.skip("not ready yet") - assert_prints(""" + yield assertv, """ o = new Object(); - print(o); - """, ["[object Object]"]) + o; + """, "[object Object]" def test_var_decl(): py.test.skip("not ready yet") @@ -321,7 +322,7 @@ assert_prints(""" var x;x=3; print(x);""", ["3"]) -def test_minus(): +def test_in(): py.test.skip("not ready yet") assert_prints(""" x = {y:3}; @@ -350,17 +351,15 @@ """, ["0","1","2","3"]) def test_eval(): - py.test.skip("not ready yet") - assert_prints(""" + assertp(""" var x = 2; - eval('x=x+1; print(x); z=2'); + eval('x=x+1; print(x); z=2;'); print(z); """, ["3","2"]) def test_arrayobject(): - py.test.skip("not ready yet") - assert_prints("""var x = new Array(); - print(x.length == 0);""", ['true']) + assertv("""var x = new Array(); + x.length == 0;""", 'true') def test_break(): py.test.skip("not ready yet") @@ -381,23 +380,20 @@ """, W_Boolean(True)) def test_semicolon(): - py.test.skip("not ready yet") - assert_prints(';', []) + assertp(';', []) def test_newwithargs(): - py.test.skip("not ready yet") - assert_prints(""" + assertp(""" var x = new Object(1,2,3,4); print(x); - """, ["[object Object]"]) + """, "[object Object]") def test_increment(): - py.test.skip("not ready yet") - assert_prints(""" + assertv(""" var x; x = 1; x++; - print(x);""", ["2"]) + x;""", 2) def test_ternaryop(): py.test.skip("not ready yet") @@ -407,27 +403,26 @@ ["yep","nope"]) def test_booleanliterals(): - assert_prints(""" + assertp(""" var x = false; var y = true; print(y); print(x);""", ["true", "false"]) def test_unarynot(): - py.test.skip("not ready yet") - assert_prints(""" + assertp(""" var x = false; print(!x); print(!!x);""", ["true", "false"]) def test_equals(): - assert_prints(""" + assertv(""" var x = 5; y = z = x; - print(y);""", ["5"]) + y;""", 5) def test_math_stuff(): - assert_prints(""" + assertp(""" var x = 5; var z = 2; print(x*z); @@ -443,7 +438,7 @@ """, ['10', '2', 'false', '3', 'NaN', 'inf', '-inf', '3', '', '-2']) def test_globalproperties(): - assert_prints( """ + assertp( """ print(NaN); print(Infinity); print(undefined); @@ -491,18 +486,17 @@ """, ['1', '2', '3']) def test_array_length(): - py.test.skip("not ready yet") - assert_prints(""" + assertp(""" var testcases = new Array(); var tc = testcases.length; print('tc'+tc); - """, ['tc0']) + """, 'tc0') def test_mod_op(): - assert_prints("print(2%2);", ['0']) + assertp("print(2%2);", '0') def test_unary_plus(): - assert_prints("print(+1);", ['1']) + assertp("print(+1);", '1') def test_delete(): py.test.skip("not ready yet") @@ -523,13 +517,10 @@ """, ['5',]) def test_stricteq(): - py.test.skip("not ready yet") - assert_prints(""" - print(2 === 2); - print(2 === 3); - print(2 !== 3); - print(2 !== 2); - """, ['true', 'false', 'true', 'false']) + yield assertv, "2 === 2;", True + yield assertv, "2 === 3;", False + yield assertv, "2 !== 3;", True + yield assertv, "2 !== 2;", False def test_with(): py.test.skip("not ready yet") @@ -551,11 +542,9 @@ """, ['4', '2', '3', '4']) def test_bitops(): - assert_prints(""" - print(2 ^ 2); - print(2 & 3); - print(2 | 3); - """, ['0', '2', '3']) + yield assertv, "2 ^ 2;", 0 + yield assertv, "2 & 3;", 2 + yield assertv, "2 | 3;", 3 def test_for_strange(): py.test.skip("not ready yet") From cfbolz at codespeak.net Mon May 21 18:58:37 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 21 May 2007 18:58:37 +0200 (CEST) Subject: [pypy-svn] r43547 - pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test Message-ID: <20070521165837.248CE807B@code0.codespeak.net> Author: cfbolz Date: Mon May 21 18:58:36 2007 New Revision: 43547 Modified: pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_timeshift.py Log: deal with default value too Modified: pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_timeshift.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_timeshift.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_timeshift.py Mon May 21 18:58:36 2007 @@ -59,7 +59,8 @@ auto_inlining(t, threshold=inline) if backendoptimize: from pypy.translator.backendopt.all import backend_optimizations - backend_optimizations(t, inline_threshold=inline) + if inline is not None: + backend_optimizations(t, inline_threshold=inline) if portal is None: portal = func if hasattr(policy, "seetranslator"): From santagada at codespeak.net Tue May 22 01:44:19 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Tue, 22 May 2007 01:44:19 +0200 (CEST) Subject: [pypy-svn] r43548 - in pypy/dist/pypy/lang/js: . test Message-ID: <20070521234419.E20FF806E@code0.codespeak.net> Author: santagada Date: Tue May 22 01:44:18 2007 New Revision: 43548 Modified: pypy/dist/pypy/lang/js/jsobj.py pypy/dist/pypy/lang/js/jsparser.py pypy/dist/pypy/lang/js/test/test_interp.py pypy/dist/pypy/lang/js/test/test_parser.py Log: fixed some test on test_interp and aplied a nice patch from Amaury Forgeot d'Arc to make js interp tests not fail on windows, and made it more compliant to the js spec Modified: pypy/dist/pypy/lang/js/jsobj.py ============================================================================== --- pypy/dist/pypy/lang/js/jsobj.py (original) +++ pypy/dist/pypy/lang/js/jsobj.py Tue May 22 01:44:18 2007 @@ -389,14 +389,19 @@ return str(self.floatval)+"W" def ToString(self): - if str(self.floatval) == str(NaN): + floatstr = str(self.floatval) + if floatstr == str(NaN): return 'NaN' + if floatstr == str(Infinity): + return 'Infinity' + if floatstr == str(-Infinity): + return '-Infinity' try: if float(int(self.floatval)) == self.floatval: return str(int(self.floatval)) except OverflowError, e: pass - return str(self.floatval) + return floatstr def ToBoolean(self): if self.floatval == 0.0 or str(self.floatval) == str(NaN): Modified: pypy/dist/pypy/lang/js/jsparser.py ============================================================================== --- pypy/dist/pypy/lang/js/jsparser.py (original) +++ pypy/dist/pypy/lang/js/jsparser.py Tue May 22 01:44:18 2007 @@ -5,7 +5,7 @@ GFILE = py.magic.autopath().dirpath().join("jsgrammar.txt") try: - t = GFILE.read() + t = GFILE.read(mode='U') regexs, rules, ToAST = parse_ebnf(t) except ParseError,e: print e.nice_error_message(filename=str(GFILE),source=t) Modified: pypy/dist/pypy/lang/js/test/test_interp.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_interp.py (original) +++ pypy/dist/pypy/lang/js/test/test_interp.py Tue May 22 01:44:18 2007 @@ -230,11 +230,11 @@ def test_if_then(): py.test.skip("not ready yet") - assert_prints(""" + assertp(""" if (1) { print(1); } - """, ["1"]) + """, "1") def test_if_then_else(): py.test.skip("not ready yet") @@ -435,14 +435,15 @@ print(Math.floor(3.2)); print(null); print(-z); - """, ['10', '2', 'false', '3', 'NaN', 'inf', '-inf', '3', '', '-2']) + """, ['10', '2', 'false', '3', 'NaN', 'Infinity', '-Infinity', + '3', '', '-2']) def test_globalproperties(): assertp( """ print(NaN); print(Infinity); print(undefined); - """, ['NaN', 'inf', 'undefined']) + """, ['NaN', 'Infinity', 'undefined']) def test_strangefunc(): py.test.skip("not ready yet") Modified: pypy/dist/pypy/lang/js/test/test_parser.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_parser.py (original) +++ pypy/dist/pypy/lang/js/test/test_parser.py Tue May 22 01:44:18 2007 @@ -12,7 +12,7 @@ GFILE = py.magic.autopath().dirpath().dirpath().join("jsgrammar.txt") try: - t = GFILE.read() + t = GFILE.read(mode='U') regexs, rules, ToAST = parse_ebnf(t) except ParseError,e: print e.nice_error_message(filename=str(GFILE),source=t) From santagada at codespeak.net Tue May 22 02:53:48 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Tue, 22 May 2007 02:53:48 +0200 (CEST) Subject: [pypy-svn] r43549 - in pypy/dist/pypy/lang/js: . test Message-ID: <20070522005348.7F847806D@code0.codespeak.net> Author: santagada Date: Tue May 22 02:53:46 2007 New Revision: 43549 Modified: pypy/dist/pypy/lang/js/astbuilder.py pypy/dist/pypy/lang/js/jsgrammar.txt pypy/dist/pypy/lang/js/operations.py pypy/dist/pypy/lang/js/test/test_interp.py Log: lots of tests running, implemented ifs and whiles Modified: pypy/dist/pypy/lang/js/astbuilder.py ============================================================================== --- pypy/dist/pypy/lang/js/astbuilder.py (original) +++ pypy/dist/pypy/lang/js/astbuilder.py Tue May 22 02:53:46 2007 @@ -44,6 +44,10 @@ '!=': operations.Ne, '!==': operations.StrictNe, '===': operations.StrictEq, + '>': operations.Gt, + '>=': operations.Ge, + '<': operations.Lt, + '<=': operations.Le, '.': operations.Member, '[': operations.Member, } @@ -54,6 +58,7 @@ '-': operations.UMinus, '++': operations.Increment, '--': operations.Decrement, + 'typeof': operations.Typeof, } LISTOP_TO_CLS = { '[': operations.Array, @@ -114,6 +119,7 @@ visit_equalityexpression = binaryop visit_logicalorexpression = binaryop visit_logicalandexpression = binaryop + visit_relationalexpression = binaryop def visit_memberexpression(self, node): if isinstance(node.children[0], Symbol) and \ @@ -158,6 +164,22 @@ return self.LISTOP_TO_CLS[op.additional_info](pos, l) visit_arrayliteral = listop # XXX elision visit_objectliteral = listop + + def visit_block(self, node): + op = node.children[0] + pos = self.get_pos(op) + l = [self.dispatch(child) for child in node.children[1:]] + return operations.Block(pos, l) + + def visit_arguments(self, node): + pos = self.get_pos(node) + nodes = [self.dispatch(child) for child in node.children[1:]] + return operations.ArgumentList(pos, nodes) + + def visit_variabledeclarationlist(self, node): + pos = self.get_pos(node) + nodes = [self.dispatch(child) for child in node.children] + return operations.VariableDeclList(pos, nodes) def visit_propertynameandvalue(self, node): pos = self.get_pos(node) @@ -174,6 +196,16 @@ pos = self.get_pos(node) body = self.dispatch(node.children[0]) return operations.Program(pos, body) + + def visit_variablestatement(self, node): + pos = self.get_pos(node) + body = self.dispatch(node.children[0]) + return operations.Variable(pos, body) + + def visit_throwstatement(self, node): + pos = self.get_pos(node) + exp = self.dispatch(node.children[0]) + return operations.Throw(pos, exp) def visit_sourceelements(self, node): pos = self.get_pos(node) @@ -188,17 +220,7 @@ def visit_expressionstatement(self, node): return self.dispatch(node.children[0]) - - def visit_variablestatement(self, node): - pos = self.get_pos(node) - body = self.dispatch(node.children[0]) - return operations.Variable(pos, body) - - def visit_variabledeclarationlist(self, node): - pos = self.get_pos(node) - nodes = [self.dispatch(child) for child in node.children] - return operations.VariableDeclList(pos, nodes) - + def visit_variabledeclaration(self, node): pos = self.get_pos(node) identifier = self.dispatch(node.children[0]) @@ -213,12 +235,7 @@ left = self.dispatch(node.children[0]) right = self.dispatch(node.children[1]) return operations.Call(pos, left, right) - - def visit_arguments(self, node): - pos = self.get_pos(node) - nodes = [self.dispatch(child) for child in node.children[1:]] - return operations.ArgumentList(pos, nodes) - + def visit_assignmentexpression(self, node): pos = self.get_pos(node) left = self.dispatch(node.children[0]) @@ -228,12 +245,7 @@ def visit_functiondeclaration(self, node): pos = self.get_pos(node) - - def visit_throwstatement(self, node): - pos = self.get_pos(node) - exp = self.dispatch(node.children[0]) - return operations.Throw(pos, exp) - + def visit_emptystatement(self, node): return operations.astundef @@ -245,3 +257,19 @@ val = self.dispatch(node.children[0]) return operations.New(pos, val) + def visit_ifstatement(self, node): + pos = self.get_pos(node) + condition = self.dispatch(node.children[0]) + ifblock = self.dispatch(node.children[1]) + if len(node.children) > 2: + elseblock = self.dispatch(node.children[2]) + else: + elseblock = operations.astundef + return operations.If(pos, condition, ifblock, elseblock) + + def visit_iterationstatement(self, node): + pos = self.get_pos(node) + if node.children[0].additional_info == 'while': + condition = self.dispatch(node.children[1]) + block = self.dispatch(node.children[2]) + return operations.While(pos, condition, block) Modified: pypy/dist/pypy/lang/js/jsgrammar.txt ============================================================================== --- pypy/dist/pypy/lang/js/jsgrammar.txt (original) +++ pypy/dist/pypy/lang/js/jsgrammar.txt Tue May 22 02:53:46 2007 @@ -13,7 +13,7 @@ | ; -statement : [";"]? +statement : | [";"] | | [";"] @@ -29,7 +29,7 @@ | ; -block : ["{"] >statementliststatementliststatementlist< Modified: pypy/dist/pypy/lang/js/operations.py ============================================================================== --- pypy/dist/pypy/lang/js/operations.py (original) +++ pypy/dist/pypy/lang/js/operations.py Tue May 22 02:53:46 2007 @@ -91,6 +91,12 @@ def decision(self, ctx, op1, op2): raise NotImplementedError +class Undefined(Statement): + def execute(self, ctx): + return w_Undefined + +astundef = Undefined(Position()) + class PropertyInit(BinaryOp): pass @@ -138,6 +144,7 @@ class Block(Statement): def __init__(self, pos, nodes): + self.pos = pos self.nodes = nodes def execute(self, ctx): @@ -286,12 +293,10 @@ opcode = "THIS" class If(Statement): - opcode = 'IF' - - def __init__(self, pos, t): - self.condition = get_obj(t, 'condition') - self.thenPart = get_obj(t, 'thenPart') - self.elsePart = get_obj(t, 'elsePart') + def __init__(self, pos, condition, thenpart, elsepart=astundef): + self.condition = condition + self.thenPart = thenpart + self.elsePart = elsepart def execute(self, ctx): temp = self.condition.eval(ctx).GetValue() @@ -870,10 +875,6 @@ return W_String("undefined") return W_String(val.GetValue().type()) -class Undefined(Statement): - def execute(self, ctx): - return None - class VariableDeclaration(Expression): def __init__(self, pos, identifier, expr=None): self.pos = pos @@ -931,9 +932,10 @@ class WhileBase(Statement): - def __init__(self, pos, t): - self.condition = get_obj(t, 'condition') - self.body = get_obj(t, 'body') + def __init__(self, pos, condition, body): + self.pos = pos + self.condition = condition + self.body = body class Do(WhileBase): opcode = 'DO' @@ -956,8 +958,6 @@ continue class While(WhileBase): - opcode = 'WHILE' - def execute(self, ctx): while self.condition.eval(ctx).ToBoolean(): try: @@ -967,6 +967,7 @@ break elif e.type == 'continue': continue + class ForIn(Statement): def __init__(self, pos, t): @@ -1030,5 +1031,3 @@ def eval(self, ctx): return W_Number(+self.expr.eval(ctx).GetValue().ToNumber()) - -astundef = Undefined(Position()) Modified: pypy/dist/pypy/lang/js/test/test_interp.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_interp.py (original) +++ pypy/dist/pypy/lang/js/test/test_interp.py Tue May 22 02:53:46 2007 @@ -5,9 +5,9 @@ import py from pypy.lang.js import interpreter -from pypy.lang.js.jsparser import parse -from pypy.lang.js.interpreter import * -from pypy.lang.js.jsobj import W_Number, W_Object, ExecutionContext +from pypy.lang.js.operations import AEC, Number, Position, Plus +from pypy.lang.js.jsobj import W_Number, W_Object, \ + ExecutionContext, W_Root, ThrowException, w_Null def test_simple(): n1 = Number(Position(), 2.0) @@ -23,10 +23,10 @@ js_int = interpreter.Interpreter() try: if isinstance(code, str): - js_int.run(load_source(code)) + js_int.run(interpreter.load_source(code)) else: for codepiece in code: - js_int.run(load_source(codepiece)) + js_int.run(interpreter.load_source(codepiece)) except ThrowException, excpt: l.append("uncaught exception: "+str(excpt.exception.ToString())) print l, assval @@ -37,7 +37,7 @@ interpreter.writer = l.append jsint = interpreter.Interpreter() try: - jsint.run(load_source(code)) + jsint.run(interpreter.load_source(code)) except ThrowException, excpt: l.append("uncaught exception: "+str(excpt.exception.ToString())) print l, prints @@ -49,11 +49,13 @@ def assertv(code, value): jsint = interpreter.Interpreter() try: - code_val = jsint.run(load_source(code)).GetValue() + code_val = jsint.run(interpreter.load_source(code)).GetValue() except ThrowException, excpt: code_val = excpt print code_val, value - if isinstance(value, bool): + if isinstance(value, W_Root): + assert AEC(jsint.global_context, code_val, value) == True + elif isinstance(value, bool): assert code_val.ToBoolean() == value elif isinstance(value, int): assert code_val.ToInt32() == value @@ -61,11 +63,6 @@ assert code_val.ToNumber() == value else: assert code_val.ToString() == value - -def assert_result(code, result): - inter = interpreter.Interpreter() - r = inter.run(load_source(code)) - assert r.ToString() == result.ToString() def test_interp_parse(): yield assertv, "1+1;", 2 @@ -210,9 +207,8 @@ """, ["3"]) def test_block(): - py.test.skip("not ready yet") - assert_result("{ 5};", W_Number(5)) - assert_result("{3; 5};", W_Number(5)) + assertv("{5;}", W_Number(5)) + assertv("{3; 5;}", W_Number(5)) def test_try_catch_finally(): py.test.skip("not ready yet") @@ -229,7 +225,6 @@ """, ["3", "5"]) def test_if_then(): - py.test.skip("not ready yet") assertp(""" if (1) { print(1); @@ -237,42 +232,39 @@ """, "1") def test_if_then_else(): - py.test.skip("not ready yet") - assert_prints(""" + assertp(""" if (0) { print(1); } else { print(2); } - """, ["2"]) + """, "2") def test_compare(): - py.test.skip("not ready yet") - assert_prints("print(1>0);",["true"]) - assert_prints("print(0>1);",["false"]) - assert_prints("print(0>0);",["false"]) - assert_prints("print(1<0);",["false"]) - assert_prints("print(0<1);",["true"]) - assert_prints("print(0<0);",["false"]) - assert_prints("print(1>=0);",["true"]) - assert_prints("print(1>=1);",["true"]) - assert_prints("print(1>=2);",["false"]) - assert_prints("print(0<=1);",["true"]) - assert_prints("print(1<=1);",["true"]) - assert_prints("print(1<=0);",["false"]) - assert_prints("print(0==0);",["true"]) - assert_prints("print(1==1);",["true"]) - assert_prints("print(0==1);",["false"]) - assert_prints("print(0!=1);",["true"]) - assert_prints("print(1!=1);",["false"]) + yield assertv, "1>0;", True + yield assertv, "0>1;", False + yield assertv, "0>0;", False + yield assertv, "1<0;", False + yield assertv, "0<1;", True + yield assertv, "0<0;", False + yield assertv, "1>=0;", True + yield assertv, "1>=1;", True + yield assertv, "1>=2;", False + yield assertv, "0<=1;", True + yield assertv, "1<=1;", True + yield assertv, "1<=0;", False + yield assertv, "0==0;", True + yield assertv, "1==1;", True + yield assertv, "0==1;", False + yield assertv, "0!=1;", True + yield assertv, "1!=1;", False def test_binary_op(): - assert_prints("print(0||0); print(1||0);",["0", "1"]) - assert_prints("print(0&&1); print(1&&1);",["0", "1"]) + yield assertp, "print(0||0); print(1||0);", ["0", "1"] + yield assertp, "print(0&&1); print(1&&1);", ["0", "1"] def test_while(): - py.test.skip("not ready yet") - assert_prints(""" + assertp(""" i = 0; while (i<3) { print(i); @@ -373,11 +365,10 @@ print('out');""", ["out"]) def test_typeof(): - py.test.skip("not ready yet") - assert_result(""" + assertv(""" var x = 3; typeof x == 'number'; - """, W_Boolean(True)) + """, True) def test_semicolon(): assertp(';', []) @@ -451,7 +442,7 @@ assert_prints(""" "'t'"; """, []) def test_null(): - assert_result("null;", w_Null) + assertv("null;", w_Null) def test_void(): py.test.skip("not ready yet") From santagada at codespeak.net Tue May 22 06:55:42 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Tue, 22 May 2007 06:55:42 +0200 (CEST) Subject: [pypy-svn] r43550 - in pypy/dist/pypy/lang/js: . test Message-ID: <20070522045542.7C678806E@code0.codespeak.net> Author: santagada Date: Tue May 22 06:55:30 2007 New Revision: 43550 Modified: pypy/dist/pypy/lang/js/astbuilder.py pypy/dist/pypy/lang/js/jsgrammar.txt pypy/dist/pypy/lang/js/operations.py pypy/dist/pypy/lang/js/test/test_interp.py Log: made while and for types easier to build an AST of. some other details to make only 25 tests still not passing. Modified: pypy/dist/pypy/lang/js/astbuilder.py ============================================================================== --- pypy/dist/pypy/lang/js/astbuilder.py (original) +++ pypy/dist/pypy/lang/js/astbuilder.py Tue May 22 06:55:30 2007 @@ -1,4 +1,4 @@ -from pypy.rlib.parsing.tree import RPythonVisitor, Symbol +from pypy.rlib.parsing.tree import RPythonVisitor, Symbol, Nonterminal from pypy.lang.js import operations #this is a noop for now @@ -59,6 +59,8 @@ '++': operations.Increment, '--': operations.Decrement, 'typeof': operations.Typeof, + 'void': operations.Void, + 'delete': operations.Delete, } LISTOP_TO_CLS = { '[': operations.Array, @@ -71,7 +73,7 @@ if isinstance(node, Symbol): value = node.additional_info source_pos = node.token.source_pos - else: + elif len(node.children) > 0: curr = node.children[0] while not isinstance(curr, Symbol): if len(curr.children): @@ -242,6 +244,7 @@ atype = node.children[1].additional_info right = self.dispatch(node.children[2]) return operations.Assignment(pos, left, right, atype) + visit_assignmentexpressionnoin = visit_assignmentexpression def visit_functiondeclaration(self, node): pos = self.get_pos(node) @@ -268,8 +271,38 @@ return operations.If(pos, condition, ifblock, elseblock) def visit_iterationstatement(self, node): + return self.dispatch(node.children[0]) + + def visit_whiles(self, node): pos = self.get_pos(node) - if node.children[0].additional_info == 'while': + itertype = node.children[0].additional_info + if itertype == 'while': condition = self.dispatch(node.children[1]) block = self.dispatch(node.children[2]) return operations.While(pos, condition, block) + elif itertype == "do": + pass + + def visit_regularfor(self, node): + pos = self.get_pos(node) + i = 1 + setup, i = self.get_next_expr(node, i) + condition, i = self.get_next_expr(node, i) + update, i = self.get_next_expr(node, i) + body, i = self.get_next_expr(node, i) + return operations.For(pos, setup, condition, update, body) + + def get_next_expr(self, node, i): + if isinstance(node.children[i], Symbol) and \ + node.children[i].additional_info in [';', ')']: + return operations.astundef, i+1 + else: + return self.dispatch(node.children[i]), i+2 + + def visit_breakstatement(self, node): + pos = self.get_pos(node) + if len(node.children) > 0: + target = self.dispatch(node.children[0]) + else: + target = operations.astundef + return operations.Break(pos, target) Modified: pypy/dist/pypy/lang/js/jsgrammar.txt ============================================================================== --- pypy/dist/pypy/lang/js/jsgrammar.txt (original) +++ pypy/dist/pypy/lang/js/jsgrammar.txt Tue May 22 06:55:30 2007 @@ -60,13 +60,29 @@ ; iterationstatement + : whiles + | regularfor + | regularvarfor + | infor + | invarfor + ; + +whiles : "do" statement ["while"] ["("] expression [")"] | "while" ["("] expression [")"] statement - | "for" ["("] expressionnoin? ";" expression? ";" expression? ")" statement - | "for" ["("] ["var"] variabledeclarationlistnoin ";" expression? ";" expression? ")" statement - | "for" ["("] lefthandsideexpression "in" expression ")" statement - | "for" ["("] ["var"] variabledeclarationnoin "in" expression ")" statement - ; + ; + +regularfor + : "for" ["("] expressionnoin? ";" expression? ";" expression? ")" statement; + +regularvarfor + : "for" ["("] ["var"] variabledeclarationlistnoin ";" expression? ";" expression? ")" statement; + +infor + : "for" ["("] lefthandsideexpression ["in"] expression [")"] statement; + +invarfor + : "for" ["("] ["var"] variabledeclarationnoin ["in"] expression [")"] statement; continuestatement : ["continue"] identifier? @@ -316,11 +332,11 @@ #noin copy relationalopnoin: "<" | ">" | "<=" | ">=" | "instanceof"; -relationalexpressionnoin : shiftexpression (relationalopnoin shiftexpression)+ +relationalexpressionnoin : shiftexpression (>relationalopnoin< shiftexpression)+ | ; -equalityexpressionnoin : relationalexpressionnoin (equalityop equalityexpressionnoin)+ +equalityexpressionnoin : relationalexpressionnoin (>equalityop< equalityexpressionnoin)+ | ; @@ -351,7 +367,7 @@ ; #lefthandside -assignmentexpressionnoin : lefthandsideexpression assignmentoperator assignmentexpressionnoin +assignmentexpressionnoin : lefthandsideexpression >assignmentoperator< assignmentexpressionnoin | ; Modified: pypy/dist/pypy/lang/js/operations.py ============================================================================== --- pypy/dist/pypy/lang/js/operations.py (original) +++ pypy/dist/pypy/lang/js/operations.py Tue May 22 06:55:30 2007 @@ -92,6 +92,9 @@ raise NotImplementedError class Undefined(Statement): + def eval(self, ctx): + return w_Undefined + def execute(self, ctx): return w_Undefined @@ -182,23 +185,19 @@ class Unconditional(Statement): - def __init__(self, pos, t): - pieces = get_string(t, 'target').split(',') - self.targtype = pieces[0] - self.targlineno = pieces[1] - self.targstart = pieces[2] - -class Break(Unconditional): - opcode = 'BREAK' + def __init__(self, pos, target): + self.pos = pos + self.target = target +class Break(Unconditional): def execute(self, ctx): raise ExecutionReturned('break', None, None) + class Continue(Unconditional): - opcode = 'CONTINUE' - def execute(self, ctx): raise ExecutionReturned('continue', None, None) + class Call(BinaryOp): def eval(self, ctx): @@ -560,8 +559,6 @@ """ the delete op, erases properties from objects """ - opcode = 'DELETE' - def eval(self, ctx): r1 = self.expr.eval(ctx) if not isinstance(r1, W_Reference): @@ -569,6 +566,7 @@ r3 = r1.GetBase() r4 = r1.GetPropertyName() return W_Boolean(r3.Delete(r4)) + class Increment(UnaryOp): """ @@ -907,11 +905,10 @@ return self.body.eval(ctx) class Void(UnaryOp): - opcode = 'VOID' - def eval(self, ctx): self.expr.eval(ctx) return w_Undefined + class With(Statement): opcode = "WITH" @@ -991,11 +988,12 @@ continue class For(Statement): - def __init__(self, pos, t): - self.setup = get_obj(t, 'setup') - self.condition = get_obj(t, 'condition') - self.update = get_obj(t, 'update') - self.body = get_obj(t, 'body') + def __init__(self, pos, setup, condition, update, body): + self.pos = pos + self.setup = setup + self.condition = condition + self.update = update + self.body = body def execute(self, ctx): self.setup.eval(ctx).GetValue() Modified: pypy/dist/pypy/lang/js/test/test_interp.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_interp.py (original) +++ pypy/dist/pypy/lang/js/test/test_interp.py Tue May 22 06:55:30 2007 @@ -106,8 +106,8 @@ assert_prints('function x() { return; };', []) def test_var_declaration(): - assert_prints('var x = 3; print(x);', ["3"]) - assert_prints('var x = 3; print(x+x);', ["6"]) + yield assertv, 'var x = 3; x;', 3 + yield assertv, 'var x = 3; x+x;', 6 def test_var_scoping(): py.test.skip("not ready yet") @@ -334,9 +334,9 @@ ,["3", "2"]) def test_for(): - py.test.skip("not ready yet") - assert_prints(""" - for (i=0; i<3; i++) { + assertp(""" + i = 0; + for (i; i<3; i++) { print(i); } print(i); @@ -354,15 +354,14 @@ x.length == 0;""", 'true') def test_break(): - py.test.skip("not ready yet") - assert_prints(""" + assertp(""" while(1){ break; } for(x=0;1==1;x++) { break; } - print('out');""", ["out"]) + print('out');""", "out") def test_typeof(): assertv(""" @@ -445,8 +444,7 @@ assertv("null;", w_Null) def test_void(): - py.test.skip("not ready yet") - assert_prints("print(void print('hello'));", + assertp("print(void print('hello'));", ["hello", "undefined"]) def test_activationprob(): @@ -491,13 +489,12 @@ assertp("print(+1);", '1') def test_delete(): - py.test.skip("not ready yet") - assert_prints(""" + assertp(""" var x = {}; x.y = 1; delete x.y; print(x.y); - """, ['undefined']) + """, 'undefined') def test_forin(): py.test.skip("not ready yet") From fijal at codespeak.net Tue May 22 10:56:49 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 22 May 2007 10:56:49 +0200 (CEST) Subject: [pypy-svn] r43551 - pypy/branch/kill-ctypes/pypy/translator/c Message-ID: <20070522085649.33B6B806E@code0.codespeak.net> Author: fijal Date: Tue May 22 10:56:47 2007 New Revision: 43551 Modified: pypy/branch/kill-ctypes/pypy/translator/c/external.py pypy/branch/kill-ctypes/pypy/translator/c/genc.py pypy/branch/kill-ctypes/pypy/translator/c/node.py Log: Simple wacking around to get more C sources compiled Modified: pypy/branch/kill-ctypes/pypy/translator/c/external.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/translator/c/external.py (original) +++ pypy/branch/kill-ctypes/pypy/translator/c/external.py Tue May 22 10:56:47 2007 @@ -3,7 +3,6 @@ from pypy.translator.c.support import USESLOTS # set to False if necessary while refactoring from pypy.translator.c.support import cdecl, somelettersfrom - class CExternalFunctionCodeGenerator(object): if USESLOTS: __slots__ = """db fnptr FUNCTYPE argtypenames resulttypename""".split() Modified: pypy/branch/kill-ctypes/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/translator/c/genc.py (original) +++ pypy/branch/kill-ctypes/pypy/translator/c/genc.py Tue May 22 10:56:47 2007 @@ -603,6 +603,22 @@ print >> f, '\treturn error;' print >> f, '}' +def extra_information(database): + includes = {} + sources = {} + for node in database.globalcontainers(): + if hasattr(node, 'includes'): + for include in node.includes: + includes[include] = True + if hasattr(node, 'sources'): + for source in node.sources: + sources[source] = True + includes = includes.keys() + includes.sort() + sources = sources.keys() + sources.sort() + return includes, sources + def gen_source_standalone(database, modulename, targetdir, entrypointname, defines={}): assert database.standalone @@ -632,13 +648,7 @@ for line in database.gcpolicy.pre_gc_code(): print >> fi, line - includes = {} - for node in database.globalcontainers(): - if hasattr(node, 'includes'): - for include in node.includes: - includes[include] = True - includes = includes.keys() - includes.sort() + includes, sources = extra_information(database) for include in includes: print >> fi, '#include <%s>' % (include,) fi.close() @@ -665,8 +675,8 @@ n = database.instrument_ncounter print >>fi, "#define INSTRUMENT_NCOUNTER %d" % n fi.close() - - return filename, sg.getextrafiles() + + return filename, sg.getextrafiles() + sources def gen_source(database, modulename, targetdir, defines={}, exports={}, @@ -696,13 +706,7 @@ for line in database.gcpolicy.pre_gc_code(): print >> fi, line - includes = {} - for node in database.globalcontainers(): - if hasattr(node, 'includes'): - for include in node.includes: - includes[include] = True - includes = includes.keys() - includes.sort() + includes, sources = extra_information(database) for include in includes: print >> fi, '#include <%s>' % (include,) fi.close() @@ -833,7 +837,7 @@ f.write(SETUP_PY % locals()) f.close() - return filename, sg.getextrafiles() + return filename, sg.getextrafiles() + sources SETUP_PY = ''' Modified: pypy/branch/kill-ctypes/pypy/translator/c/node.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/translator/c/node.py (original) +++ pypy/branch/kill-ctypes/pypy/translator/c/node.py Tue May 22 10:56:47 2007 @@ -619,6 +619,8 @@ self.ptrname = self.name def make_funcgens(self): + if hasattr(self.obj, 'sources'): + self.sources = self.obj.sources self.funcgens = select_function_code_generators(self.obj, self.db, self.name) if self.funcgens: argnames = self.funcgens[0].argnames() #Assume identical for all funcgens From fijal at codespeak.net Tue May 22 10:57:41 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 22 May 2007 10:57:41 +0200 (CEST) Subject: [pypy-svn] r43552 - in pypy/branch/kill-ctypes/pypy/rpython/lltypesystem: . test Message-ID: <20070522085741.BA7DA806E@code0.codespeak.net> Author: fijal Date: Tue May 22 10:57:41 2007 New Revision: 43552 Added: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rffi.py Log: A simple wrapper to allow easy defining function as external Added: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- (empty file) +++ pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py Tue May 22 10:57:41 2007 @@ -0,0 +1,9 @@ + +from pypy.rpython.lltypesystem.lltype import functionptr, FuncType + +def llexternal(name, args, result, sources=[], includes=[]): + ext_type = FuncType(args, result) + return functionptr(ext_type, name, external='C', sources=tuple(sources), + includes=tuple(includes)) + + Added: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rffi.py ============================================================================== --- (empty file) +++ pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rffi.py Tue May 22 10:57:41 2007 @@ -0,0 +1,38 @@ + +from pypy.rpython.lltypesystem.rffi import llexternal +from pypy.translator.c.test.test_genc import compile +from pypy.rpython.lltypesystem.lltype import Signed +from pypy.tool.udir import udir + +def test_basic(): + c_source = """ + int z(int x) + { + return (x + 3); + } + """ + c_file = udir.join("stuff.c") + c_file.write(c_source) + z = llexternal('z', [Signed], Signed, sources=[str(c_file)]) + + def f(): + return z(8) + + xf = compile(f, []) + assert xf() == 8+3 + +def test_hashdefine(): + c_source = """ + #define X(i) (i+3) + """ + + c_file = udir.join("stuff.c") + c_file.write(c_source) + + z = llexternal('X', [Signed], Signed, includes=[str(c_file)]) + + def f(): + return z(8) + + xf = compile(f, []) + assert xf() == 8+3 From cfbolz at codespeak.net Tue May 22 11:02:24 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 22 May 2007 11:02:24 +0200 (CEST) Subject: [pypy-svn] r43553 - pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator Message-ID: <20070522090224.BAD6D8071@code0.codespeak.net> Author: cfbolz Date: Tue May 22 11:02:23 2007 New Revision: 43553 Modified: pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator/bookkeeper.py Log: missed one Modified: pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator/bookkeeper.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator/bookkeeper.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator/bookkeeper.py Tue May 22 11:02:23 2007 @@ -1,7 +1,7 @@ import py from pypy.tool.tls import tlsobject from pypy.tool.ansi_print import ansi_log -from pypy.rlib import objectmodel +from pypy.rlib import jit from pypy.objspace.flow.model import copygraph, SpaceOperation, Constant from pypy.objspace.flow.model import Variable, Block, Link, FunctionGraph from pypy.annotation import model as annmodel @@ -15,7 +15,7 @@ log = py.log.Producer("hintannotate") py.log.setconsumer("hintannotate", ansi_log) -TIMESHIFTMAP = {Constant(objectmodel._we_are_jitted): +TIMESHIFTMAP = {Constant(jit._we_are_jitted): Constant(1, lltype.Signed)} class GraphDesc(object): From cfbolz at codespeak.net Tue May 22 11:12:39 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 22 May 2007 11:12:39 +0200 (CEST) Subject: [pypy-svn] r43554 - pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator/test Message-ID: <20070522091239.A5FCB807B@code0.codespeak.net> Author: cfbolz Date: Tue May 22 11:12:33 2007 New Revision: 43554 Modified: pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator/test/test_annotator.py Log: another missed import :-( Modified: pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator/test/test_annotator.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator/test/test_annotator.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator/test/test_annotator.py Tue May 22 11:12:33 2007 @@ -5,7 +5,7 @@ from pypy.jit.hintannotator.bookkeeper import HintBookkeeper from pypy.jit.hintannotator.model import * from pypy.rpython.lltypesystem import lltype -from pypy.rlib.objectmodel import hint, we_are_jitted +from pypy.rlib.jit import hint, we_are_jitted from pypy.annotation import model as annmodel from pypy.objspace.flow import model as flowmodel from pypy.translator.backendopt.inline import auto_inlining From cfbolz at codespeak.net Tue May 22 11:24:32 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 22 May 2007 11:24:32 +0200 (CEST) Subject: [pypy-svn] r43555 - pypy/branch/prolog-jit-experiments/pypy/rlib Message-ID: <20070522092432.B10088082@code0.codespeak.net> Author: cfbolz Date: Tue May 22 11:24:31 2007 New Revision: 43555 Modified: pypy/branch/prolog-jit-experiments/pypy/rlib/jit.py Log: missing import Modified: pypy/branch/prolog-jit-experiments/pypy/rlib/jit.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/rlib/jit.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/rlib/jit.py Tue May 22 11:24:31 2007 @@ -1,4 +1,5 @@ from pypy.rpython.extregistry import ExtRegistryEntry +from pypy.rlib.objectmodel import CDefinedIntSymbolic def purefunction(func): func._pure_function_ = True From cfbolz at codespeak.net Tue May 22 11:30:01 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 22 May 2007 11:30:01 +0200 (CEST) Subject: [pypy-svn] r43556 - in pypy/branch/prolog-jit-experiments/pypy: jit/hintannotator jit/hintannotator/test jit/timeshifter/test jit/tl module/pypyjit Message-ID: <20070522093001.3306A8084@code0.codespeak.net> Author: cfbolz Date: Tue May 22 11:30:00 2007 New Revision: 43556 Added: pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator/policy.py (contents, props changed) Modified: pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator/annotator.py pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator/test/test_annotator.py pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_timeshift.py pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_vdict.py pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_vlist.py pypy/branch/prolog-jit-experiments/pypy/jit/tl/targettiny1.py pypy/branch/prolog-jit-experiments/pypy/jit/tl/targettiny2.py pypy/branch/prolog-jit-experiments/pypy/module/pypyjit/portal.py Log: factor out common parts of the PyrologHintAnnotatorPolicy and the PyPyHintAnnotatorPolicy Modified: pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator/annotator.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator/annotator.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator/annotator.py Tue May 22 11:30:00 2007 @@ -1,49 +1,13 @@ -from pypy.annotation import policy from pypy.annotation import model as annmodel from pypy.annotation.annrpython import RPythonAnnotator, BlockedInference from pypy.annotation.annrpython import raise_nicer_exception from pypy.objspace.flow.model import Variable from pypy.jit.hintannotator import model as hintmodel from pypy.jit.hintannotator.bookkeeper import HintBookkeeper +from pypy.jit.hintannotator.policy import HintAnnotatorPolicy from pypy.rpython.lltypesystem import lltype -class HintAnnotatorPolicy(policy.AnnotatorPolicy): - novirtualcontainer = False - oopspec = False - entrypoint_returns_red = True - - def __init__(self, novirtualcontainer = None, - oopspec = None, - entrypoint_returns_red = None): - if novirtualcontainer is not None: - self.novirtualcontainer = novirtualcontainer - if oopspec is not None: - self.oopspec = oopspec - if entrypoint_returns_red is not None: - self.entrypoint_returns_red = entrypoint_returns_red - - def look_inside_graph(self, graph): - return True - - -class StopAtXPolicy(HintAnnotatorPolicy): - """Useful for tests.""" - novirtualcontainer = True - oopspec = True - - def __init__(self, *funcs): - self.funcs = funcs - - def look_inside_graph(self, graph): - try: - if graph.func in self.funcs: - return False - except AttributeError: - pass - return True - - class HintAnnotator(RPythonAnnotator): def __init__(self, translator=None, base_translator=None, policy=None): Added: pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator/policy.py ============================================================================== --- (empty file) +++ pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator/policy.py Tue May 22 11:30:00 2007 @@ -0,0 +1,146 @@ +from pypy.annotation import policy +from pypy.annotation.specialize import getuniquenondirectgraph + +class HintAnnotatorPolicy(policy.AnnotatorPolicy): + novirtualcontainer = False + oopspec = False + entrypoint_returns_red = True + + def __init__(self, novirtualcontainer = None, + oopspec = None, + entrypoint_returns_red = None): + if novirtualcontainer is not None: + self.novirtualcontainer = novirtualcontainer + if oopspec is not None: + self.oopspec = oopspec + if entrypoint_returns_red is not None: + self.entrypoint_returns_red = entrypoint_returns_red + + def look_inside_graph(self, graph): + return True + + +class StopAtXPolicy(HintAnnotatorPolicy): + """Useful for tests.""" + novirtualcontainer = True + oopspec = True + + def __init__(self, *funcs): + self.funcs = funcs + + def look_inside_graph(self, graph): + try: + if graph.func in self.funcs: + return False + except AttributeError: + pass + return True + +class ManualGraphPolicy(HintAnnotatorPolicy): + novirtualcontainer = True + oopspec = True + opaquepurefunctions = False + + def seetranslator(self, t): + if self.opaquepurefunctions: + from pypy.jit.hintannotator.bookkeeper import ImpurityAnalyzer + self.analyzer = ImpurityAnalyzer(t) + self.translator = t + self.bookkeeper = t.annotator.bookkeeper + self.timeshift_graphs = {} + portal = getattr(PORTAL, 'im_func', PORTAL) + portal_graph = graphof(t, portal) + self.fill_timeshift_graphs(t, portal_graph) + + def look_inside_graph(self, graph): + if graph in self.timeshift_graphs: + return self.timeshift_graphs[graph] + # don't look into pure functions + if (self.opaquepurefunctions and + not self.analyzer.analyze_direct_call(graph)): + return False + try: + func = graph.func + except AttributeError: + return True + if hasattr(func, '_look_inside_me_'): + return func._look_inside_me_ + # explicitly pure functions are always opaque + if getattr(func, '_pure_function_', False): + return False + mod = func.__module__ or '?' + return self.look_inside_graph_of_module(graph, mod) + + def look_inside_graph_of_module(self, graph, func, mod): + return True + + def fill_timeshift_graphs(self, t, portal_graph): + # subclasses should have their own + pass + + def _graph(self, func): + func = getattr(func, 'im_func', func) + desc = self.bookkeeper.getdesc(func) + return getuniquenondirectgraph(desc) + + def seefunc(self, fromfunc, *tofuncs): + targetgraphs = {} + for tofunc in tofuncs: + targetgraphs[_graph(tofunc)] = True + graphs = graphs_on_the_path_to(self.translator, self._graph(fromfunc), + targetgraphs) + for graph in graphs: + result_graphs[graph] = True + + def seepath(self, *path): + for i in range(1, len(path)): + self.seefunc(path[i-1], path[i]) + + def seegraph(self, func, look=True): + graph = _graph(func) + result_graphs[graph] = look + + + +def enumerate_reachable_graphs(translator, startgraph): + from pypy.translator.backendopt.support import find_calls_from + pending = [(startgraph, None)] + yield pending[0] + seen = {startgraph: True} + while pending: + yield None # hack: a separator meaning "length increases now" + nextlengthlist = [] + nextseen = {} + for node in pending: + head, tail = node + for block, callee in find_calls_from(translator, head): + if callee not in seen: + newnode = callee, node + yield newnode + nextlengthlist.append(newnode) + nextseen[callee] = True + pending = nextlengthlist + seen.update(nextseen) + yield None + +def graphs_on_the_path_to(translator, startgraph, targetgraphs): + targetgraphs = targetgraphs.copy() + result = {} + found = {} + for node in enumerate_reachable_graphs(translator, startgraph): + if node is None: # hack: a separator meaning "length increases now" + for graph in found: + del targetgraphs[graph] + found.clear() + if not targetgraphs: + return result + elif node[0] in targetgraphs: + found[node[0]] = True + while node is not None: + head, tail = node + result[head] = True + node = tail + raise Exception("did not reach all targets:\nmissing %r" % ( + targetgraphs.keys(),)) + + Modified: pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator/test/test_annotator.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator/test/test_annotator.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator/test/test_annotator.py Tue May 22 11:30:00 2007 @@ -1,7 +1,7 @@ import py from pypy.translator.translator import TranslationContext, graphof -from pypy.jit.hintannotator.annotator import HintAnnotator, HintAnnotatorPolicy -from pypy.jit.hintannotator.annotator import StopAtXPolicy +from pypy.jit.hintannotator.annotator import HintAnnotator +from pypy.jit.hintannotator.policy import StopAtXPolicy, HintAnnotatorPolicy from pypy.jit.hintannotator.bookkeeper import HintBookkeeper from pypy.jit.hintannotator.model import * from pypy.rpython.lltypesystem import lltype Modified: pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_timeshift.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_timeshift.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_timeshift.py Tue May 22 11:30:00 2007 @@ -1,8 +1,8 @@ import py import sys from pypy.translator.translator import TranslationContext, graphof -from pypy.jit.hintannotator.annotator import HintAnnotator, HintAnnotatorPolicy -from pypy.jit.hintannotator.annotator import StopAtXPolicy +from pypy.jit.hintannotator.annotator import HintAnnotator +from pypy.jit.hintannotator.policy import StopAtXPolicy, HintAnnotatorPolicy from pypy.jit.hintannotator.bookkeeper import HintBookkeeper from pypy.jit.hintannotator.model import * from pypy.jit.timeshifter.hrtyper import HintRTyper, originalconcretetype Modified: pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_vdict.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_vdict.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_vdict.py Tue May 22 11:30:00 2007 @@ -1,4 +1,4 @@ -from pypy.jit.hintannotator.annotator import HintAnnotatorPolicy +from pypy.jit.hintannotator.policy import HintAnnotatorPolicy from pypy.jit.timeshifter.test.test_timeshift import TimeshiftingTests from pypy.rlib.jit import hint Modified: pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_vlist.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_vlist.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_vlist.py Tue May 22 11:30:00 2007 @@ -1,4 +1,4 @@ -from pypy.jit.hintannotator.annotator import HintAnnotatorPolicy +from pypy.jit.hintannotator.policy import HintAnnotatorPolicy from pypy.jit.timeshifter.test.test_timeshift import TimeshiftingTests from pypy.rlib.jit import hint Modified: pypy/branch/prolog-jit-experiments/pypy/jit/tl/targettiny1.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/jit/tl/targettiny1.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/jit/tl/targettiny1.py Tue May 22 11:30:00 2007 @@ -24,7 +24,7 @@ # ____________________________________________________________ -from pypy.jit.hintannotator.annotator import HintAnnotatorPolicy +from pypy.jit.hintannotator.policy import HintAnnotatorPolicy class MyHintAnnotatorPolicy(HintAnnotatorPolicy): novirtualcontainer = True Modified: pypy/branch/prolog-jit-experiments/pypy/jit/tl/targettiny2.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/jit/tl/targettiny2.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/jit/tl/targettiny2.py Tue May 22 11:30:00 2007 @@ -24,7 +24,7 @@ # ____________________________________________________________ -from pypy.jit.hintannotator.annotator import HintAnnotatorPolicy +from pypy.jit.hintannotator.policy import HintAnnotatorPolicy class MyHintAnnotatorPolicy(HintAnnotatorPolicy): novirtualcontainer = True Modified: pypy/branch/prolog-jit-experiments/pypy/module/pypyjit/portal.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/module/pypyjit/portal.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/module/pypyjit/portal.py Tue May 22 11:30:00 2007 @@ -2,23 +2,10 @@ from pypy.module.pypyjit.newbool import NewBoolDesc from pypy.translator.translator import graphof from pypy.annotation.specialize import getuniquenondirectgraph -from pypy.jit.hintannotator.annotator import HintAnnotatorPolicy +from pypy.jit.hintannotator.policy import ManualGraphPolicy -class PyPyHintAnnotatorPolicy(HintAnnotatorPolicy): - novirtualcontainer = True - oopspec = True - - def __init__(self, timeshift_graphs): - self.timeshift_graphs = timeshift_graphs - - def look_inside_graph(self, graph): - if graph in self.timeshift_graphs: - return self.timeshift_graphs[graph] - try: - func = graph.func - except AttributeError: - return True - mod = func.__module__ or '?' +class PyPyHintAnnotatorPolicy(ManualGraphPolicy): + def look_inside_graph_of_module(self, graph, func, module): if mod.startswith('pypy.objspace'): return False if '_geninterp_' in func.func_globals: # skip all geninterped stuff @@ -38,167 +25,99 @@ return False return True -forbidden_modules = {'pypy.interpreter.gateway': True, - #'pypy.interpreter.baseobjspace': True, - 'pypy.interpreter.typedef': True, - 'pypy.interpreter.eval': True, - 'pypy.interpreter.function': True, - 'pypy.interpreter.pytraceback': True, - } - -def enumerate_reachable_graphs(translator, startgraph): - from pypy.translator.backendopt.support import find_calls_from - pending = [(startgraph, None)] - yield pending[0] - seen = {startgraph: True} - while pending: - yield None # hack: a separator meaning "length increases now" - nextlengthlist = [] - nextseen = {} - for node in pending: - head, tail = node - for block, callee in find_calls_from(translator, head): - if callee not in seen: - newnode = callee, node - yield newnode - nextlengthlist.append(newnode) - nextseen[callee] = True - pending = nextlengthlist - seen.update(nextseen) - yield None - -def graphs_on_the_path_to(translator, startgraph, targetgraphs): - targetgraphs = targetgraphs.copy() - result = {} - found = {} - for node in enumerate_reachable_graphs(translator, startgraph): - if node is None: # hack: a separator meaning "length increases now" - for graph in found: - del targetgraphs[graph] - found.clear() - if not targetgraphs: - return result - elif node[0] in targetgraphs: - found[node[0]] = True - while node is not None: - head, tail = node - result[head] = True - node = tail - raise Exception("did not reach all targets:\nmissing %r" % ( - targetgraphs.keys(),)) - - -def timeshift_graphs(t, portal_graph, log): - import pypy - result_graphs = {} - - bk = t.annotator.bookkeeper - - def _graph(func): - func = getattr(func, 'im_func', func) - desc = bk.getdesc(func) - return getuniquenondirectgraph(desc) - - def seefunc(fromfunc, *tofuncs): - targetgraphs = {} - for tofunc in tofuncs: - targetgraphs[_graph(tofunc)] = True - graphs = graphs_on_the_path_to(t, _graph(fromfunc), targetgraphs) - for graph in graphs: - if graph not in result_graphs: - log('including graph %s' % (graph,)) - result_graphs[graph] = True - - def seepath(*path): - for i in range(1, len(path)): - seefunc(path[i-1], path[i]) - - def seegraph(func, look=True): - graph = _graph(func) - if look: - extra = "" - if look != True: - extra = " substituted with %s" % look - log('including graph %s%s' % (graph, extra)) - else: - log('excluding graph %s' % (graph,)) - result_graphs[graph] = look - - def seebinary(opname): + def seebinary(self, opname): name2 = name1 = opname[:3].lower() if name1 in ('and', 'or'): name1 += '_' - descr_impl = getattr(pypy.objspace.descroperation.DescrOperation, name1) + descr_impl = getattr( + pypy.objspace.descroperation.DescrOperation, name1) obj_impl = getattr(pypy.objspace.std.intobject, name2 + '__Int_Int') - seepath(getattr(pypy.interpreter.pyframe.PyFrame, 'BINARY_'+ opname), - descr_impl, - obj_impl) - seepath(descr_impl, - pypy.objspace.std.typeobject.W_TypeObject.is_heaptype) + self.seepath( + getattr(pypy.interpreter.pyframe.PyFrame, 'BINARY_'+ opname), + descr_impl, + obj_impl) + self.seepath(descr_impl, + pypy.objspace.std.typeobject.W_TypeObject.is_heaptype) descr_impl = getattr(pypy.objspace.descroperation.DescrOperation, 'inplace_' + name2) - seepath(getattr(pypy.interpreter.pyframe.PyFrame, 'INPLACE_'+ opname), - descr_impl, - obj_impl) - seepath(descr_impl, - pypy.objspace.std.typeobject.W_TypeObject.is_heaptype) + self.seepath( + getattr(pypy.interpreter.pyframe.PyFrame, 'INPLACE_'+ opname), + descr_impl, + obj_impl) + self.seepath(descr_impl, + pypy.objspace.std.typeobject.W_TypeObject.is_heaptype) - def seeunary(opname, name=None): + def seeunary(self, opname, name=None): if name is None: name = opname.lower() - descr_impl = getattr(pypy.objspace.descroperation.DescrOperation, name) - seepath(getattr(pypy.interpreter.pyframe.PyFrame, 'UNARY_' + opname), + descr_impl = getattr( + pypy.objspace.descroperation.DescrOperation, name) + self.seepath( + getattr(pypy.interpreter.pyframe.PyFrame, 'UNARY_' + opname), descr_impl, getattr(pypy.objspace.std.intobject, name + '__Int')) - seepath(descr_impl, + self.seepath(descr_impl, pypy.objspace.std.typeobject.W_TypeObject.is_heaptype) - def seecmp(name): + def seecmp(self, name): descr_impl = getattr(pypy.objspace.descroperation.DescrOperation, name) - seepath(pypy.interpreter.pyframe.PyFrame.COMPARE_OP, + self.seepath( + pypy.interpreter.pyframe.PyFrame.COMPARE_OP, descr_impl, getattr(pypy.objspace.std.intobject, name +'__Int_Int'), pypy.objspace.std.Space.newbool) - seepath(descr_impl, + self.seepath( + descr_impl, pypy.objspace.std.typeobject.W_TypeObject.is_heaptype) - # -------------------- - for binop in 'ADD SUBTRACT MULTIPLY AND OR XOR'.split(): - seebinary(binop) - for cmpname in 'lt le eq ne ge gt'.split(): - seecmp(cmpname) - seepath(pypy.interpreter.pyframe.PyFrame.UNARY_NOT, - pypy.objspace.std.Space.not_) - seeunary('INVERT') - seeunary('POSITIVE', 'pos') - seeunary('NEGATIVE', 'neg') - - seepath(pypy.objspace.descroperation._invoke_binop, - pypy.objspace.descroperation._check_notimplemented) - seepath(pypy.objspace.std.intobject.add__Int_Int, - pypy.objspace.std.inttype.wrapint, - pypy.objspace.std.intobject.W_IntObject.__init__) - seepath(pypy.objspace.descroperation.DescrOperation.add, - pypy.objspace.std.Space.type, - pypy.objspace.std.Space.gettypeobject) - seepath(pypy.objspace.descroperation.DescrOperation.add, - pypy.objspace.std.Space.is_w) - seegraph(pypy.interpreter.pyframe.PyFrame.execute_frame, False) - # -------------------- - # special timeshifting logic for newbool - seegraph(pypy.objspace.std.Space.newbool, NewBoolDesc) - seepath(pypy.interpreter.pyframe.PyFrame.JUMP_IF_TRUE, - pypy.objspace.std.Space.is_true) - seepath(pypy.interpreter.pyframe.PyFrame.JUMP_IF_FALSE, - pypy.objspace.std.Space.is_true) - - # - seepath(pypy.interpreter.pyframe.PyFrame.CALL_FUNCTION, - pypy.interpreter.function.Function.funccall_valuestack) - seepath(pypy.interpreter.pyframe.PyFrame.CALL_FUNCTION, - pypy.interpreter.function.Function.funccall_obj_valuestack) + def fill_timeshift_graphs(self, portal_graph): + import pypy + + # -------------------- + for binop in 'ADD SUBTRACT MULTIPLY AND OR XOR'.split(): + self.seebinary(binop) + for cmpname in 'lt le eq ne ge gt'.split(): + self.seecmp(cmpname) + self.seepath(pypy.interpreter.pyframe.PyFrame.UNARY_NOT, + pypy.objspace.std.Space.not_) + self.seeunary('INVERT') + self.seeunary('POSITIVE', 'pos') + self.seeunary('NEGATIVE', 'neg') + + self.seepath(pypy.objspace.descroperation._invoke_binop, + pypy.objspace.descroperation._check_notimplemented) + self.seepath(pypy.objspace.std.intobject.add__Int_Int, + pypy.objspace.std.inttype.wrapint, + pypy.objspace.std.intobject.W_IntObject.__init__) + self.seepath(pypy.objspace.descroperation.DescrOperation.add, + pypy.objspace.std.Space.type, + pypy.objspace.std.Space.gettypeobject) + self.seepath(pypy.objspace.descroperation.DescrOperation.add, + pypy.objspace.std.Space.is_w) + self.seegraph(pypy.interpreter.pyframe.PyFrame.execute_frame, False) + # -------------------- + # special timeshifting logic for newbool + self.seegraph(pypy.objspace.std.Space.newbool, NewBoolDesc) + self.seepath(pypy.interpreter.pyframe.PyFrame.JUMP_IF_TRUE, + pypy.objspace.std.Space.is_true) + self.seepath(pypy.interpreter.pyframe.PyFrame.JUMP_IF_FALSE, + pypy.objspace.std.Space.is_true) + + # + self.seepath(pypy.interpreter.pyframe.PyFrame.CALL_FUNCTION, + pypy.interpreter.function.Function.funccall_valuestack) + self.seepath(pypy.interpreter.pyframe.PyFrame.CALL_FUNCTION, + pypy.interpreter.function.Function.funccall_obj_valuestack) - return result_graphs + + +forbidden_modules = {'pypy.interpreter.gateway': True, + #'pypy.interpreter.baseobjspace': True, + 'pypy.interpreter.typedef': True, + 'pypy.interpreter.eval': True, + 'pypy.interpreter.function': True, + 'pypy.interpreter.pytraceback': True, + } def get_portal(drv): From cfbolz at codespeak.net Tue May 22 11:31:04 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 22 May 2007 11:31:04 +0200 (CEST) Subject: [pypy-svn] r43557 - pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter Message-ID: <20070522093104.80E948084@code0.codespeak.net> Author: cfbolz Date: Tue May 22 11:31:03 2007 New Revision: 43557 Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/portal.py Log: make prolog use the new facilities too Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/portal.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/portal.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/portal.py Tue May 22 11:31:03 2007 @@ -1,4 +1,4 @@ -from pypy.jit.hintannotator.annotator import HintAnnotatorPolicy +from pypy.jit.hintannotator.policy import ManualGraphPolicy from pypy.lang.prolog.interpreter import term, engine from pypy.translator.translator import graphof from pypy.annotation.specialize import getuniquenondirectgraph @@ -13,33 +13,8 @@ PORTAL = engine.Engine.portal_try_rule.im_func -class PyrologHintAnnotatorPolicy(HintAnnotatorPolicy): - novirtualcontainer = True - oopspec = True - - def seetranslator(self, t): - from pypy.jit.hintannotator.bookkeeper import ImpurityAnalyzer - self.analyzer = ImpurityAnalyzer(t) - portal = getattr(PORTAL, 'im_func', PORTAL) - portal_graph = graphof(t, portal) - self.timeshift_graphs = timeshift_graphs(t, portal_graph) - - def look_inside_graph(self, graph): - if graph in self.timeshift_graphs: - return self.timeshift_graphs[graph] - # don't look into pure functions - if not self.analyzer.analyze_direct_call(graph): - return False - - try: - func = graph.func - except AttributeError: - return True - if hasattr(func, '_look_inside_me_'): - return func._look_inside_me_ - if getattr(func, '_pure_function_', False): - return False - mod = func.__module__ or '?' +class PyrologHintAnnotatorPolicy(ManualGraphPolicy): + def look_inside_graph_of_module(self, graph, mod): if mod in forbidden_modules: return False if mod in good_modules: @@ -48,109 +23,36 @@ return False return True -def enumerate_reachable_graphs(translator, startgraph): - from pypy.translator.backendopt.support import find_calls_from - pending = [(startgraph, None)] - yield pending[0] - seen = {startgraph: True} - while pending: - yield None # hack: a separator meaning "length increases now" - nextlengthlist = [] - nextseen = {} - for node in pending: - head, tail = node - for block, callee in find_calls_from(translator, head): - if callee not in seen: - newnode = callee, node - yield newnode - nextlengthlist.append(newnode) - nextseen[callee] = True - pending = nextlengthlist - seen.update(nextseen) - yield None - -def graphs_on_the_path_to(translator, startgraph, targetgraphs): - targetgraphs = targetgraphs.copy() - result = {} - found = {} - for node in enumerate_reachable_graphs(translator, startgraph): - if node is None: # hack: a separator meaning "length increases now" - for graph in found: - del targetgraphs[graph] - found.clear() - if not targetgraphs: - return result - elif node[0] in targetgraphs: - found[node[0]] = True - while node is not None: - head, tail = node - result[head] = True - node = tail - raise Exception("did not reach all targets:\nmissing %r" % ( - targetgraphs.keys(),)) - - -def timeshift_graphs(t, portal_graph): - import pypy - result_graphs = {} - - bk = t.annotator.bookkeeper - - def _graph(func): - func = getattr(func, 'im_func', func) - desc = bk.getdesc(func) - return getuniquenondirectgraph(desc) - - def seefunc(fromfunc, *tofuncs): - targetgraphs = {} - for tofunc in tofuncs: - targetgraphs[_graph(tofunc)] = True - graphs = graphs_on_the_path_to(t, _graph(fromfunc), targetgraphs) - for graph in graphs: - result_graphs[graph] = True - - def seepath(*path): - for i in range(1, len(path)): - seefunc(path[i-1], path[i]) - - def seegraph(func, look=True): - graph = _graph(func) - if look: - extra = "" - if look != True: - extra = " substituted with %s" % look - result_graphs[graph] = look - - for cls in [term.Var, term.Term, term.Number, term.Float, term.Atom]: - seegraph(cls.copy) - seegraph(cls.__init__) - seegraph(cls.copy_and_unify) - for cls in [term.Term, term.Number, term.Atom]: - seegraph(cls.copy_and_basic_unify) - seegraph(cls.dereference) - seegraph(cls.copy_and_basic_unify) - for cls in [term.Var, term.Term, term.Number, term.Atom]: - seegraph(cls.get_unify_hash) - for cls in [term.Callable, term.Atom, term.Term]: - seegraph(cls.get_prolog_signature) - seegraph(PORTAL) - seegraph(pypy.lang.prolog.interpreter.engine.Heap.newvar) - seegraph(pypy.lang.prolog.interpreter.term.Rule.clone_and_unify_head) - seegraph(pypy.lang.prolog.interpreter.engine.Engine.call) - seegraph(pypy.lang.prolog.interpreter.engine.Engine._call) - seegraph(pypy.lang.prolog.interpreter.engine.Engine.user_call) - seegraph(pypy.lang.prolog.interpreter.engine.Engine._user_call) - seegraph(pypy.lang.prolog.interpreter.engine.Engine.try_rule) - seegraph(pypy.lang.prolog.interpreter.engine.Engine._try_rule) - seegraph(pypy.lang.prolog.interpreter.engine.Engine.main_loop) - seegraph(pypy.lang.prolog.interpreter.engine.Engine.dispatch_bytecode) - seegraph(pypy.lang.prolog.interpreter.engine.LinkedRules.find_applicable_rule) - seegraph(pypy.lang.prolog.interpreter.engine.Continuation.call) - seegraph(term.Term.unify_hash_of_child) - for cls in [engine.Continuation, engine.LimitedScopeContinuation, - pypy.lang.prolog.builtin.control.AndContinuation]: - seegraph(cls._call) - return result_graphs + def fill_timeshift_graphs(self, t, portal_graph): + for cls in [term.Var, term.Term, term.Number, term.Float, term.Atom]: + self.seegraph(cls.copy) + self.seegraph(cls.__init__) + self.seegraph(cls.copy_and_unify) + for cls in [term.Term, term.Number, term.Atom]: + self.seegraph(cls.copy_and_basic_unify) + self.seegraph(cls.dereference) + self.seegraph(cls.copy_and_basic_unify) + for cls in [term.Var, term.Term, term.Number, term.Atom]: + self.seegraph(cls.get_unify_hash) + for cls in [term.Callable, term.Atom, term.Term]: + self.seegraph(cls.get_prolog_signature) + self.seegraph(PORTAL) + self.seegraph(engine.Heap.newvar) + self.seegraph(term.Rule.clone_and_unify_head) + self.seegraph(engine.Engine.call) + self.seegraph(engine.Engine._call) + self.seegraph(engine.Engine.user_call) + self.seegraph(engine.Engine._user_call) + self.seegraph(engine.Engine.try_rule) + self.seegraph(engine.Engine._try_rule) + self.seegraph(engine.Engine.main_loop) + self.seegraph(engine.Engine.dispatch_bytecode) + self.seegraph(engine.LinkedRules.find_applicable_rule) + self.seegraph(engine.Continuation.call) + self.seegraph(term.Term.unify_hash_of_child) + for cls in [engine.Continuation, engine.LimitedScopeContinuation, + self.pypy.lang.prolog.builtin.control.AndContinuation]: + self.seegraph(cls._call) def get_portal(drv): t = drv.translator From cfbolz at codespeak.net Tue May 22 12:16:52 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 22 May 2007 12:16:52 +0200 (CEST) Subject: [pypy-svn] r43559 - pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator Message-ID: <20070522101652.4C7498096@code0.codespeak.net> Author: cfbolz Date: Tue May 22 12:16:51 2007 New Revision: 43559 Modified: pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator/policy.py Log: fix problems Modified: pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator/policy.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator/policy.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator/policy.py Tue May 22 12:16:51 2007 @@ -1,5 +1,6 @@ from pypy.annotation import policy from pypy.annotation.specialize import getuniquenondirectgraph +from pypy.translator.translator import graphof class HintAnnotatorPolicy(policy.AnnotatorPolicy): novirtualcontainer = False @@ -48,7 +49,7 @@ self.translator = t self.bookkeeper = t.annotator.bookkeeper self.timeshift_graphs = {} - portal = getattr(PORTAL, 'im_func', PORTAL) + portal = getattr(self.PORTAL, 'im_func', self.PORTAL) portal_graph = graphof(t, portal) self.fill_timeshift_graphs(t, portal_graph) @@ -90,15 +91,15 @@ graphs = graphs_on_the_path_to(self.translator, self._graph(fromfunc), targetgraphs) for graph in graphs: - result_graphs[graph] = True + self.timeshift_graphs[graph] = True def seepath(self, *path): for i in range(1, len(path)): self.seefunc(path[i-1], path[i]) def seegraph(self, func, look=True): - graph = _graph(func) - result_graphs[graph] = look + graph = self._graph(func) + self.timeshift_graphs[graph] = look From niko at codespeak.net Tue May 22 14:41:10 2007 From: niko at codespeak.net (niko at codespeak.net) Date: Tue, 22 May 2007 14:41:10 +0200 (CEST) Subject: [pypy-svn] r43561 - in pypy/dist/pypy/translator: cli/test jvm jvm/test oosupport/test_template Message-ID: <20070522124110.B53188074@code0.codespeak.net> Author: niko Date: Tue May 22 14:41:08 2007 New Revision: 43561 Added: pypy/dist/pypy/translator/oosupport/test_template/objectmodel.py Modified: pypy/dist/pypy/translator/cli/test/test_objectmodel.py pypy/dist/pypy/translator/jvm/constant.py pypy/dist/pypy/translator/jvm/database.py pypy/dist/pypy/translator/jvm/generator.py pypy/dist/pypy/translator/jvm/genjvm.py pypy/dist/pypy/translator/jvm/metavm.py pypy/dist/pypy/translator/jvm/opcodes.py pypy/dist/pypy/translator/jvm/test/test_objectmodel.py pypy/dist/pypy/translator/jvm/typesystem.py Log: implement weak refs for genjvm; refactor test_objectmodel from cli into oosupport Modified: pypy/dist/pypy/translator/cli/test/test_objectmodel.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_objectmodel.py (original) +++ pypy/dist/pypy/translator/cli/test/test_objectmodel.py Tue May 22 14:41:08 2007 @@ -1,63 +1,10 @@ import py from pypy.translator.cli.test.runtest import CliTest -from pypy.rlib.test.test_objectmodel import BaseTestObjectModel - -from pypy.rlib.objectmodel import cast_object_to_weakgcaddress,\ - cast_weakgcaddress_to_object +from pypy.translator.oosupport.test_template.objectmodel import \ + BaseTestObjectModel def skip_r_dict(self): py.test.skip('r_dict support is still incomplete') class TestCliObjectModel(CliTest, BaseTestObjectModel): test_rtype_r_dict_bm = skip_r_dict - - def test_rdict_of_void_copy(self): - from pypy.rlib.test.test_objectmodel import r_dict, strange_key_eq, strange_key_hash - def fn(): - d = r_dict(strange_key_eq, strange_key_hash) - d['hello'] = None - d['world'] = None - d1 = d.copy() - return len(d1) - assert self.interpret(fn, []) == 2 - - # this test is copied from TestLLtype in - # rlib/test_objectmodel.py. It is not in TestOOtype because at - # the moment llinterpret can't handle cast_*weakadr* - def test_cast_to_and_from_weakaddress(self): - class A(object): - pass - class B(object): - pass - def f(): - a = A() - addr = cast_object_to_weakgcaddress(a) - return a is cast_weakgcaddress_to_object(addr, A) - res = self.interpret(f, []) - assert res -## def g(): -## a = A() -## addr = cast_object_to_weakgcaddress(a) -## return cast_weakgcaddress_to_int(addr) -## assert isinstance(self.interpret(f, []), int) - - def test_weakref_const(self): - class A(object): - def __init__(self): - self.x = 42 - a = A() - weak = cast_object_to_weakgcaddress(a) - def f(): - a.x = 10 - b = cast_weakgcaddress_to_object(weak, A) - return b.x - assert self.interpret(f, []) == 10 - - def test_weakref_const_null(self): - class A(object): - pass - weak = cast_object_to_weakgcaddress(None) - def f(): - b = cast_weakgcaddress_to_object(weak, A) - return b - assert self.interpret(f, []) is None Modified: pypy/dist/pypy/translator/jvm/constant.py ============================================================================== --- pypy/dist/pypy/translator/jvm/constant.py (original) +++ pypy/dist/pypy/translator/jvm/constant.py Tue May 22 14:41:08 2007 @@ -1,25 +1,31 @@ from pypy.rpython.ootypesystem import ootype +from pypy.objspace.flow import model as flowmodel from pypy.translator.jvm.generator import \ Field, Method, CUSTOMDICTMAKE from pypy.translator.oosupport.constant import \ BaseConstantGenerator, RecordConst, InstanceConst, ClassConst, \ - StaticMethodConst, CustomDictConst + StaticMethodConst, CustomDictConst, WeakRefConst, push_constant from pypy.translator.jvm.typesystem import \ - jPyPyConst, jObject, jVoid + jPyPyConst, jObject, jVoid, jWeakRef # ___________________________________________________________________________ # Constant Generator class JVMConstantGenerator(BaseConstantGenerator): - + # _________________________________________________________________ # Constant Operations # # We store constants in static fields of the jPyPyConst class. def _init_constant(self, const): - fieldty = self.db.lltype_to_cts(const.OOTYPE()) - const.fieldobj = Field(jPyPyConst.name, const.name, fieldty, True) + # Determine the Java type of the constant: some constants + # (weakrefs) do not have an OOTYPE, so if it returns None use + # jtype() + JFIELDOOTY = const.OOTYPE() + if not JFIELDOOTY: jfieldty = const.jtype() + else: jfieldty = self.db.lltype_to_cts(JFIELDOOTY) + const.fieldobj = Field(jPyPyConst.name, const.name, jfieldty, True) def push_constant(self, gen, const): const.fieldobj.load(gen) @@ -92,3 +98,27 @@ gen.new_with_jtype(self.hash_jcls) gen.emit(CUSTOMDICTMAKE) +class JVMWeakRefConst(WeakRefConst): + + # Ensure that weak refs are initialized last: + PRIORITY = 200 + + def jtype(self): + return jWeakRef + + def create_pointer(self, gen): + gen.prepare_cast_ptr_to_weak_address() + if not self.value: + TYPE = ootype.ROOT + gen.push_null(TYPE) + else: + TYPE = self.value._TYPE + push_constant(self.db, self.value._TYPE, self.value, gen) + gen.finalize_cast_ptr_to_weak_address(TYPE) + + def initialize_data(self, gen): + gen.pop(ootype.ROOT) + return True + + + Modified: pypy/dist/pypy/translator/jvm/database.py ============================================================================== --- pypy/dist/pypy/translator/jvm/database.py (original) +++ pypy/dist/pypy/translator/jvm/database.py Tue May 22 14:41:08 2007 @@ -16,6 +16,7 @@ jStringBuilder, jInt, jVoid, jString, jChar, jPyPyConst, jObject, \ jThrowable from pypy.translator.jvm.builtin import JvmBuiltInType +from pypy.rpython.lltypesystem.llmemory import WeakGcAddress from pypy.translator.oosupport.database import Database as OODatabase @@ -452,7 +453,8 @@ ootype.Char: jvmtype.jChar, # byte would be sufficient, but harder ootype.UniChar: jvmtype.jChar, ootype.Class: jvmtype.jClass, - ootype.ROOT: jvmtype.jObject # treat like a scalar + ootype.ROOT: jvmtype.jObject, # treat like a scalar + WeakGcAddress: jvmtype.jWeakRef } # Dictionary for non-scalar types; in this case, if we see the key, we Modified: pypy/dist/pypy/translator/jvm/generator.py ============================================================================== --- pypy/dist/pypy/translator/jvm/generator.py (original) +++ pypy/dist/pypy/translator/jvm/generator.py Tue May 22 14:41:08 2007 @@ -11,7 +11,7 @@ jObject, jByteArray, jPyPyExcWrap, jIntegerClass, jLongClass, \ jDoubleClass, jCharClass, jStringBuilder, JvmScalarType, jArrayList, \ jObjectArray, jPyPyInterlink, jPyPyCustomDict, jPyPyEquals, \ - jPyPyHashCode, jMap + jPyPyHashCode, jMap, jWeakRef # ___________________________________________________________________________ # Miscellaneous helper functions @@ -238,6 +238,7 @@ NEW = IntClassNameOpcode('new') DUP = Opcode('dup') DUP2 = Opcode('dup2') +DUP_X1 = Opcode('dup_x1') POP = Opcode('pop') POP2 = Opcode('pop2') SWAP = Opcode('swap') @@ -1084,7 +1085,37 @@ self.emit(DCONST_1) else: # Big hack to avoid exponential notation: - self.emit(LDC2, "%22.22f" % value) + self.emit(LDC2, "%22.22f" % value) + + def prepare_cast_ptr_to_weak_address(self): + """ + To cast a pointer to a weak ref is a 2-step process. + First, invoke this routine. Then, invoke what is needed + to push the value, then invoke finalize_cast_ptr_to_weak_address + """ + self.emit(NEW, jWeakRef) + self.emit(DUP) + + def finalize_cast_ptr_to_weak_address(self, OOTYPE): + """ + After prepare_cast_ptr_to_weak_address has been called, and the + ptr to cast has been pushed, you can invoke this routine. + OOTYPE should be the type of value which was pushed. + The result will be that at the top of the stack is a weak reference. + """ + self.prepare_generic_argument(OOTYPE) + ctor = Method.c(jWeakRef, (jObject,)) + self.emit(ctor) + + def cast_weak_address_to_ptr(self, OOTYPE): + """ + If a weak ref is at the top of the stack, yields the object + that this weak ref is a pointer to. OOTYPE is the kind of object + you had a weak reference to. + """ + get_mthd = Method.v(jWeakRef, 'get', (), jObject) + self.emit(get_mthd) + self.prepare_generic_result(OOTYPE) # __________________________________________________________________ # Methods invoked directly by strings in jvm/opcode.py Modified: pypy/dist/pypy/translator/jvm/genjvm.py ============================================================================== --- pypy/dist/pypy/translator/jvm/genjvm.py (original) +++ pypy/dist/pypy/translator/jvm/genjvm.py Tue May 22 14:41:08 2007 @@ -18,7 +18,8 @@ from pypy.translator.jvm.opcodes import opcodes from pypy.rpython.ootypesystem import ootype from pypy.translator.jvm.constant import \ - JVMConstantGenerator, JVMStaticMethodConst, JVMCustomDictConst + JVMConstantGenerator, JVMStaticMethodConst, JVMCustomDictConst, \ + JVMWeakRefConst from pypy.translator.jvm.prebuiltnodes import create_interlink_node class JvmError(Exception): @@ -208,6 +209,7 @@ ConstantGenerator = JVMConstantGenerator CustomDictConst = JVMCustomDictConst StaticMethodConst = JVMStaticMethodConst + WeakRefConst = JVMWeakRefConst def __init__(self, tmpdir, translator, entrypoint): """ Modified: pypy/dist/pypy/translator/jvm/metavm.py ============================================================================== --- pypy/dist/pypy/translator/jvm/metavm.py (original) +++ pypy/dist/pypy/translator/jvm/metavm.py Tue May 22 14:41:08 2007 @@ -114,3 +114,19 @@ self._load_func(generator, *op.args[4:7]) generator.emit(jvmgen.CUSTOMDICTMAKE) NewCustomDict = _NewCustomDict() + +class _CastPtrToWeakAddress(MicroInstruction): + def render(self, generator, op): + arg = op.args[0] + generator.prepare_cast_ptr_to_weak_address() + generator.load(arg) + generator.finalize_cast_ptr_to_weak_address(arg.concretetype) + generator.store(op.result) +CastPtrToWeakAddress = _CastPtrToWeakAddress() + +class _CastWeakAddressToPtr(MicroInstruction): + def render(self, generator, op): + RESULTTYPE = op.result.concretetype + generator.cast_weak_address_to_ptr(RESULTTYPE) +CastWeakAddressToPtr = _CastWeakAddressToPtr() + Modified: pypy/dist/pypy/translator/jvm/opcodes.py ============================================================================== --- pypy/dist/pypy/translator/jvm/opcodes.py (original) +++ pypy/dist/pypy/translator/jvm/opcodes.py Tue May 22 14:41:08 2007 @@ -9,7 +9,8 @@ PushArg, PushAllArgs, StoreResult, InstructionList, New, DoNothing, Call,\ SetField, GetField, DownCast, RuntimeNew, OOString, CastTo from pypy.translator.jvm.metavm import \ - IndirectCall, JvmCallMethod, TranslateException, NewCustomDict + IndirectCall, JvmCallMethod, TranslateException, NewCustomDict, \ + CastPtrToWeakAddress, CastWeakAddressToPtr import pypy.translator.jvm.generator as jvmgen import pypy.translator.jvm.typesystem as jvmtype @@ -59,9 +60,9 @@ 'hint': [PushArg(0), StoreResult], 'direct_call': [Call, StoreResult], 'indirect_call': [PushAllArgs, IndirectCall, StoreResult], - # - #'cast_ptr_to_weakadr': [PushAllArgs, 'newobj instance void class %s::.ctor(object)' % WEAKREF], - #'cast_weakadr_to_ptr': [CastWeakAdrToPtr], + + 'cast_ptr_to_weakadr': [CastPtrToWeakAddress], + 'cast_weakadr_to_ptr': CastWeakAddressToPtr, #'gc__collect': 'call void class [mscorlib]System.GC::Collect()', #'resume_point': Ignore, Modified: pypy/dist/pypy/translator/jvm/test/test_objectmodel.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_objectmodel.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_objectmodel.py Tue May 22 14:41:08 2007 @@ -1,9 +1,7 @@ import py from pypy.translator.jvm.test.runtest import JvmTest -from pypy.rlib.test.test_objectmodel import BaseTestObjectModel - -from pypy.rlib.objectmodel import cast_object_to_weakgcaddress,\ - cast_weakgcaddress_to_object +from pypy.translator.oosupport.test_template.objectmodel import \ + BaseTestObjectModel class TestJvmObjectModel(JvmTest, BaseTestObjectModel): pass Modified: pypy/dist/pypy/translator/jvm/typesystem.py ============================================================================== --- pypy/dist/pypy/translator/jvm/typesystem.py (original) +++ pypy/dist/pypy/translator/jvm/typesystem.py Tue May 22 14:41:08 2007 @@ -168,6 +168,7 @@ jMath = JvmClassType('java.lang.Math') jList = JvmInterfaceType('java.util.List') jArrayList = JvmClassType('java.util.ArrayList') +jWeakRef = JvmClassType('java.lang.ref.WeakReference') jPyPy = JvmClassType('pypy.PyPy') jPyPyExcWrap = JvmClassType('pypy.ExceptionWrapper') jPyPyConst = JvmClassType('pypy.Constant') Added: pypy/dist/pypy/translator/oosupport/test_template/objectmodel.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/oosupport/test_template/objectmodel.py Tue May 22 14:41:08 2007 @@ -0,0 +1,57 @@ +import py +from pypy.rlib.test.test_objectmodel import BaseTestObjectModel as RLibBase + +from pypy.rlib.objectmodel import cast_object_to_weakgcaddress,\ + cast_weakgcaddress_to_object + +class BaseTestObjectModel(RLibBase): + def test_rdict_of_void_copy(self): + from pypy.rlib.test.test_objectmodel import r_dict, strange_key_eq, strange_key_hash + def fn(): + d = r_dict(strange_key_eq, strange_key_hash) + d['hello'] = None + d['world'] = None + d1 = d.copy() + return len(d1) + assert self.interpret(fn, []) == 2 + + # this test is copied from TestLLtype in + # rlib/test_objectmodel.py. It is not in TestOOtype because at + # the moment llinterpret can't handle cast_*weakadr* + def test_cast_to_and_from_weakaddress(self): + class A(object): + pass + class B(object): + pass + def f(): + a = A() + addr = cast_object_to_weakgcaddress(a) + return a is cast_weakgcaddress_to_object(addr, A) + res = self.interpret(f, []) + assert res +## def g(): +## a = A() +## addr = cast_object_to_weakgcaddress(a) +## return cast_weakgcaddress_to_int(addr) +## assert isinstance(self.interpret(f, []), int) + + def test_weakref_const(self): + class A(object): + def __init__(self): + self.x = 42 + a = A() + weak = cast_object_to_weakgcaddress(a) + def f(): + a.x = 10 + b = cast_weakgcaddress_to_object(weak, A) + return b.x + assert self.interpret(f, []) == 10 + + def test_weakref_const_null(self): + class A(object): + pass + weak = cast_object_to_weakgcaddress(None) + def f(): + b = cast_weakgcaddress_to_object(weak, A) + return b + assert self.interpret(f, []) is None From pdg at codespeak.net Tue May 22 16:07:59 2007 From: pdg at codespeak.net (pdg at codespeak.net) Date: Tue, 22 May 2007 16:07:59 +0200 (CEST) Subject: [pypy-svn] r43563 - pypy/dist/pypy/translator/jvm/test Message-ID: <20070522140759.42506806E@code0.codespeak.net> Author: pdg Date: Tue May 22 16:07:57 2007 New Revision: 43563 Added: pypy/dist/pypy/translator/jvm/test/test_builtin.py Modified: pypy/dist/pypy/translator/jvm/test/test_class.py Log: translator/jvm/test - added test_builtin and updated test_class Added: pypy/dist/pypy/translator/jvm/test/test_builtin.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/jvm/test/test_builtin.py Tue May 22 16:07:57 2007 @@ -0,0 +1,9 @@ +import py +from pypy.tool import udir +from pypy.translator.jvm.test.runtest import JvmTest +from pypy.rpython.test.test_rbuiltin import BaseTestRbuiltin + +class TestJavaBuiltin(JvmTest, BaseTestRbuiltin): + def test_os(self): + py.test.skip("Jvm os support uncertain") + Modified: pypy/dist/pypy/translator/jvm/test/test_class.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_class.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_class.py Tue May 22 16:07:57 2007 @@ -7,5 +7,46 @@ def test_overridden_classattr_as_defaults(self): py.test.skip("JVM doesn't support overridden default value yet") + def test_abstract_method(self): + class Base(object): + pass + class A(Base): + def f(self, x): + return x+1 + class B(Base): + def f(self, x): + return x+2 + def call(obj, x): + return obj.f(x) + def fn(x): + a = A() + b = B() + return call(a, x) + call(b, x) + assert self.interpret(fn, [0]) == 3 + + def test_abstract_method2(self): + class Root(object): + pass + class Class1(Root): + pass + class Derived(Class1): + x = 1 + class Class2(Root): + x = 2 + def get_x(obj): + return obj.x + def fn(): + derived = Derived() + cls2 = Class2() + return get_x(derived) + get_x(cls2) + assert self.interpret(fn, []) == 3 + + def test_same_name(self): + py.test.skip("JVM doesn't support classes with the same name") + + def test_ctr_location(self): + py.test.skip("Ask cuni if this applies to JVM -I don't think so") + + #class TestCliSpecialCase(CliTest, BaseTestRspecialcase): # pass From arigo at codespeak.net Tue May 22 20:13:26 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 22 May 2007 20:13:26 +0200 (CEST) Subject: [pypy-svn] r43566 - pypy/extradoc/talk/dyla2007 Message-ID: <20070522181326.DC34F806E@code0.codespeak.net> Author: arigo Date: Tue May 22 20:13:25 2007 New Revision: 43566 Added: pypy/extradoc/talk/dyla2007/dyla.txt (contents, props changed) Log: Starting to write text. I guess that the style is kind of suited for this kind of paper, but it's probably nevertheless waiting for pedronis to come along and hint that all my paragraphs should be rewritten upside-down :-) Added: pypy/extradoc/talk/dyla2007/dyla.txt ============================================================================== --- (empty file) +++ pypy/extradoc/talk/dyla2007/dyla.txt Tue May 22 20:13:25 2007 @@ -0,0 +1,77 @@ +=============================================================================== + How to Not write Virtual Machines for Dynamic Languages +=============================================================================== + + +Abstract +======== + +Dynamic languages are traditionally implemented by writing a virtual +machine for them, centered around an interpreter and/or a built-in +compiler and providing the object model and memory management. A +relatively recent alternative to this approach is to run the dynamic +language on top of an existing, well-tuned object-oriented virtual +machine like Java's. In this paper, we outline the reasons for this +trend and explore its limitations. We contrast it with a different, +complementary alternative: flexibly generating virtual machines instead +of writing them by hand. This approach has been validated by the PyPy +project, with which we can automatically insert many features into the +generated virtual machines - including good just-in-time compilers tuned +to the dynamic language at hand. + + +Introduction +============ + +Dynamic languages are traditionally implemented by writing a virtual +machine for them in a low-level language like C, or in a language that +can relatively easily be turned into C. The machine implements an +object model supporting the high level dynamic language's objects. It +typically provides features like automatic garbage collection. Recent +languages like Python, Ruby and Perl have complicated semantics which +are most easily mapped to a simple interpreter operating on syntax trees +or bytecode; simpler languages like Lisp and Self typically have more +efficient implementations based on just-in-time code generation. + +The efforts required to build a new virtual machine are relatively +important. This is particularly true for languages which are complex +and in constant evolution. This is a major reason for keeping the +corresponding virtual machines simple. The C implementation of Python, +for example, is a simple bytecode interpreter and provides no more +advanced memory management techniques than reference counting coupled +with a reference cycle detector. This implementation is manageable for +an Open Source community with a limited amount of resources. The same +constraint can be found in many academic projects. + +However, there is a recent trend that enters in conflict with the goal +of having to maintain essentially a single, simple enough implementation +for a given programming language. This trend is to build an +implementation of the language on top of another virtual machine, like +the Java VM or the .NET one. + +As in the case of Python and Ruby, this results in multiple +implementations of the language, with an oldest custom VM written in C +which is the "official" version in the sense that its evolution defines +the evolution of the language itself. The other implementations - +Jython and JRuby on the JVM, IronPython and IronRuby on .NET - +ultimately need to be tediously kept up-to-date. + +Java (and increasingly .NET) provides well-tuned object-oriented virtual +machines which offer many features that are of interest to all +programmers including language implementors (a higher level of +expression, automatic memory management, richer libraries...). They +also provide a higher level "base object model", which enables better +cross-language integration: a Jython program can directly import Java +classes and vice-versa. + +What is the best way to implement dynamic languages? +---------------------------------------------------- + +The question is thus how the increased benefits of reusing +object-oriented virtual machines (OO VM) can be balanced against the +fact that this introduces new implementations which may split the total +effort of the community in pieces, and get increasingly +resource-consuming to keep in sync with each other. + +Standardizing on one OO VM would further increase interoperability +and avoid the division-of-efforts issue. (to be continued) From cfbolz at codespeak.net Wed May 23 09:24:43 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 23 May 2007 09:24:43 +0200 (CEST) Subject: [pypy-svn] r43568 - in pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter: . test Message-ID: <20070523072443.B5E07806B@code0.codespeak.net> Author: cfbolz Date: Wed May 23 09:24:42 2007 New Revision: 43568 Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/portal.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/term.py pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/test/test_jit.py Log: cleaning up a bit Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/engine.py Wed May 23 09:24:42 2007 @@ -10,11 +10,11 @@ DEBUG = False # bytecodes: -CALL = chr(0) -USER_CALL = chr(1) -TRY_RULE = chr(2) -CONTINUATION = chr(3) -DONE = chr(4) +CALL = 'a' +USER_CALL = 'u' +TRY_RULE = 't' +CONTINUATION = 'c' +DONE = 'd' class Continuation(object): @@ -232,8 +232,7 @@ from pypy.lang.prolog.interpreter.parsing import parse_file trees = parse_file(s, self.parser, Engine._build_and_run, self) - def call(self, query, continuation=DONOTHING, choice_point=True, - inline=False): + def call(self, query, continuation=DONOTHING, choice_point=True): assert isinstance(query, Callable) if not choice_point: return (CALL, query, continuation, None) @@ -263,15 +262,12 @@ hint(where, concrete=True) hint(rule, concrete=True) while 1: - #hint(None, global_merge_point=True) - #print " " * self.depth, where, query if where == DONE: return next next = self.dispatch_bytecode(where, query, continuation, rule) where, query, continuation, rule = next where = hint(where, promote=True) - def dispatch_bytecode(self, where, query, continuation, rule): if where == CALL: next = self._call(query, continuation) @@ -309,15 +305,7 @@ error.throw_existence_error( "procedure", query.get_prolog_signature()) - #XXX make a nice method - if isinstance(query, Term): - unify_hash = [] - i = 0 - while i < len(query.args): - unify_hash.append(query.unify_hash_of_child(i)) - i += 1 - else: - unify_hash = [] + unify_hash = query.unify_hash_of_children(self.heap) rulechain = startrulechain.find_applicable_rule(unify_hash) if rulechain is None: # none of the rules apply @@ -325,11 +313,12 @@ rule = rulechain.rule rulechain = rulechain.next oldstate = self.heap.branch() - while rulechain: - rulechain = rulechain.find_applicable_rule(unify_hash) - if rulechain is None: - self.heap.discard(oldstate) - break + while 1: + if rulechain is not None: + rulechain = rulechain.find_applicable_rule(unify_hash) + choice_point = rulechain is not None + else: + choice_point = False hint(rule, concrete=True) if rule.contains_cut: continuation = LimitedScopeContinuation(continuation) @@ -345,31 +334,29 @@ continuation) raise else: + inline = False #XXX rule.body is None # inline facts try: - result = self.try_rule(rule, query, continuation) + # for the last rule (rulechain is None), this will always + # return, because choice_point is False + result = self.try_rule(rule, query, continuation, + choice_point=choice_point, + inline=inline) self.heap.discard(oldstate) return result except UnificationFailed: + assert choice_point self.heap.revert(oldstate) rule = rulechain.rule rulechain = rulechain.next - hint(rule, concrete=True) - if rule.contains_cut: - continuation = LimitedScopeContinuation(continuation) - try: - return self.try_rule(rule, query, continuation) - except CutException, e: - if continuation.scope_active: - return self.continue_after_cut(e.continuation, continuation) - raise - return self.try_rule(rule, query, continuation, choice_point=False) def try_rule(self, rule, query, continuation=DONOTHING, choice_point=True, inline=False): - if not we_are_jitted(): - return self.portal_try_rule(rule, query, continuation, choice_point) if not choice_point: return (TRY_RULE, query, continuation, rule) + if not we_are_jitted(): + return self.portal_try_rule(rule, query, continuation, choice_point) + if inline: + return self.main_loop(TRY_RULE, query, continuation, rule) #if _is_early_constant(rule): # rule = hint(rule, promote=True) # return self.portal_try_rule(rule, query, continuation, choice_point) @@ -380,20 +367,25 @@ def portal_try_rule(self, rule, query, continuation, choice_point): hint(None, global_merge_point=True) - #hint(choice_point, concrete=True) - #if not choice_point: - # return self._try_rule(rule, query, continuation) + hint(choice_point, concrete=True) + if not choice_point: + return self._try_rule(rule, query, continuation) where = TRY_RULE next = (DONE, None, None, None) hint(where, concrete=True) hint(rule, concrete=True) + signature = hint(query.signature, promote=True) while 1: hint(None, global_merge_point=True) - rule = hint(rule, promote=True) if where == DONE: return next + if rule is not None: + assert rule.signature == signature next = self.dispatch_bytecode(where, query, continuation, rule) where, query, continuation, rule = next + rule = hint(rule, promote=True) + if query is not None: + signature = hint(query.signature, promote=True) where = hint(where, promote=True) def _try_rule(self, rule, query, continuation): Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/portal.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/portal.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/portal.py Wed May 23 09:24:42 2007 @@ -11,9 +11,11 @@ 'pypy.lang.prolog.builtin.register': True } + PORTAL = engine.Engine.portal_try_rule.im_func class PyrologHintAnnotatorPolicy(ManualGraphPolicy): + PORTAL = PORTAL def look_inside_graph_of_module(self, graph, mod): if mod in forbidden_modules: return False @@ -24,6 +26,7 @@ return True def fill_timeshift_graphs(self, t, portal_graph): + import pypy for cls in [term.Var, term.Term, term.Number, term.Float, term.Atom]: self.seegraph(cls.copy) self.seegraph(cls.__init__) @@ -36,6 +39,7 @@ self.seegraph(cls.get_unify_hash) for cls in [term.Callable, term.Atom, term.Term]: self.seegraph(cls.get_prolog_signature) + self.seegraph(cls.unify_hash_of_children) self.seegraph(PORTAL) self.seegraph(engine.Heap.newvar) self.seegraph(term.Rule.clone_and_unify_head) @@ -48,10 +52,11 @@ self.seegraph(engine.Engine.main_loop) self.seegraph(engine.Engine.dispatch_bytecode) self.seegraph(engine.LinkedRules.find_applicable_rule) + for method in "branch revert discard newvar extend maxvar".split(): + self.seegraph(getattr(engine.Heap, method)) self.seegraph(engine.Continuation.call) - self.seegraph(term.Term.unify_hash_of_child) for cls in [engine.Continuation, engine.LimitedScopeContinuation, - self.pypy.lang.prolog.builtin.control.AndContinuation]: + pypy.lang.prolog.builtin.control.AndContinuation]: self.seegraph(cls._call) def get_portal(drv): Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/term.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/term.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/term.py Wed May 23 09:24:42 2007 @@ -46,14 +46,11 @@ def clone_compress_vars(self, vars_new_indexes, offset): return self - def get_unify_hash(self): + def get_unify_hash(self, heap): # if two non-var objects return two different numbers # they must not be unifiable raise NotImplementedError("abstract base class") - def unify_hash_of_child(self, i): - raise KeyError - @specialize.arg(3) def unify(self, other, heap, occurs_check=False): raise NotImplementedError("abstract base class") @@ -147,7 +144,12 @@ vars_new_indexes[self.index] = index return Var.newvar(index) - def get_unify_hash(self): + def get_unify_hash(self, heap): + if heap is not None: + self = self.dereference(heap) + if isinstance(self, Var): + return 0 + return self.get_unify_hash(heap) return 0 def contains_var(self, var, heap): @@ -220,6 +222,9 @@ def get_prolog_signature(self): raise NotImplementedError("abstract base") + def unify_hash_of_children(self, heap): + raise NotImplementedError("abstract base") + class Atom(Callable): TAG = tag() @@ -250,16 +255,19 @@ def copy_and_basic_unify(self, other, heap, memo): hint(self, concrete=True) - if isinstance(other, Atom) and (hint(self is other, promote=True) or + if isinstance(other, Atom) and (self is other or other.name == self.name): return self else: raise UnificationFailed - def get_unify_hash(self): + def get_unify_hash(self, heap): name = hint(self.name, promote=True) return intmask(hash(name) << TAGBITS | self.TAG) + def unify_hash_of_children(self, heap): + return [] + def get_prolog_signature(self): return Term("/", [self, NUMBER_0]) @@ -301,7 +309,7 @@ def __repr__(self): return "Number(%r)" % (self.num, ) - def get_unify_hash(self): + def get_unify_hash(self, heap): return intmask(self.num << TAGBITS | self.TAG) NUMBER_0 = Number(0) @@ -329,7 +337,7 @@ else: raise UnificationFailed - def get_unify_hash(self): + def get_unify_hash(self, heap): #XXX no clue whether this is a good idea... m, e = math.frexp(self.num) m = intmask(int(m / 2 * 2 ** (32 - TAGBITS))) @@ -364,7 +372,7 @@ else: raise UnificationFailed - def get_unify_hash(self): + def get_unify_hash(self, heap): return intmask(id(self) << TAGBITS | self.TAG) @@ -461,12 +469,17 @@ else: return self - def get_unify_hash(self): + def get_unify_hash(self, heap): signature = hint(self.signature, promote=True) return intmask(hash(signature) << TAGBITS | self.TAG) - def unify_hash_of_child(self, i): - return self.args[i].get_unify_hash() + def unify_hash_of_children(self, heap): + unify_hash = [] + i = 0 + while i < len(self.args): + unify_hash.append(self.args[i].get_unify_hash(heap)) + i += 1 + return unify_hash def get_prolog_signature(self): return Term("/", [Atom.newatom(self.name), Number(len(self.args))]) @@ -495,7 +508,7 @@ self.numvars = len(d) self.signature = self.head.signature if isinstance(head, Term): - self.unify_hash = [arg.get_unify_hash() for arg in head.args] + self.unify_hash = [arg.get_unify_hash(None) for arg in head.args] self._does_contain_cut() def _does_contain_cut(self): Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/test/test_jit.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/test/test_jit.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/test/test_jit.py Wed May 23 09:24:42 2007 @@ -39,16 +39,20 @@ res = self.timeshift_from_portal(main, portal.PORTAL, [1], policy=POLICY, - backendoptimize=True) + backendoptimize=True, + inline=0.0) assert res == True res = self.timeshift_from_portal(main, portal.PORTAL, [0], policy=POLICY, - backendoptimize=True) + backendoptimize=True, + inline=0.0) assert res == True def test_and(self): e = get_engine(""" + h(X) :- f(X). + h(a). b(a). a(a). f(X) :- b(X), a(X). @@ -59,7 +63,7 @@ def main(n): e.heap.reset() if n == 0: - e.call(term.Term("f", [X])) + e.call(term.Term("h", [X])) return isinstance(X.dereference(e.heap), term.Atom) else: return False @@ -69,7 +73,8 @@ res = self.timeshift_from_portal(main, portal.PORTAL, [0], policy=POLICY, - backendoptimize=True) + backendoptimize=True, + inline=0.0) assert res == True def test_append(self): @@ -93,7 +98,8 @@ e.heap.reset() res = self.timeshift_from_portal(main, portal.PORTAL, [0], policy=POLICY, - backendoptimize=True) + backendoptimize=True, + inline=0.0) assert res == True @@ -120,7 +126,8 @@ res = self.timeshift_from_portal(main, portal.PORTAL, [0], policy=POLICY, - backendoptimize=True) + backendoptimize=True, + inline=0.0) assert res == True def test_loop(self): @@ -141,6 +148,7 @@ res = self.timeshift_from_portal(main, portal.PORTAL, [0], policy=POLICY, - backendoptimize=True) + backendoptimize=True, + inline=0.0) assert res == True From fijal at codespeak.net Wed May 23 15:02:53 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 23 May 2007 15:02:53 +0200 (CEST) Subject: [pypy-svn] r43570 - in pypy/branch/kill-ctypes/pypy: rpython/lltypesystem rpython/lltypesystem/test translator/c Message-ID: <20070523130253.BF10B806E@code0.codespeak.net> Author: fijal Date: Wed May 23 15:02:52 2007 New Revision: 43570 Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rffi.py pypy/branch/kill-ctypes/pypy/translator/c/funcgen.py Log: Add mapping from str -> char*. Kind of works, but requires some more thinking probably. Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py Wed May 23 15:02:52 2007 @@ -1,9 +1,19 @@ -from pypy.rpython.lltypesystem.lltype import functionptr, FuncType +from pypy.rpython.lltypesystem import lltype def llexternal(name, args, result, sources=[], includes=[]): - ext_type = FuncType(args, result) - return functionptr(ext_type, name, external='C', sources=tuple(sources), - includes=tuple(includes)) + ext_type = lltype.FuncType(args, result) + return lltype.functionptr(ext_type, name, external='C', + sources=tuple(sources), + includes=tuple(includes)) +CCHARP = lltype.Array(lltype.Char, hints={'nolength': True}) +# various type mapping +# str -> char* +def str2charp(s): + array = lltype.malloc(CCHARP, len(s) + 1, flavor='raw') + for i in range(len(s)): + array[i] = s[i] + array[len(s)] = chr(0) + return array Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rffi.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rffi.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rffi.py Wed May 23 15:02:52 2007 @@ -1,7 +1,9 @@ -from pypy.rpython.lltypesystem.rffi import llexternal +#import py +from pypy.rpython.lltypesystem.rffi import llexternal, str2charp, CCHARP from pypy.translator.c.test.test_genc import compile -from pypy.rpython.lltypesystem.lltype import Signed +from pypy.rpython.lltypesystem.lltype import Signed, Ptr, Char, malloc +from pypy.rpython.lltypesystem import lltype from pypy.tool.udir import udir def test_basic(): @@ -36,3 +38,46 @@ xf = compile(f, []) assert xf() == 8+3 + +def test_string(): + z = llexternal('strlen', [Ptr(CCHARP)], Signed, includes=['stdio.h']) + + def f(): + s = str2charp("xxx") + res = z(s) + lltype.free(s, flavor='raw') + return res + + xf = compile(f, [], backendopt=False) + assert xf() == 3 + +def test_stringstar(): + import py + py.test.skip("completely broken") + c_source = """ + int f(char *args[]) { + char **p = args; + int l = 0; + while (p) { + l += strlen(p*); + p++; + } + return (l); + } + """ + elem_T = lltype.FixedSizeArray(lltype.Char, 1) + T = lltype.FixedSizeArray(Ptr(elem_T), 1) # this is char** + z = llexternal('f', [Ptr(T)], Signed, includes=['stdio.h']) + alloc_T = lltype.Array(CCHARP) + + def f(): + ss = malloc(alloc_T, 4, flavor='raw') + ref1, ss[0] = str2charp("xxx") + ref2, ss[1] = str2charp("x") + ref3, ss[2] = str2charp("xxxx") + _, ss[3] = lltype.nullptr(elem_T) + to_fun = lltype._subarray._makeptr(ss._obj, 0, ss._solid) + return z(to_fun) + + xf = compile(f, []) + assert xf() == 8 Modified: pypy/branch/kill-ctypes/pypy/translator/c/funcgen.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/translator/c/funcgen.py (original) +++ pypy/branch/kill-ctypes/pypy/translator/c/funcgen.py Wed May 23 15:02:52 2007 @@ -590,6 +590,8 @@ else: raise NotImplementedError + OP_FLAVORED_MALLOC_VARSIZE = OP_FLAVORED_MALLOC + def OP_FLAVORED_FREE(self, op): flavor = op.args[0].value if flavor == "raw": From fijal at codespeak.net Wed May 23 15:08:38 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 23 May 2007 15:08:38 +0200 (CEST) Subject: [pypy-svn] r43572 - pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test Message-ID: <20070523130838.D6BF8806E@code0.codespeak.net> Author: fijal Date: Wed May 23 15:08:38 2007 New Revision: 43572 Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rffi.py Log: Kill completely broken test not to confuse pedronis Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rffi.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rffi.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rffi.py Wed May 23 15:08:38 2007 @@ -50,34 +50,3 @@ xf = compile(f, [], backendopt=False) assert xf() == 3 - -def test_stringstar(): - import py - py.test.skip("completely broken") - c_source = """ - int f(char *args[]) { - char **p = args; - int l = 0; - while (p) { - l += strlen(p*); - p++; - } - return (l); - } - """ - elem_T = lltype.FixedSizeArray(lltype.Char, 1) - T = lltype.FixedSizeArray(Ptr(elem_T), 1) # this is char** - z = llexternal('f', [Ptr(T)], Signed, includes=['stdio.h']) - alloc_T = lltype.Array(CCHARP) - - def f(): - ss = malloc(alloc_T, 4, flavor='raw') - ref1, ss[0] = str2charp("xxx") - ref2, ss[1] = str2charp("x") - ref3, ss[2] = str2charp("xxxx") - _, ss[3] = lltype.nullptr(elem_T) - to_fun = lltype._subarray._makeptr(ss._obj, 0, ss._solid) - return z(to_fun) - - xf = compile(f, []) - assert xf() == 8 From jlg at codespeak.net Wed May 23 16:00:23 2007 From: jlg at codespeak.net (jlg at codespeak.net) Date: Wed, 23 May 2007 16:00:23 +0200 (CEST) Subject: [pypy-svn] r43573 - in pypy/dist/pypy/lang/scheme: . test Message-ID: <20070523140023.05F99807B@code0.codespeak.net> Author: jlg Date: Wed May 23 16:00:23 2007 New Revision: 43573 Modified: pypy/dist/pypy/lang/scheme/object.py (contents, props changed) pypy/dist/pypy/lang/scheme/ssparser.py (contents, props changed) pypy/dist/pypy/lang/scheme/test/test_object.py (contents, props changed) pypy/dist/pypy/lang/scheme/test/test_parser.py (contents, props changed) Log: tabs removed Modified: pypy/dist/pypy/lang/scheme/object.py ============================================================================== --- pypy/dist/pypy/lang/scheme/object.py (original) +++ pypy/dist/pypy/lang/scheme/object.py Wed May 23 16:00:23 2007 @@ -1,30 +1,84 @@ +import autopath class W_Root(object): + def to_string(self): + return '' - def is_pair(self): - return False + def to_boolean(self): + return False - def to_string(self): - return '' + def __str__(self): + return self.to_string() + "W" - def to_boolean(self): - return False - - def __str__(self): - return self.to_string() + def __repr__(self): + return "" class W_Boolean(W_Root): - pass + def __init__(self, val): + self.boolval = bool(val) + + def to_string(self): + if self.boolval: + return "#t" + return "#f" + + def to_boolean(self): + return self.boolval + + def __repr__(self): + return "" + +class W_String(W_Root): + def __init__(self, val): + self.strval = val + + def to_string(self): + return self.strval + + def __repr__(self): + return "" + +class W_Fixnum(W_Root): + def __init__(self, val): + self.fixnumval = int(val) + + def to_string(self): + return str(self.fixnumval) + + def to_number(self): + return self.to_fixnum() + + def to_fixnum(self): + return self.fixnumval + + def to_float(self): + return float(self.fixnumval) + +class W_Float(W_Root): + def __init__(self, val): + self.floatval = float(val) + + def to_string(self): + return str(self.floatval) + + def to_number(self): + return self.to_float() + + def to_fixnum(self): + return int(self.floatval) + + def to_float(self): + return self.floatval -class W_False(W_Boolean): +class W_Pair(W_Root): + def __init__(self, car, cdr): + self.car = car + self.cdr = cdr - def to_string(self): - return "#f" + def to_string(self): + return "(" + self.car.to_string() + " . " + self.cdr.to_string() + ")" -class W_True(W_Boolean): +class W_Nil(W_Root): + def to_string(self): + return "()" - def to_boolean(self): - return True - - def to_string(self): - return "#t" Modified: pypy/dist/pypy/lang/scheme/ssparser.py ============================================================================== --- pypy/dist/pypy/lang/scheme/ssparser.py (original) +++ pypy/dist/pypy/lang/scheme/ssparser.py Wed May 23 16:00:23 2007 @@ -1,6 +1,7 @@ import autopath from pypy.rlib.parsing.ebnfparse import parse_ebnf, make_parse_function from pypy.rlib.parsing.parsing import ParseError +from pypy.rlib.parsing.tree import RPythonVisitor DEBUG = False @@ -12,17 +13,30 @@ ''' try: - regexs, rules, ToAST = parse_ebnf(grammar) + regexs, rules, ToAST = parse_ebnf(grammar) except ParseError, e: - #print e.nice_error_message() - raise + #print e.nice_error_message() + raise parsef = make_parse_function(regexs, rules, eof=True) def parse(code): - t = parsef(code) - tree = t.visit(ToAST()) - if DEBUG: - ToAST().transform(t).view() - return tree + t = parsef(code) + tree = t.visit(ToAST())[0] + if DEBUG: + ToAST().transform(t).view() + return tree + +class ASTBuilder(RPythonVisitor): + + def visit_STRING(self, node): + print node.symbol + ":" + node.additional_info + + def visit_IDENTIFIER(self, node): + print node.symbol + ":" + node.additional_info + + def visit_sexpr(self, node): + print node.symbol + ":(" + nodes = [self.dispatch(child) for child in node.children] + print ")" Modified: pypy/dist/pypy/lang/scheme/test/test_object.py ============================================================================== --- pypy/dist/pypy/lang/scheme/test/test_object.py (original) +++ pypy/dist/pypy/lang/scheme/test/test_object.py Wed May 23 16:00:23 2007 @@ -1,12 +1,39 @@ from pypy.lang.scheme.object import * def test_false(): - w_false = W_False() - assert w_false.to_boolean() is False - assert isinstance(w_false, W_Boolean) + w_false = W_Boolean(False) + assert w_false.to_boolean() is False def test_true(): - w_true = W_True() - assert w_true.to_boolean() is True - assert isinstance(w_true, W_Boolean) + w_true = W_Boolean(True) + assert w_true.to_boolean() is True +def test_string(): + str = "Hello World!" + w_str = W_String(str) + assert str == w_str.to_string() + +def test_fixnum(): + num = 12345 + w_num = W_Fixnum(num) + assert num == w_num.to_fixnum() + assert float(num) == w_num.to_float() + +def test_float(): + num = 12345.567 + w_num = W_Float(num) + assert num == w_num.to_float() + assert int(num) == w_num.to_fixnum() + +def test_pair(): + c1 = W_Fixnum(1) + c2 = W_String("c2") + c3 = W_Float(0.3) + c4 = W_Nil() + p2 = W_Pair(c3, c4) + p1 = W_Pair(c2, p2) + p = W_Pair(c1, p1) + assert p.car == c1 + assert p.cdr.car == c2 + assert p.cdr.cdr.car == c3 + assert p.cdr.cdr.cdr == c4 Modified: pypy/dist/pypy/lang/scheme/test/test_parser.py ============================================================================== --- pypy/dist/pypy/lang/scheme/test/test_parser.py (original) +++ pypy/dist/pypy/lang/scheme/test/test_parser.py Wed May 23 16:00:23 2007 @@ -2,35 +2,41 @@ from pypy.rlib.parsing.parsing import Symbol, Nonterminal def test_simple_sexpr(): - #parse simple sexpr - t = parse(r'''(+ 1 2)''') - assert isinstance(t[0], Nonterminal) - assert len(t[0].children) == 3 + #parse simple sexpr + t = parse(r'''(+ 1 2)''') + assert isinstance(t, Nonterminal) + assert len(t.children) == 3 def test_string(): - #parse string - t = parse(r'''"don't beleive \"them\""''') - assert isinstance(t[0], Symbol) + #parse string + t = parse(r'''"don't beleive \"them\""''') + assert isinstance(t, Symbol) def test_complex_sexpr(): - #parse more complex sexpr - t = parse(r''' - (define (fac n) ; comment - (if (< n 2) n - (* (fac (- n 1)) n))) - ''') - assert isinstance(t[0], Nonterminal) - assert len(t[0].children) == 3 - assert isinstance(t[0].children[0], Symbol) - assert isinstance(t[0].children[1], Nonterminal) - assert isinstance(t[0].children[2], Nonterminal) + #parse more complex sexpr + t = parse(r''' + (define (fac n) ; comment + (if (< n 2) n + (* (fac (- n 1)) n))) + ''') + assert isinstance(t, Nonterminal) + assert len(t.children) == 3 + assert isinstance(t.children[0], Symbol) + assert isinstance(t.children[1], Nonterminal) + assert isinstance(t.children[2], Nonterminal) def test_ident_gen(): - ch_list = "+-*/azAZ09<=>-_~!$%&:?^" - for char in ch_list: - yield check_ident_ch, char + ch_list = "+-*/azAZ09<=>-_~!$%&:?^" + for char in ch_list: + yield check_ident_ch, char def check_ident_ch(char): - t = parse("(" + char + ")") - assert isinstance(t[0], Nonterminal) - assert isinstance(t[0].children[0], Symbol) + t = parse("(" + char + ")") + assert isinstance(t, Nonterminal) + assert isinstance(t.children[0], Symbol) + +def test_ast(): + t = parse(r'''(define var "(+ 1 2 (sqrt 4))")''') + astb = ASTBuilder() + astb.dispatch(t) + #assert False From cfbolz at codespeak.net Wed May 23 17:22:36 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 23 May 2007 17:22:36 +0200 (CEST) Subject: [pypy-svn] r43576 - pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/test Message-ID: <20070523152236.9E8B5806E@code0.codespeak.net> Author: cfbolz Date: Wed May 23 17:22:31 2007 New Revision: 43576 Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/test/test_jit.py Log: skip the prolog jit tests for now Modified: pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/test/test_jit.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/test/test_jit.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/lang/prolog/interpreter/test/test_jit.py Wed May 23 17:22:31 2007 @@ -6,6 +6,7 @@ POLICY = portal.PyrologHintAnnotatorPolicy() +py.test.skip() class TestPortal(PortalTest): small = False From arigo at codespeak.net Wed May 23 17:24:33 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 23 May 2007 17:24:33 +0200 (CEST) Subject: [pypy-svn] r43578 - pypy/extradoc/talk/dyla2007 Message-ID: <20070523152433.C32D08071@code0.codespeak.net> Author: arigo Date: Wed May 23 17:24:33 2007 New Revision: 43578 Modified: pypy/extradoc/talk/dyla2007/dyla.txt Log: Finished the introduction. Modified: pypy/extradoc/talk/dyla2007/dyla.txt ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.txt (original) +++ pypy/extradoc/talk/dyla2007/dyla.txt Wed May 23 17:24:33 2007 @@ -68,10 +68,33 @@ ---------------------------------------------------- The question is thus how the increased benefits of reusing -object-oriented virtual machines (OO VM) can be balanced against the +object-oriented virtual machines (OO VMs) can be balanced against the fact that this introduces new implementations which may split the total -effort of the community in pieces, and get increasingly +effort of the community in pieces that get increasingly resource-consuming to keep in sync with each other. -Standardizing on one OO VM would further increase interoperability -and avoid the division-of-efforts issue. (to be continued) +Standardizing on one OO VM would further increase interoperability and +avoid the division-of-efforts issue. In our opinion, though, a single +OO VM for everybody is not a practical idea; we consider the loss of +diversity to be more a problem than a solution, and any single OO VM +comes with limitations and trade-offs that may not be optimal for every +use case. + +The argument we will make in the present paper is that it is possible to +benefit from and integrate with OO VMs while keeping the dynamic +language implemented by a single, simple source code base. The idea is +to write an interpreter for that language in another sufficiently +high-level but less dynamic language. This interpreter plays the role +of a specification for the dynamic language. With a good enough +translation toolchain we can then generate whole virtual machines from +this specification - either full custom VMs for C-level operating +systems, or layers on top of various OO VMs. In other words, +metaprogramming techniques can be used to successfully replace a +foreseeable one-VM-fits-all standardization attempt. + +The argument boils down to: VMs for dynamic languages should not be +written by hand! The justification is based on the successful +completion of the PyPy project, which proves that the approach is +feasible in practice. Just as importantly, it also brings new insights +and concrete benefits in term of flexibility and performance that go +beyond the state of the art. From arigo at codespeak.net Wed May 23 17:52:22 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 23 May 2007 17:52:22 +0200 (CEST) Subject: [pypy-svn] r43579 - pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test Message-ID: <20070523155222.E763A8077@code0.codespeak.net> Author: arigo Date: Wed May 23 17:52:21 2007 New Revision: 43579 Modified: pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_portal.py pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_timeshift.py Log: (cfbolz, arigo) Revert some changes to .view(). Modified: pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_portal.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_portal.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_portal.py Wed May 23 17:52:21 2007 @@ -65,8 +65,8 @@ self.hrtyper.specialize(origportalgraph=origportalgraph, view = conftest.option.view and self.small) - if conftest.option.view and self.small: - t.viewcg() + #if conftest.option.view and self.small: + # t.viewcg() self.postprocess_timeshifting() self.readportalgraph = self.hrtyper.readportalgraph Modified: pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_timeshift.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_timeshift.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_timeshift.py Wed May 23 17:52:21 2007 @@ -73,7 +73,7 @@ for v in graph1.getargs()]) hannotator.simplify() if conftest.option.view: - hannotator.translator.viewcg() + hannotator.translator.view() return hs, hannotator, rtyper class TimeshiftingTests(object): From arigo at codespeak.net Wed May 23 17:55:17 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 23 May 2007 17:55:17 +0200 (CEST) Subject: [pypy-svn] r43580 - pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test Message-ID: <20070523155517.A94608077@code0.codespeak.net> Author: arigo Date: Wed May 23 17:55:17 2007 New Revision: 43580 Modified: pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_portal.py Log: (cfbolz, arigo) Refactored to be more enterprise-ready. Modified: pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_portal.py ============================================================================== --- pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_portal.py (original) +++ pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_portal.py Wed May 23 17:55:17 2007 @@ -66,7 +66,7 @@ view = conftest.option.view and self.small) #if conftest.option.view and self.small: - # t.viewcg() + # t.view() self.postprocess_timeshifting() self.readportalgraph = self.hrtyper.readportalgraph From arigo at codespeak.net Wed May 23 17:59:05 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 23 May 2007 17:59:05 +0200 (CEST) Subject: [pypy-svn] r43581 - pypy/dist/pypy/interpreter Message-ID: <20070523155905.ADDC08077@code0.codespeak.net> Author: arigo Date: Wed May 23 17:59:03 2007 New Revision: 43581 Added: pypy/dist/pypy/interpreter/baseobjspace.py.merge.tmp - copied, changed from r43580, pypy/dist/pypy/interpreter/baseobjspace.py Log: merging of http://codespeak.net/svn/pypy/branch/prolog-jit-experiments/pypy/interpreter/baseobjspace.py revisions 43364 to 43580: ------------------------------------------------------------------------ r43540 | cfbolz | 2007-05-21 18:37:23 +0200 (Mon, 21 May 2007) | 2 lines move jit-related stuff to its own file in rlib.jit ------------------------------------------------------------------------ r43366 | cfbolz | 2007-05-14 15:24:04 +0200 (Mon, 14 May 2007) | 5 lines Be afraid. Very afraid. A branch to experiment with rewriting the Prolog interpreter to make it work with the JIT. ------------------------------------------------------------------------ From arigo at codespeak.net Wed May 23 17:59:07 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 23 May 2007 17:59:07 +0200 (CEST) Subject: [pypy-svn] r43582 - pypy/dist/pypy/annotation/test Message-ID: <20070523155907.B76DE8077@code0.codespeak.net> Author: arigo Date: Wed May 23 17:59:07 2007 New Revision: 43582 Added: pypy/dist/pypy/annotation/test/test_annrpython.py.merge.tmp - copied, changed from r43580, pypy/dist/pypy/annotation/test/test_annrpython.py Log: merging of http://codespeak.net/svn/pypy/branch/prolog-jit-experiments/pypy/annotation/test/test_annrpython.py revisions 43364 to 43580: ------------------------------------------------------------------------ r43540 | cfbolz | 2007-05-21 18:37:23 +0200 (Mon, 21 May 2007) | 2 lines move jit-related stuff to its own file in rlib.jit ------------------------------------------------------------------------ r43366 | cfbolz | 2007-05-14 15:24:04 +0200 (Mon, 14 May 2007) | 5 lines Be afraid. Very afraid. A branch to experiment with rewriting the Prolog interpreter to make it work with the JIT. ------------------------------------------------------------------------ From arigo at codespeak.net Wed May 23 17:59:12 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 23 May 2007 17:59:12 +0200 (CEST) Subject: [pypy-svn] r43583 - pypy/dist/pypy/jit/timeshifter/test Message-ID: <20070523155912.79CEE807E@code0.codespeak.net> Author: arigo Date: Wed May 23 17:59:11 2007 New Revision: 43583 Added: pypy/dist/pypy/jit/timeshifter/test/test_portal.py.merge.tmp - copied, changed from r43580, pypy/dist/pypy/jit/timeshifter/test/test_portal.py Log: merging of http://codespeak.net/svn/pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_portal.py revisions 43364 to 43580: ------------------------------------------------------------------------ r43580 | arigo | 2007-05-23 17:55:17 +0200 (Wed, 23 May 2007) | 2 lines (cfbolz, arigo) Refactored to be more enterprise-ready. ------------------------------------------------------------------------ r43579 | arigo | 2007-05-23 17:52:21 +0200 (Wed, 23 May 2007) | 2 lines (cfbolz, arigo) Revert some changes to .view(). ------------------------------------------------------------------------ r43540 | cfbolz | 2007-05-21 18:37:23 +0200 (Mon, 21 May 2007) | 2 lines move jit-related stuff to its own file in rlib.jit ------------------------------------------------------------------------ r43405 | cfbolz | 2007-05-15 15:26:05 +0200 (Tue, 15 May 2007) | 3 lines make a test out of the prolog jit so that I can look at the graphs. some more changes around. ------------------------------------------------------------------------ r43366 | cfbolz | 2007-05-14 15:24:04 +0200 (Mon, 14 May 2007) | 5 lines Be afraid. Very afraid. A branch to experiment with rewriting the Prolog interpreter to make it work with the JIT. ------------------------------------------------------------------------ From arigo at codespeak.net Wed May 23 17:59:15 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 23 May 2007 17:59:15 +0200 (CEST) Subject: [pypy-svn] r43584 - pypy/dist/pypy/jit/hintannotator/test Message-ID: <20070523155915.5DB448090@code0.codespeak.net> Author: arigo Date: Wed May 23 17:59:14 2007 New Revision: 43584 Added: pypy/dist/pypy/jit/hintannotator/test/test_annotator.py.merge.tmp - copied, changed from r43580, pypy/dist/pypy/jit/hintannotator/test/test_annotator.py Log: merging of http://codespeak.net/svn/pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator/test/test_annotator.py revisions 43364 to 43580: ------------------------------------------------------------------------ r43556 | cfbolz | 2007-05-22 11:30:00 +0200 (Tue, 22 May 2007) | 3 lines factor out common parts of the PyrologHintAnnotatorPolicy and the PyPyHintAnnotatorPolicy ------------------------------------------------------------------------ r43554 | cfbolz | 2007-05-22 11:12:33 +0200 (Tue, 22 May 2007) | 2 lines another missed import :-( ------------------------------------------------------------------------ r43473 | cfbolz | 2007-05-18 14:06:41 +0200 (Fri, 18 May 2007) | 2 lines Allow functions to be manually marked as "pure". ------------------------------------------------------------------------ r43366 | cfbolz | 2007-05-14 15:24:04 +0200 (Mon, 14 May 2007) | 5 lines Be afraid. Very afraid. A branch to experiment with rewriting the Prolog interpreter to make it work with the JIT. ------------------------------------------------------------------------ From arigo at codespeak.net Wed May 23 18:00:34 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 23 May 2007 18:00:34 +0200 (CEST) Subject: [pypy-svn] r43585 - in pypy/dist/pypy: annotation/test interpreter jit/hintannotator jit/hintannotator/test jit/timeshifter jit/timeshifter/test jit/tl lang/automata lang/prolog module/pypyjit objspace/std rlib rpython rpython/lltypesystem translator/goal Message-ID: <20070523160034.BFB278077@code0.codespeak.net> Author: arigo Date: Wed May 23 18:00:33 2007 New Revision: 43585 Added: pypy/dist/pypy/annotation/test/test_annrpython.py - copied unchanged from r43584, pypy/dist/pypy/annotation/test/test_annrpython.py.merge.tmp pypy/dist/pypy/interpreter/baseobjspace.py - copied unchanged from r43584, pypy/dist/pypy/interpreter/baseobjspace.py.merge.tmp pypy/dist/pypy/interpreter/gateway.py - copied unchanged from r43584, pypy/branch/prolog-jit-experiments/pypy/interpreter/gateway.py pypy/dist/pypy/interpreter/pyframe.py - copied unchanged from r43584, pypy/branch/prolog-jit-experiments/pypy/interpreter/pyframe.py pypy/dist/pypy/interpreter/pyopcode.py - copied unchanged from r43584, pypy/branch/prolog-jit-experiments/pypy/interpreter/pyopcode.py pypy/dist/pypy/jit/hintannotator/annotator.py - copied unchanged from r43584, pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator/annotator.py pypy/dist/pypy/jit/hintannotator/bookkeeper.py - copied unchanged from r43584, pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator/bookkeeper.py pypy/dist/pypy/jit/hintannotator/policy.py - copied unchanged from r43584, pypy/branch/prolog-jit-experiments/pypy/jit/hintannotator/policy.py pypy/dist/pypy/jit/hintannotator/test/test_annotator.py - copied unchanged from r43584, pypy/dist/pypy/jit/hintannotator/test/test_annotator.py.merge.tmp pypy/dist/pypy/jit/timeshifter/hrtyper.py - copied unchanged from r43584, pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/hrtyper.py pypy/dist/pypy/jit/timeshifter/rtimeshift.py - copied unchanged from r43584, pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/rtimeshift.py pypy/dist/pypy/jit/timeshifter/test/test_frontend.py - copied unchanged from r43584, pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_frontend.py pypy/dist/pypy/jit/timeshifter/test/test_portal.py - copied unchanged from r43584, pypy/dist/pypy/jit/timeshifter/test/test_portal.py.merge.tmp pypy/dist/pypy/jit/timeshifter/test/test_promotion.py - copied unchanged from r43584, pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_promotion.py pypy/dist/pypy/jit/timeshifter/test/test_timeshift.py - copied unchanged from r43584, pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_timeshift.py pypy/dist/pypy/jit/timeshifter/test/test_vdict.py - copied unchanged from r43584, pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_vdict.py pypy/dist/pypy/jit/timeshifter/test/test_virtualizable.py - copied unchanged from r43584, pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_virtualizable.py pypy/dist/pypy/jit/timeshifter/test/test_vlist.py - copied unchanged from r43584, pypy/branch/prolog-jit-experiments/pypy/jit/timeshifter/test/test_vlist.py pypy/dist/pypy/jit/tl/ - copied from r43584, pypy/branch/prolog-jit-experiments/pypy/jit/tl/ pypy/dist/pypy/lang/automata/ - copied from r43584, pypy/branch/prolog-jit-experiments/pypy/lang/automata/ pypy/dist/pypy/lang/prolog/ - copied from r43584, pypy/branch/prolog-jit-experiments/pypy/lang/prolog/ pypy/dist/pypy/module/pypyjit/ - copied from r43584, pypy/branch/prolog-jit-experiments/pypy/module/pypyjit/ pypy/dist/pypy/objspace/std/multimethod.py - copied unchanged from r43584, pypy/branch/prolog-jit-experiments/pypy/objspace/std/multimethod.py pypy/dist/pypy/objspace/std/objspace.py - copied unchanged from r43584, pypy/branch/prolog-jit-experiments/pypy/objspace/std/objspace.py pypy/dist/pypy/objspace/std/typeobject.py - copied unchanged from r43584, pypy/branch/prolog-jit-experiments/pypy/objspace/std/typeobject.py pypy/dist/pypy/rlib/jit.py - copied unchanged from r43584, pypy/branch/prolog-jit-experiments/pypy/rlib/jit.py pypy/dist/pypy/rlib/objectmodel.py - copied unchanged from r43584, pypy/branch/prolog-jit-experiments/pypy/rlib/objectmodel.py pypy/dist/pypy/rpython/llinterp.py - copied unchanged from r43584, pypy/branch/prolog-jit-experiments/pypy/rpython/llinterp.py pypy/dist/pypy/rpython/lltypesystem/ - copied from r43584, pypy/branch/prolog-jit-experiments/pypy/rpython/lltypesystem/ pypy/dist/pypy/translator/goal/ - copied from r43584, pypy/branch/prolog-jit-experiments/pypy/translator/goal/ Removed: pypy/dist/pypy/annotation/test/test_annrpython.py.merge.tmp pypy/dist/pypy/interpreter/baseobjspace.py.merge.tmp pypy/dist/pypy/jit/hintannotator/test/test_annotator.py.merge.tmp pypy/dist/pypy/jit/timeshifter/test/test_portal.py.merge.tmp Log: (cfbolz, arigo) Merge the prolog-jit-experiments branch. From arigo at codespeak.net Wed May 23 18:01:44 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 23 May 2007 18:01:44 +0200 (CEST) Subject: [pypy-svn] r43586 - pypy/branch/prolog-jit-experiments Message-ID: <20070523160144.91E09806E@code0.codespeak.net> Author: arigo Date: Wed May 23 18:01:44 2007 New Revision: 43586 Removed: pypy/branch/prolog-jit-experiments/ Log: (cfbolz, arigo) Delete merged branch. From arigo at codespeak.net Wed May 23 18:05:09 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 23 May 2007 18:05:09 +0200 (CEST) Subject: [pypy-svn] r43587 - pypy/extradoc/talk/dyla2007 Message-ID: <20070523160509.57110806E@code0.codespeak.net> Author: arigo Date: Wed May 23 18:05:09 2007 New Revision: 43587 Added: pypy/extradoc/talk/dyla2007/outline.txt - copied unchanged from r43574, pypy/extradoc/talk/dyla2007/draft.txt Removed: pypy/extradoc/talk/dyla2007/draft.txt Modified: pypy/extradoc/talk/dyla2007/dyla.txt Log: Small fix. Rename a file. Modified: pypy/extradoc/talk/dyla2007/dyla.txt ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.txt (original) +++ pypy/extradoc/talk/dyla2007/dyla.txt Wed May 23 18:05:09 2007 @@ -93,8 +93,8 @@ foreseeable one-VM-fits-all standardization attempt. The argument boils down to: VMs for dynamic languages should not be -written by hand! The justification is based on the successful -completion of the PyPy project, which proves that the approach is +written by hand! The justification is based on the +PyPy project, which proves that the approach is feasible in practice. Just as importantly, it also brings new insights and concrete benefits in term of flexibility and performance that go beyond the state of the art. From fijal at codespeak.net Wed May 23 18:35:24 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 23 May 2007 18:35:24 +0200 (CEST) Subject: [pypy-svn] r43588 - pypy/dist/pypy/translator/c Message-ID: <20070523163524.BFD558074@code0.codespeak.net> Author: fijal Date: Wed May 23 18:35:24 2007 New Revision: 43588 Modified: pypy/dist/pypy/translator/c/funcgen.py Log: Remove unused local var Modified: pypy/dist/pypy/translator/c/funcgen.py ============================================================================== --- pypy/dist/pypy/translator/c/funcgen.py (original) +++ pypy/dist/pypy/translator/c/funcgen.py Wed May 23 18:35:24 2007 @@ -552,7 +552,6 @@ elength, itemtype, eresult) - tail = '\n}' esize = 'sizeof(%s)-sizeof(%s)+%s*sizeof(%s)' % ( cdecl(typename, ''), itemtype, From arigo at codespeak.net Wed May 23 23:03:24 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 23 May 2007 23:03:24 +0200 (CEST) Subject: [pypy-svn] r43591 - pypy/extradoc/talk/dyla2007 Message-ID: <20070523210324.353A3807C@code0.codespeak.net> Author: arigo Date: Wed May 23 23:03:22 2007 New Revision: 43591 Modified: pypy/extradoc/talk/dyla2007/dyla.txt pypy/extradoc/talk/dyla2007/outline.txt Log: Intermediate check-in (this is in a rather inconsistent state) Modified: pypy/extradoc/talk/dyla2007/dyla.txt ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.txt (original) +++ pypy/extradoc/talk/dyla2007/dyla.txt Wed May 23 23:03:22 2007 @@ -28,10 +28,11 @@ can relatively easily be turned into C. The machine implements an object model supporting the high level dynamic language's objects. It typically provides features like automatic garbage collection. Recent -languages like Python, Ruby and Perl have complicated semantics which -are most easily mapped to a simple interpreter operating on syntax trees -or bytecode; simpler languages like Lisp and Self typically have more -efficient implementations based on just-in-time code generation. +languages like Python, Ruby, Perl and JavaScript have complicated +semantics which are most easily mapped to a simple interpreter operating +on syntax trees or bytecode; simpler languages like Lisp and Self +typically have more efficient implementations based on just-in-time code +generation. The efforts required to build a new virtual machine are relatively important. This is particularly true for languages which are complex @@ -43,26 +44,32 @@ an Open Source community with a limited amount of resources. The same constraint can be found in many academic projects. -However, there is a recent trend that enters in conflict with the goal -of having to maintain essentially a single, simple enough implementation -for a given programming language. This trend is to build an -implementation of the language on top of another virtual machine, like -the Java VM or the .NET one. +Limitations of C implementations are XXX (limited GC, limited speed, +C-only platforms, lack of flexibility). -As in the case of Python and Ruby, this results in multiple +To address some of these issues, a recent trend is to build an +implementation of the language on top of another virtual machine. Java +(and increasingly .NET) provides well-tuned object-oriented virtual +machines which offer many features that are of interest to all +programmers including language implementors (a higher level of +expression, automatic memory management, good just-in-time compilers, +richer libraries...). They also provide a higher level "base object +model", which enables better cross-language integration: a Jython +program, for example, can directly import Java classes and vice-versa. + +This trend enters in conflict with the goal of having to maintain +essentially a single, simple enough implementation for a given +programming language: as the language becomes popular, there will be a +demand for having it run on various platforms - high-level VMs as well +as C-level environments. + +In the case of Python and Ruby, this resulted in multiple implementations of the language, with an oldest custom VM written in C which is the "official" version in the sense that its evolution defines the evolution of the language itself. The other implementations - Jython and JRuby on the JVM, IronPython and IronRuby on .NET - ultimately need to be tediously kept up-to-date. -Java (and increasingly .NET) provides well-tuned object-oriented virtual -machines which offer many features that are of interest to all -programmers including language implementors (a higher level of -expression, automatic memory management, richer libraries...). They -also provide a higher level "base object model", which enables better -cross-language integration: a Jython program can directly import Java -classes and vice-versa. What is the best way to implement dynamic languages? ---------------------------------------------------- Modified: pypy/extradoc/talk/dyla2007/outline.txt ============================================================================== --- pypy/extradoc/talk/dyla2007/outline.txt (original) +++ pypy/extradoc/talk/dyla2007/outline.txt Wed May 23 23:03:22 2007 @@ -3,6 +3,21 @@ ~~~~~ + +If we write VMs by hand: + +* let's say we pick C, then (typical limitations: GC, perf, ...) + +* let's say we pick one OO VM, then advantages (GC, tools, interop) + but some of them disputable (perf, object model may mismatch) + +* above all that, implementation proliferation problem + + +=> Metaprogramming is good + + + Introduction ============ @@ -29,12 +44,13 @@ * implementations should be generated - don't write VMs by hand any more -The OO VM Arguments +Writing VMs by hand =================== - Writing dynamic language implementations in an OO VM gives advantages over writing custom VMs in C. Supposedly we get: * more interoperability than the C level + * cross-platform portability * better tools * better implementation of low-level issues like GC * better performance @@ -55,7 +71,8 @@ it gives a higher-level language but also an imposed model in which concepts must be mapped, which may be anywhere from easy to mostly impossible depending on the language we want to implement - (ref... e.g. bad functional languages support for .NET?) + (ref... e.g. bad Prolog on .NET and JVM) + * "cross-platform portability": XXX - Not all arguments are bogus: we get better tools and GCs in theory, and interoperability with the rest of the VM @@ -108,13 +125,17 @@ - Don't write dynamic language implementations "by hand" * writing them more abstractly at a high level has mostly only advantages + * we can still reuse existing good OO VMs - Don't write VMs by hand any more * certainly not language-specific VMs - * but even general-purpose OO VMs have trade-offs + * but even general-purpose OO VMs have trade-offs and are too much work + unless you have the necessary resources - Let's write more metaprogramming translation toolchains - * diversity is good (high level SSI good as starting base) + * diversity is good + * very large, mostly unexplored design space (some of PyPy's choices will + probably turn out to not be optimal) * ultimately a better investment of efforts than writing general-purpose VMs From antocuni at codespeak.net Thu May 24 13:37:27 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 24 May 2007 13:37:27 +0200 (CEST) Subject: [pypy-svn] r43593 - in pypy/dist/pypy/rpython: lltypesystem ootypesystem test Message-ID: <20070524113727.85EEF80AA@code0.codespeak.net> Author: antocuni Date: Thu May 24 13:37:27 2007 New Revision: 43593 Modified: pypy/dist/pypy/rpython/lltypesystem/rstr.py pypy/dist/pypy/rpython/ootypesystem/rstr.py pypy/dist/pypy/rpython/test/test_rstr.py Log: test&fix for int('+') and int('+ 42') Modified: pypy/dist/pypy/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/rstr.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/rstr.py Thu May 24 13:37:27 2007 @@ -650,8 +650,12 @@ i += 1 elif chars[i] == '+': i += 1; + # skip whitespaces between sign and digits + while i < strlen and chars[i] == ' ': + i += 1 #now get digits val = 0 + oldpos = i while i < strlen: c = ord(chars[i]) if ord('a') <= c <= ord('z'): @@ -666,6 +670,8 @@ break val = val * base + digit i += 1 + if i == oldpos: + raise ValueError # catch strings like '+' and '+ ' #skip trailing whitespace while i < strlen and chars[i] == ' ': i += 1 Modified: pypy/dist/pypy/rpython/ootypesystem/rstr.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/rstr.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/rstr.py Thu May 24 13:37:27 2007 @@ -172,8 +172,12 @@ i += 1 elif s.ll_stritem_nonneg(i) == '+': i += 1; + # skip whitespaces between sign and digits + while i < strlen and s.ll_stritem_nonneg(i) == ' ': + i += 1 #now get digits val = 0 + oldpos = i while i < strlen: c = ord(s.ll_stritem_nonneg(i)) if ord('a') <= c <= ord('z'): @@ -188,6 +192,8 @@ break val = val * base + digit i += 1 + if i == oldpos: + raise ValueError # catch strings like '+' and '+ ' #skip trailing whitespace while i < strlen and s.ll_stritem_nonneg(i) == ' ': i += 1 Modified: pypy/dist/pypy/rpython/test/test_rstr.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rstr.py (original) +++ pypy/dist/pypy/rpython/test/test_rstr.py Thu May 24 13:37:27 2007 @@ -577,7 +577,7 @@ raises(TyperError, self.interpret, fn, ()) def test_int(self): - s1 = [ '42', '01001', 'abc', 'ABC', '4aBc', ' 12ef ', '+42', 'foo', '42foo', '42.1', ''] + s1 = [ '42', '01001', 'abc', 'ABC', '4aBc', ' 12ef ', '+42', 'foo', '42foo', '42.1', '', '+ 42'] def fn(i, base): s = s1[i] res = int(s, base) @@ -593,7 +593,7 @@ assert res == expected def test_int_valueerror(self): - s1 = ['42g', '?'] + s1 = ['42g', '?', '+', '+ '] def fn(i): try: return int(s1[i]) @@ -603,6 +603,8 @@ assert res == -654 res = self.interpret(fn, [1]) assert res == -654 + res = self.interpret(fn, [2]) + assert res == -654 def test_float(self): f = ['', ' ', '0', '1', '-1.5', '1.5E2', '2.5e-1', ' 0 ', '?'] From tismer at codespeak.net Thu May 24 15:17:10 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Thu, 24 May 2007 15:17:10 +0200 (CEST) Subject: [pypy-svn] r43597 - pypy/extradoc/talk/rupy2007 Message-ID: <20070524131710.99BB880B2@code0.codespeak.net> Author: tismer Date: Thu May 24 15:17:06 2007 New Revision: 43597 Modified: pypy/extradoc/talk/rupy2007/talk.txt Log: a few typos and spelings Modified: pypy/extradoc/talk/rupy2007/talk.txt ============================================================================== --- pypy/extradoc/talk/rupy2007/talk.txt (original) +++ pypy/extradoc/talk/rupy2007/talk.txt Thu May 24 15:17:06 2007 @@ -134,7 +134,7 @@ Thunk object space ================== -* Build as different object space +* Built as different object space (object operation basis) * Can do lazy computation of objects @@ -142,7 +142,7 @@ Taint object space =================== -* Also build as different object space +* Also built as different object space * Few lines hack @@ -179,8 +179,8 @@ Interpreters in RPython ============================= -* Usefull -* There is working python interpreter +* Useful +* There is a working python interpreter * prolog interpreter * JavaScript interpreter coming soon * Ruby? Perl?? (with JIT almost for free) From arigo at codespeak.net Thu May 24 15:20:12 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 24 May 2007 15:20:12 +0200 (CEST) Subject: [pypy-svn] r43598 - pypy/extradoc/talk/dyla2007 Message-ID: <20070524132012.C890F80B5@code0.codespeak.net> Author: arigo Date: Thu May 24 15:20:11 2007 New Revision: 43598 Modified: pypy/extradoc/talk/dyla2007/outline.txt Log: Reorganized the outline. The introduction should be shorter now, with the bluk of the current situation and trend described in the following section. Modified: pypy/extradoc/talk/dyla2007/outline.txt ============================================================================== --- pypy/extradoc/talk/dyla2007/outline.txt (original) +++ pypy/extradoc/talk/dyla2007/outline.txt Thu May 24 15:20:11 2007 @@ -3,52 +3,51 @@ ~~~~~ - -If we write VMs by hand: - -* let's say we pick C, then (typical limitations: GC, perf, ...) - -* let's say we pick one OO VM, then advantages (GC, tools, interop) - but some of them disputable (perf, object model may mismatch) - -* above all that, implementation proliferation problem - - -=> Metaprogramming is good - - - Introduction ============ - Some kind of intro about dynamic language implementations - In the context of academic or Open Source communities: - limited resources + limited resources (applies also to DSLs) + +.. (keep the rest of the intro very short!) .. - What is the best way to implement dynamic languages? - Current situation: - * interpreters only - * for C (Posix, Windows), as custom VMs - * for Java (common in academia) - * for .NET (recent) - Trend: - * more support from OO VMs, better integration - * standardizing on one OO VM to increase interoperability + Current situation (e.g. Python): for C (Posix, Windows) as + custom VMs, several ones; for Java; for .NET; other incomplete + ones. Problem in our context: redundant and out-of-sync + implementations of the dyn language, divides the available resources. + +- Trend: + * OO VMs give more support and people want integration with OO VMs anyway + * so, some propose to standardize on one OO VM to increase interoperability and reduce redundancy and out-of-sync implementations of the dyn language - Our view on the issue: - * high-level VMs come with limitations - * a single OO VM for everybody is not a good idea + * we're opposed to that conclusion * this is really a metaprogramming issue, not a standardization issue * implementations should be generated - don't write VMs by hand any more -Writing VMs by hand -=================== +Advantages of OO VMs +==================== + +- C-level implementations of dyn language VMs have issues and trade-offs + (GC, perf, flexibility). This alone leads to multiple implementations + (Stackless, Psyco). + +- People eventually ask for better integration with successful OO VMs, + mere bridges are temporary hacks => more implementations + +- Proliferation of implementations, division of the effort, troublesome + if resources are limited. Implementations written for a specific OO VM + have advantages over custom VMs in C, so people propose as a solution + to standardize on one OO VM and just have this one implementation of + the dyn language. -- Writing dynamic language implementations in an OO VM gives advantages - over writing custom VMs in C. Supposedly we get: +- Let's first explore the alledged advantages of writing dynamic language + implementations in an OO VM over writing custom VMs in C: * more interoperability than the C level * cross-platform portability * better tools @@ -60,22 +59,25 @@ (expand each topic with concrete supporting data...) - Some of these are only partially true: - * there is no single best base VM and unclear if there will ever be: - so far, all major dynamic languages have multiple implementations - (Perl has only a C/Posix impl) and it's probably a good thing * "better performance": so far it's the other way around, and the VM's JIT compilers don't help much the very dynamic languages (expand...) * "better GCs" is unclear - obvious in theory, but OO VMs tend to have quite large memory overheads to start with (ref...) + * "cross-platform portability": yes to some extend, but e.g. C/Posix is + relatively portable too * "ease of implementation" disputable: it's really a trade-off: it gives a higher-level language but also an imposed model in which concepts must be mapped, which may be anywhere from easy to mostly impossible depending on the language we want to implement (ref... e.g. bad Prolog on .NET and JVM) - * "cross-platform portability": XXX -- Not all arguments are bogus: we get better tools and GCs in theory, - and interoperability with the rest of the VM +- Not all arguments are bogus: we get better tools and in theory GCs and + other low-level aspects, and of course interoperability with the rest + of the VM + +- A single best base OO VM for everyone would help reducing the + proliferation of implementations; but that's unlikely to occur, + and it would come with trade-offs too Metaprogramming Is Good From cfbolz at codespeak.net Thu May 24 16:06:39 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 24 May 2007 16:06:39 +0200 (CEST) Subject: [pypy-svn] r43599 - in pypy/dist/pypy/rpython: . test Message-ID: <20070524140639.BF25D80B9@code0.codespeak.net> Author: cfbolz Date: Thu May 24 16:06:39 2007 New Revision: 43599 Modified: pypy/dist/pypy/rpython/rbuiltin.py pypy/dist/pypy/rpython/test/test_rbuiltin.py Log: make min, max work for floats too. Modified: pypy/dist/pypy/rpython/rbuiltin.py ============================================================================== --- pypy/dist/pypy/rpython/rbuiltin.py (original) +++ pypy/dist/pypy/rpython/rbuiltin.py Thu May 24 16:06:39 2007 @@ -233,11 +233,7 @@ return vlist[0] def rtype_builtin_min(hop): - rint1, rint2 = hop.args_r - assert isinstance(rint1, IntegerRepr) - assert isinstance(rint2, IntegerRepr) - assert rint1.lowleveltype == rint2.lowleveltype - v1, v2 = hop.inputargs(rint1, rint2) + v1, v2 = hop.inputargs(hop.r_result, hop.r_result) return hop.gendirectcall(ll_min, v1, v2) def ll_min(i1, i2): @@ -246,11 +242,7 @@ return i2 def rtype_builtin_max(hop): - rint1, rint2 = hop.args_r - assert isinstance(rint1, IntegerRepr) - assert isinstance(rint2, IntegerRepr) - assert rint1.lowleveltype == rint2.lowleveltype - v1, v2 = hop.inputargs(rint1, rint2) + v1, v2 = hop.inputargs(hop.r_result, hop.r_result) return hop.gendirectcall(ll_max, v1, v2) def ll_max(i1, i2): Modified: pypy/dist/pypy/rpython/test/test_rbuiltin.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rbuiltin.py (original) +++ pypy/dist/pypy/rpython/test/test_rbuiltin.py Thu May 24 16:06:39 2007 @@ -113,6 +113,24 @@ assert self.interpret(fn, (2, 2)) == 2 assert self.interpret(fn, (-1, -12)) == -1 + def test_float_min(self): + def fn(i, j): + return min(i, j) + assert self.interpret(fn, (1.9, 2.)) == 1.9 + assert self.interpret(fn, (1.5, -1.4)) == -1.4 + + def test_float_int_min(self): + def fn(i, j): + return min(i, j) + assert self.interpret(fn, (1.9, 2)) == 1.9 + assert self.interpret(fn, (1.5, -1)) == -1 + + def test_float_max(self): + def fn(i, j): + return max(i,j) + assert self.interpret(fn, (1.0, 2.)) == 2 + assert self.interpret(fn, (1.1, -1)) == 1.1 + def test_builtin_math_floor(self): import math def fn(f): From cfbolz at codespeak.net Thu May 24 16:30:14 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 24 May 2007 16:30:14 +0200 (CEST) Subject: [pypy-svn] r43600 - in pypy/dist/pypy/lang/prolog/interpreter: . test Message-ID: <20070524143014.09B8E80BE@code0.codespeak.net> Author: cfbolz Date: Thu May 24 16:30:14 2007 New Revision: 43600 Modified: pypy/dist/pypy/lang/prolog/interpreter/arithmetic.py pypy/dist/pypy/lang/prolog/interpreter/error.py pypy/dist/pypy/lang/prolog/interpreter/helper.py pypy/dist/pypy/lang/prolog/interpreter/parsing.py pypy/dist/pypy/lang/prolog/interpreter/term.py pypy/dist/pypy/lang/prolog/interpreter/test/test_parsing.py Log: refactor arithmetic functions a bit. rename the .num attribute of Float instances too floatval, to be less confusing. Modified: pypy/dist/pypy/lang/prolog/interpreter/arithmetic.py ============================================================================== --- pypy/dist/pypy/lang/prolog/interpreter/arithmetic.py (original) +++ pypy/dist/pypy/lang/prolog/interpreter/arithmetic.py Thu May 24 16:30:14 2007 @@ -9,25 +9,83 @@ pattern_to_function = {} -def wrap_builtin_operation(pattern, unwrap_spec, can_overflow): - code = ["def f(engine, query):"] +class CodeCollector(object): + def __init__(self): + self.code = [] + self.blocks = [] + + def emit(self, line): + for line in line.split("\n"): + self.code.append(" " * (4 * len(self.blocks)) + line) + + def start_block(self, blockstarter): + assert blockstarter.endswith(":") + self.emit(blockstarter) + self.blocks.append(blockstarter) + + def end_block(self, starterpart=""): + block = self.blocks.pop() + assert starterpart in block, "ended wrong block %s with %s" % ( + block, starterpart) + + def tostring(self): + assert not self.blocks + return "\n".join(self.code) + +def wrap_builtin_operation(name, pattern, unwrap_spec, can_overflow, intversion): + code = CodeCollector() + code.start_block("def prolog_%s(engine, query):" % name) for i, spec in enumerate(unwrap_spec): varname = "var%s" % (i, ) - code.append( - " %s = eval_arithmetic(engine, query.args[%s])" % (varname, i)) - code.append( - " v%s = 0" % (i, )) - code.append(" if isinstance(%s, term.Number):" % (varname, )) - code.append(" v%s = %s.num" % (i, varname)) + code.emit("%s = eval_arithmetic(engine, query.args[%s])" % + (varname, i)) + for i, spec in enumerate(unwrap_spec): + varname = "var%s" % (i, ) + if spec == "int": + code.start_block( + "if not isinstance(%s, term.Number):" % (varname, )) + code.emit("error.throw_type_error('int', %s)" % (varname, )) + code.end_block("if") + if "expr" in unwrap_spec and intversion: + # check whether all arguments are ints + for i, spec in enumerate(unwrap_spec): + varname = "var%s" % (i, ) + if spec == "int": + continue + code.start_block( + "if isinstance(%s, term.Number):" % (varname, )) + code.emit("v%s = var%s.num" % (i, i)) + code.emit("return term.Number(int(%s))" % (pattern, )) + for i, spec in enumerate(unwrap_spec): + if spec == "int": + continue + code.end_block("if") + + #general case in an extra function + args = ", ".join(["var%s" % i for i in range(len(unwrap_spec))]) + code.emit("return general_%s(%s)" % (name, args)) + code.end_block("def") + code.start_block("def general_%s(%s):" % (name, args)) + for i, spec in enumerate(unwrap_spec): + varname = "var%s" % (i, ) + code.emit("v%s = 0" % (i, )) + code.start_block("if isinstance(%s, term.Number):" % (varname, )) + code.emit("v%s = %s.num" % (i, varname)) + code.end_block("if") + expected = 'int' if spec == "expr": - code.append(" elif isinstance(%s, term.Float):" % (varname, )) - code.append(" v%s = %s.num" % (i, varname)) - code.append(" else:") - code.append(" error.throw_type_error('int', %s)" % (varname, )) - code.append(" return norm_float(%s)" % pattern) + code.start_block("elif isinstance(%s, term.Float):" % (varname, )) + code.emit("v%s = %s.floatval" % (i, varname)) + code.end_block("elif") + expected = 'float' + code.start_block("else:") + code.emit("error.throw_type_error('%s', %s)" % (expected, varname, )) + code.end_block("else") + code.emit("return norm_float(term.Float(%s))" % pattern) + code.end_block("def") miniglobals = globals().copy() - exec py.code.Source("\n".join(code)).compile() in miniglobals - return miniglobals['f'] + exec py.code.Source(code.tostring()).compile() in miniglobals + return miniglobals["prolog_" + name] wrap_builtin_operation._annspecialcase_ = 'specialize:memo' @@ -36,14 +94,14 @@ if isinstance(query, term.Number): return query if isinstance(query, term.Float): - return norm_float(query.num) + return norm_float(query) if isinstance(query, term.Atom): #XXX beautify that if query.name == "pi": return term.Float(math.pi) if query.name == "e": return term.Float(math.e) - raise error.UncatchableError("not implemented") + error.throw_type_error("evaluable", query.get_prolog_signature()) if isinstance(query, term.Term): func = arithmetic_functions.get(query.signature, None) if func is None: @@ -51,37 +109,45 @@ return func(engine, query) raise error.UncatchableError("not implemented") -def norm_float(v): +def norm_float(obj): + v = obj.floatval if v == int(v): return term.Number(int(v)) else: - return term.Float(v) + return obj simple_functions = [ - ("+", ["expr", "expr"], "v0 + v1", True), - ("-", ["expr", "expr"], "v0 - v1", True), - ("*", ["expr", "expr"], "v0 * v1", True), - ("//", ["int", "int"], "v0 / v1", True), - ("**", ["expr", "expr"], "v0 ** v1", True), - (">>", ["int", "int"], "v0 >> v1", False), - ("<<", ["int", "int"], "intmask(v0 << v1)", False), - ("\\/", ["int", "int"], "v0 | v1", False), - ("/\\", ["int", "int"], "v0 & v1", False), - ("xor", ["int", "int"], "v0 ^ v1", False), - ("mod", ["int", "int"], "v0 % v1", False), - ("\\", ["int"], "v0 ^ 0", False), - ("abs", ["expr"], "abs(v0)", True), -# ("max", ["expr", "expr"], "max(v0, v1)", False), -# ("min", ["expr", "expr"], "min(v0, v1)", False), - ("round", ["expr"], "int(v0 + 0.5)", False), - ("floor", ["expr"], "math.floor(v0)", False), - ("ceiling", ["expr"], "math.ceil(v0)", False), - ("floor", ["expr"], "math.floor(v0)", False), - ("float_fractional_part", ["expr"], "v0 - int(v0)", False), - ("float_integer_part", ["expr"], "int(v0)", False), + ("+", ["expr", "expr"], "v0 + v1", True, True), + ("-", ["expr", "expr"], "v0 - v1", True, True), + ("*", ["expr", "expr"], "v0 * v1", True, True), + ("//", ["int", "int"], "v0 / v1", True, False), + ("**", ["expr", "expr"], "v0 ** v1", True, True), + (">>", ["int", "int"], "v0 >> v1", False, False), + ("<<", ["int", "int"], "intmask(v0 << v1)", False, + False), + ("\\/", ["int", "int"], "v0 | v1", False, False), + ("/\\", ["int", "int"], "v0 & v1", False, False), + ("xor", ["int", "int"], "v0 ^ v1", False, False), + ("mod", ["int", "int"], "v0 % v1", False, False), + ("\\", ["int"], "~v0", False, False), + ("abs", ["expr"], "abs(v0)", True, True), + ("max", ["expr", "expr"], "max(v0, v1)", False, True), + ("min", ["expr", "expr"], "min(v0, v1)", False, True), + ("round", ["expr"], "int(v0 + 0.5)", False, False), + ("floor", ["expr"], "math.floor(v0)", False, True), #XXX + ("ceiling", ["expr"], "math.ceil(v0)", False, True), #XXX + ("float_fractional_part", ["expr"], "v0 - int(v0)", False, True), #XXX + ("float_integer_part", ["expr"], "int(v0)", False, True), ] -for prolog_name, unwrap_spec, pattern, overflow in simple_functions: - f = wrap_builtin_operation(pattern, unwrap_spec, overflow) +for prolog_name, unwrap_spec, pattern, overflow, intversion in simple_functions: + # the name is purely for flowgraph viewing reasons + if prolog_name.replace("_", "").isalnum(): + name = prolog_name + else: + import unicodedata + name = "".join([unicodedata.name(unicode(c)).replace(" ", "_").replace("-", "").lower() for c in prolog_name]) + f = wrap_builtin_operation(name, pattern, unwrap_spec, overflow, + intversion) signature = "%s/%s" % (prolog_name, len(unwrap_spec)) arithmetic_functions[signature] = f Modified: pypy/dist/pypy/lang/prolog/interpreter/error.py ============================================================================== --- pypy/dist/pypy/lang/prolog/interpreter/error.py (original) +++ pypy/dist/pypy/lang/prolog/interpreter/error.py Thu May 24 16:30:14 2007 @@ -25,8 +25,6 @@ def __init__(self, continuation): self.continuation = continuation - pass - def throw_instantiation_error(): from pypy.lang.prolog.interpreter import term raise CatchableError(term.Atom.newatom("instantiation_error")) Modified: pypy/dist/pypy/lang/prolog/interpreter/helper.py ============================================================================== --- pypy/dist/pypy/lang/prolog/interpreter/helper.py (original) +++ pypy/dist/pypy/lang/prolog/interpreter/helper.py Thu May 24 16:30:14 2007 @@ -36,7 +36,7 @@ if isinstance(obj, term.Number): return obj.num elif isinstance(obj, term.Float): - f = obj.num; i = int(f) + f = obj.floatval; i = int(f) if f == i: return i error.throw_type_error('integer', obj) @@ -74,6 +74,6 @@ elif isinstance(obj, term.Number): return str(obj.num) elif isinstance(obj, term.Float): - return str(obj.num) + return str(obj.floatval) error.throw_type_error("atomic", obj) Modified: pypy/dist/pypy/lang/prolog/interpreter/parsing.py ============================================================================== --- pypy/dist/pypy/lang/prolog/interpreter/parsing.py (original) +++ pypy/dist/pypy/lang/prolog/interpreter/parsing.py Thu May 24 16:30:14 2007 @@ -321,7 +321,7 @@ if isinstance(child, Number): return Number(factor * child.num) if isinstance(child, Float): - return Float(factor * child.num) + return Float(factor * child.floatval) return Term(name, children) def build_list(self, node): @@ -391,7 +391,7 @@ if isinstance(result, Number): return Number(-result.num) elif isinstance(result, Float): - return Float(-result.num) + return Float(-result.floatval) return self.visit(node.children[1]) def visit_listexpr(self, node): Modified: pypy/dist/pypy/lang/prolog/interpreter/term.py ============================================================================== --- pypy/dist/pypy/lang/prolog/interpreter/term.py (original) +++ pypy/dist/pypy/lang/prolog/interpreter/term.py Thu May 24 16:30:14 2007 @@ -318,12 +318,12 @@ TAG = tag() STANDARD_ORDER = 2 _immutable_ = True - def __init__(self, num): - self.num = num + def __init__(self, floatval): + self.floatval = floatval @specialize.arg(3) def basic_unify(self, other, heap, occurs_check=False): - if isinstance(other, Float) and other.num == self.num: + if isinstance(other, Float) and other.floatval == self.floatval: return raise UnificationFailed @@ -332,22 +332,22 @@ def copy_and_basic_unify(self, other, heap, memo): hint(self, concrete=True) - if isinstance(other, Float) and other.num == self.num: + if isinstance(other, Float) and other.floatval == self.floatval: return self else: raise UnificationFailed def get_unify_hash(self, heap): #XXX no clue whether this is a good idea... - m, e = math.frexp(self.num) + m, e = math.frexp(self.floatval) m = intmask(int(m / 2 * 2 ** (32 - TAGBITS))) return intmask(m << TAGBITS | self.TAG) def __str__(self): - return repr(self.num) + return repr(self.floatval) def __repr__(self): - return "Float(%r)" % (self.num, ) + return "Float(%r)" % (self.floatval, ) class BlackBox(NonVar): # meant to be subclassed @@ -589,10 +589,10 @@ if isinstance(obj2, Number): return rcmp(obj1.num, obj2.num) elif isinstance(obj2, Float): - return rcmp(obj1.num, obj2.num) + return rcmp(obj1.num, obj2.floatval) if isinstance(obj1, Float): if isinstance(obj2, Number): - return rcmp(obj1.num, obj2.num) + return rcmp(obj1.floatval, obj2.num) elif isinstance(obj2, Float): - return rcmp(obj1.num, obj2.num) + return rcmp(obj1.floatval, obj2.floatval) assert 0 Modified: pypy/dist/pypy/lang/prolog/interpreter/test/test_parsing.py ============================================================================== --- pypy/dist/pypy/lang/prolog/interpreter/test/test_parsing.py (original) +++ pypy/dist/pypy/lang/prolog/interpreter/test/test_parsing.py Thu May 24 16:30:14 2007 @@ -94,7 +94,7 @@ facts = builder.build(t) assert len(facts) == 2 assert facts[0].args[1].num == -1 - assert facts[1].args[1].num == -1.345 + assert facts[1].args[1].floatval == -1.345 t = parse_file(""" X = -1. arg(X, h(a, b, c), b), X = 2. From cfbolz at codespeak.net Thu May 24 17:05:17 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 24 May 2007 17:05:17 +0200 (CEST) Subject: [pypy-svn] r43601 - pypy/dist/pypy/lang/prolog/interpreter/test Message-ID: <20070524150517.5408F80BF@code0.codespeak.net> Author: cfbolz Date: Thu May 24 17:05:17 2007 New Revision: 43601 Modified: pypy/dist/pypy/lang/prolog/interpreter/test/test_arithmetic.py Log: more tests Modified: pypy/dist/pypy/lang/prolog/interpreter/test/test_arithmetic.py ============================================================================== --- pypy/dist/pypy/lang/prolog/interpreter/test/test_arithmetic.py (original) +++ pypy/dist/pypy/lang/prolog/interpreter/test/test_arithmetic.py Thu May 24 17:05:17 2007 @@ -10,10 +10,78 @@ assert_true("X is 1 + 2, X = 3.") assert_true("X is 1.2 + 2.8, X = 4.") assert_false("X is 1.1 + 2.8, X = 4.0.") + assert_true("X is 1 - 2, X = -1.") + assert_true("X is 1.2 - 1.2, X = 0.") assert_true("X is 2 * -2, X = -4.") + assert_true("X is 2 * -2.1, X = -4.2.") assert_true("X is 2 + -2, X = 0.") assert_true("X is 2 // -2, X = -1.") + assert_true("X is 1 << 4, X = 16.") + assert_true("X is 128 >> 7, X = 1.") + assert_true("X is 12 \\/ 10, X = 14.") + assert_true("X is 12 /\\ 10, X = 8.") + assert_true("X is 12 xor 10, X = 6.") + + assert_true("X is max(12, 13), X = 13.") + assert_true("X is min(12, 13), X = 12.") + assert_true("X is max(12, 13.9), X = 13.9.") + assert_true("X is min(12.1, 13), X = 12.1.") + + assert_true("X is abs(42), X = 42.") + assert_true("X is abs(-42), X = 42.") + assert_true("X is abs(42.42), X = 42.42.") + assert_true("X is abs(-42.42), X = 42.42.") + + assert_true("X is round(0), X = 0.") + assert_true("X is round(0.3), X = 0.") + assert_true("X is round(0.4), X = 0.") + assert_true("X is round(0.5), X = 1.") + assert_true("X is round(0.6), X = 1.") + assert_true("X is round(1), X = 1.") + assert_true("X is round(-0.3), X = 0.") + assert_true("X is round(-0.4), X = 0.") + assert_true("X is round(-0.5), X = 0.") + #assert_true("X is round(-0.6), X = -1.") #XXX fix round + #assert_true("X is round(-1), X = -1.") + + assert_true("X is ceiling(0), X = 0.") + assert_true("X is ceiling(0.3), X = 1.") + assert_true("X is ceiling(0.4), X = 1.") + assert_true("X is ceiling(0.5), X = 1.") + assert_true("X is ceiling(0.6), X = 1.") + assert_true("X is ceiling(1), X = 1.") + assert_true("X is ceiling(-0.3), X = 0.") + assert_true("X is ceiling(-0.4), X = 0.") + assert_true("X is ceiling(-0.5), X = 0.") + assert_true("X is ceiling(-0.6), X = 0.") + assert_true("X is ceiling(-1), X = -1.") + + assert_true("X is floor(0), X = 0.") + assert_true("X is floor(0.3), X = 0.") + assert_true("X is floor(0.4), X = 0.") + assert_true("X is floor(0.5), X = 0.") + assert_true("X is floor(0.6), X = 0.") + assert_true("X is floor(1), X = 1.") + assert_true("X is floor(-0.3), X = -1.") + assert_true("X is floor(-0.4), X = -1.") + assert_true("X is floor(-0.5), X = -1.") + assert_true("X is floor(-0.6), X = -1.") + assert_true("X is floor(-1), X = -1.") + + assert_true("X is float_integer_part(0), X = 0.") + assert_true("X is float_integer_part(0.3), X = 0.") + assert_true("X is float_integer_part(0.4), X = 0.") + assert_true("X is float_integer_part(0.5), X = 0.") + assert_true("X is float_integer_part(0.6), X = 0.") + assert_true("X is float_integer_part(1), X = 1.") + assert_true("X is float_integer_part(-0.3), X = 0.") + assert_true("X is float_integer_part(-0.4), X = 0.") + assert_true("X is float_integer_part(-0.5), X = 0.") + assert_true("X is float_integer_part(-0.6), X = 0.") + assert_true("X is float_integer_part(-1), X = -1.") + + def test_comparison(): assert_true("1 =:= 1.0.") assert_true("1 + 1 > 1.") From arigo at codespeak.net Thu May 24 18:01:35 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 24 May 2007 18:01:35 +0200 (CEST) Subject: [pypy-svn] r43602 - pypy/extradoc/talk/dyla2007 Message-ID: <20070524160135.AC04F80C4@code0.codespeak.net> Author: arigo Date: Thu May 24 18:01:34 2007 New Revision: 43602 Modified: pypy/extradoc/talk/dyla2007/outline.txt Log: Outline tweaks. Modified: pypy/extradoc/talk/dyla2007/outline.txt ============================================================================== --- pypy/extradoc/talk/dyla2007/outline.txt (original) +++ pypy/extradoc/talk/dyla2007/outline.txt Thu May 24 18:01:34 2007 @@ -90,18 +90,18 @@ - PyPy architecture in metaprogramming terms: * very expressive object language (RPython) for language VMs and semantics * very expressive metalanguage (Python) for analysis and susccessive transformation - * SSI for representing VM spec on meta level * transformations add aspects and specialize to fit virtual or hardware runtime environment - Makes interpreters easy to write, update, and generally experiment with * more expressiveness helps on all levels (use d12's - security references and texts) + security references and texts, mention the thunk space) * the requirement "specify an interpreter for your language" is much less strong than "fit it into the OO VM's model" [e.g. Pyrolog on .NET] * transformations [Stackless] - We can get similarly good GCs and tools + * (if we really want - still less work than writing a complete good OO VM) * no easier or harder than what needs to be put in an OO VM * existing GCs can also be reused * a metaprogramming translation toolchain requires a lot of work, From santagada at codespeak.net Thu May 24 20:45:07 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Thu, 24 May 2007 20:45:07 +0200 (CEST) Subject: [pypy-svn] r43603 - in pypy/dist/pypy/lang/js: . test test/ecma Message-ID: <20070524184507.E96A1809A@code0.codespeak.net> Author: santagada Date: Thu May 24 20:45:07 2007 New Revision: 43603 Modified: pypy/dist/pypy/lang/js/astbuilder.py pypy/dist/pypy/lang/js/interpreter.py pypy/dist/pypy/lang/js/js_interactive.py pypy/dist/pypy/lang/js/jsgrammar.txt pypy/dist/pypy/lang/js/jsobj.py pypy/dist/pypy/lang/js/operations.py pypy/dist/pypy/lang/js/test/ecma/conftest.py pypy/dist/pypy/lang/js/test/ecma/shell.js pypy/dist/pypy/lang/js/test/test_interp.py Log: completed scoping, the fijal idea for that was great Modified: pypy/dist/pypy/lang/js/astbuilder.py ============================================================================== --- pypy/dist/pypy/lang/js/astbuilder.py (original) +++ pypy/dist/pypy/lang/js/astbuilder.py Thu May 24 20:45:07 2007 @@ -1,32 +1,6 @@ from pypy.rlib.parsing.tree import RPythonVisitor, Symbol, Nonterminal from pypy.lang.js import operations -#this is a noop for now -def varfinder(opnode): - return [] - if isinstance(opnode, operations.Vars): - return [opnode,] - elif hasattr(opnode, "nodes"): - temp = [] - for op in opnode.nodes: - temp.extend(varfinder(op)) - return temp - elif hasattr(opnode, "body"): - return varfinder(opnode.body) - else: - return [] - -#this is a noop for now -def funcfinder(opnode): - return [] - if isinstance(opnode, operations.Function): - return [opnode,] - elif hasattr(opnode, "nodes"): - return [funcfinder(op) for op in opnode.nodes] - elif hasattr(opnode, "body"): - return funcfinder(opnode.body) - else: - return [] class ASTBuilder(RPythonVisitor): BINOP_TO_CLS = { @@ -48,8 +22,10 @@ '>=': operations.Ge, '<': operations.Lt, '<=': operations.Le, - '.': operations.Member, + '.': operations.MemberDot, '[': operations.Member, + ',': operations.Comma, + 'in': operations.In, } UNOP_TO_CLS = { '~': operations.BitwiseNot, @@ -110,7 +86,6 @@ pos = self.get_pos(op) right = self.dispatch(node.children[i * 2 + 2]) result = self.BINOP_TO_CLS[op.additional_info](pos, left, right) - print left, right left = result return left visit_additiveexpression = binaryop @@ -122,6 +97,7 @@ visit_logicalorexpression = binaryop visit_logicalandexpression = binaryop visit_relationalexpression = binaryop + visit_expression = binaryop def visit_memberexpression(self, node): if isinstance(node.children[0], Symbol) and \ @@ -177,7 +153,12 @@ pos = self.get_pos(node) nodes = [self.dispatch(child) for child in node.children[1:]] return operations.ArgumentList(pos, nodes) - + + def visit_formalparameterlist(self, node): + pos = self.get_pos(node) + nodes = [self.dispatch(child) for child in node.children] + return operations.ArgumentList(pos, nodes) + def visit_variabledeclarationlist(self, node): pos = self.get_pos(node) nodes = [self.dispatch(child) for child in node.children] @@ -193,12 +174,14 @@ pos = self.get_pos(node) name = node.additional_info return operations.Identifier(pos, name) - + def visit_program(self, node): + self.varlists = [] + self.funclists = [] pos = self.get_pos(node) body = self.dispatch(node.children[0]) return operations.Program(pos, body) - + def visit_variablestatement(self, node): pos = self.get_pos(node) body = self.dispatch(node.children[0]) @@ -211,27 +194,53 @@ def visit_sourceelements(self, node): pos = self.get_pos(node) - nodes = [self.dispatch(child) for child in node.children] - var_decl = [] - func_decl = [] - for node in nodes: - var_decl.extend(varfinder(node)) - func_decl.extend(funcfinder(node)) - + self.varlists.append(set()) + self.funclists.append({}) + nodes=[] + for child in node.children: + node = self.dispatch(child) + if node is not None: + nodes.append(node) + var_decl = self.varlists.pop() + func_decl = self.funclists.pop() + print var_decl, func_decl return operations.SourceElements(pos, var_decl, func_decl, nodes) - def visit_expressionstatement(self, node): - return self.dispatch(node.children[0]) - + def functioncommon(self, node, declaration=True): + pos = self.get_pos(node) + i=0 + identifier, i = self.get_next_expr(node, i) + parameters, i = self.get_next_expr(node, i) + functionbody, i = self.get_next_expr(node, i) + if parameters == operations.astundef: + p = [] + else: + p = [pident.get_literal() for pident in parameters.nodes] + funcobj = operations.FunctionStatement(pos, identifier, p, functionbody) + if declaration: + self.funclists[-1][identifier.get_literal()] = funcobj + return funcobj + + def visit_functiondeclaration(self, node): + self.functioncommon(node) + return None + + def visit_functionexpression(self, node): + return self.functioncommon(node, declaration=False) + def visit_variabledeclaration(self, node): pos = self.get_pos(node) identifier = self.dispatch(node.children[0]) + self.varlists[-1].add(identifier.get_literal()) if len(node.children) > 1: expr = self.dispatch(node.children[1]) else: expr = None return operations.VariableDeclaration(pos, identifier, expr) + def visit_expressionstatement(self, node): + return self.dispatch(node.children[0]) + def visit_callexpression(self, node): pos = self.get_pos(node) left = self.dispatch(node.children[0]) @@ -245,10 +254,7 @@ right = self.dispatch(node.children[2]) return operations.Assignment(pos, left, right, atype) visit_assignmentexpressionnoin = visit_assignmentexpression - - def visit_functiondeclaration(self, node): - pos = self.get_pos(node) - + def visit_emptystatement(self, node): return operations.astundef @@ -294,7 +300,7 @@ def get_next_expr(self, node, i): if isinstance(node.children[i], Symbol) and \ - node.children[i].additional_info in [';', ')']: + node.children[i].additional_info in [';', ')', '(', '}']: return operations.astundef, i+1 else: return self.dispatch(node.children[i]), i+2 @@ -306,3 +312,38 @@ else: target = operations.astundef return operations.Break(pos, target) + + def visit_returnstatement(self, node): + pos = self.get_pos(node) + if len(node.children) > 0: + value = self.dispatch(node.children[0]) + else: + value = operations.astundef + return operations.Return(pos, value) + + def visit_conditionalexpression(self, node): + pos = self.get_pos(node) + condition = self.dispatch(node.children[0]) + truepart = self.dispatch(node.children[2]) + falsepart = self.dispatch(node.children[3]) + return operations.Conditional(pos, condition, truepart, falsepart) + + def visit_trystatement(self, node): + pos = self.get_pos(node) + tryblock = self.dispatch(node.children[0]) + catchparam = None + catchblock = None + finallyblock = None + if node.children[1].children[0].additional_info == "catch": + catchparam = self.dispatch(node.children[1].children[1]) + catchblock = self.dispatch(node.children[1].children[2]) + if len(node.children) > 2: + finallyblock = self.dispatch(node.children[2].children[1]) + else: + finallyblock = self.dispatch(node.children[1].children[1]) + return operations.Try(pos, tryblock, catchparam, catchblock, finallyblock) + + def visit_primaryexpression(self, node): + pos = self.get_pos(node) + return operations.This(pos, 'this') + \ No newline at end of file Modified: pypy/dist/pypy/lang/js/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/js/interpreter.py (original) +++ pypy/dist/pypy/lang/js/interpreter.py Thu May 24 20:45:07 2007 @@ -14,25 +14,15 @@ temp_tree = parse(script_source) return astb.dispatch(temp_tree) -import cPickle as pickle import os.path def load_file(filename): # NOT RPYTHON base, ext = os.path.splitext(filename) - jscname = base+".jsc" - if os.path.isfile(jscname): - jsc = open(jscname, 'r') - t = pickle.load(jsc) - jsc.close() - else: - f = open(filename) - t = parse(f.read()) - f.close() - jsc = open(jscname, 'w') - pickle.dump(t, jsc, protocol=2) - jsc.close() - return from_tree(t) + f = open(filename) + t = load_source(f.read()) + f.close() + return t def evaljs(ctx, args, this): Modified: pypy/dist/pypy/lang/js/js_interactive.py ============================================================================== --- pypy/dist/pypy/lang/js/js_interactive.py (original) +++ pypy/dist/pypy/lang/js/js_interactive.py Thu May 24 20:45:07 2007 @@ -95,9 +95,10 @@ loadjs(interp.global_context, [W_String(filename)], None) # XXX we should catch more stuff here, like not implemented # and such - except (jsparser.JsSyntaxError, ThrowException), e: - if isinstance(e, jsparser.JsSyntaxError): + except (jsparser.ParseError, ThrowException), e: + if isinstance(e, jsparser.ParseError): print "\nSyntax error!" + raise elif isinstance(e, ThrowException): print "\nJS Exception thrown!" return Modified: pypy/dist/pypy/lang/js/jsgrammar.txt ============================================================================== --- pypy/dist/pypy/lang/js/jsgrammar.txt (original) +++ pypy/dist/pypy/lang/js/jsgrammar.txt Thu May 24 20:45:07 2007 @@ -129,10 +129,10 @@ | ["try"] block finally ; -catch : ["catch"] ["("] identifier [")"] block +catch : "catch" ["("] identifier [")"] block ; -finally : ["finally"] block +finally : "finally" block ; identifier : ; @@ -205,10 +205,10 @@ | ; -functiondeclaration : ["function"] identifier ["("] formalparameterlist? [")"] ["{"] functionbody? ["}"] +functiondeclaration : ["function"] identifier "(" formalparameterlist? ")" ["{"] functionbody? "}" ; -functionexpression : ["function"] identifier? ["("] formalparameterlist? [")"] ["{"] functionbody? ["}"] +functionexpression : ["function"] identifier? "(" formalparameterlist? ")" ["{"] functionbody? "}" ; formalparameterlist : identifier [","] >formalparameterlist< @@ -313,7 +313,7 @@ ; -conditionalexpression : logicalorexpression "?" assignmentexpression ":" assignmentexpression +conditionalexpression : logicalorexpression "?" assignmentexpression [":"] assignmentexpression | ; @@ -325,7 +325,7 @@ | ">>=" | ">>>=" | "&=" | "^=" | "\|=" ; -expression : assignmentexpression ([","] assignmentexpression)+ +expression : assignmentexpression ("," assignmentexpression)+ | ; Modified: pypy/dist/pypy/lang/js/jsobj.py ============================================================================== --- pypy/dist/pypy/lang/js/jsobj.py (original) +++ pypy/dist/pypy/lang/js/jsobj.py Thu May 24 20:45:07 2007 @@ -143,13 +143,14 @@ def Call(self, ctx, args=[], this=None): act = ActivationObject() - for i in range(len(self.callfunc.params)): - arg = self.callfunc.params[i] + paramn = len(self.callfunc.params) + for i in range(paramn): + paramname = self.callfunc.params[i] try: value = args[i] except IndexError: value = w_Undefined - act.Put(self.callfunc.params[i], value) + act.Put(paramname, value) act.Put('this', this) w_Arguments = W_Arguments(self, args) act.Put('arguments', w_Arguments) @@ -455,23 +456,29 @@ return str(self.list_w) class ExecutionContext(object): - def __init__(self): - self.scope = [] - self.this = None - self.variable = None - self.debug = False - self.property = Property('',w_Undefined) #Attribute flags for new vars + def __init__(self, scope, this=None, variable=None, debug=False, jsproperty=None): + assert scope is not None + self.scope = scope + if this is None: + self.this = scope[-1] + else: + self.this = this + + if variable is None: + self.variable = self.scope[0] + else: + self.variable = variable + self.debug = debug + if jsproperty is None: + self.property = Property('',w_Undefined) #Attribute flags for new vars + else: + self.property = jsproperty def __str__(self): - return ""%(str(self.scope)) + return ""%(self.scope, self.variable) def assign(self, name, value): - """ - assign to property name, creating it if it doesn't exist - """ pass - #ref = self.resolve_identifier(name) - #if ref. def get_global(self): return self.scope[-1] @@ -494,31 +501,28 @@ def global_context(w_global=W_Object()): - ctx = ExecutionContext() - ctx.push_object(w_global) - ctx.this = w_global - ctx.property = Property('', w_Undefined, dd=True) + ctx = ExecutionContext([w_global], + this = w_global, + variable = w_global, + jsproperty = Property('', w_Undefined, dd=True)) return ctx def function_context(scope, activation, this=None): - ctx = ExecutionContext() - ctx.scope = scope[:] + newscope = scope[:] + ctx = ExecutionContext(newscope, + this = this, + jsproperty = Property('', w_Undefined, dd=True)) ctx.push_object(activation) - if this is None: - ctx.this = ctx.get_global() - else: - ctx.this = this - ctx.property = Property('', w_Undefined, dd=True) return ctx def eval_context(calling_context): - ctx = ExecutionContext() - ctx.scope = calling_context.scope[:] - ctx.this = calling_context.this - ctx.variable = calling_context.variable - ctx.property = Property('', w_Undefined) + ctx = ExecutionContext(calling_context.scope[:], + this = calling_context.this, + variable = calling_context.variable, + jsproperty = Property('', w_Undefined)) return ctx + class W_Reference(W_Root): """Reference Type""" def __init__(self, property_name, base=None): @@ -534,7 +538,7 @@ def PutValue(self, w, ctx): base = self.base - if self.base is None: + if base is None: base = ctx.scope[-1] base.Put(self.property_name, w) return w Modified: pypy/dist/pypy/lang/js/operations.py ============================================================================== --- pypy/dist/pypy/lang/js/operations.py (original) +++ pypy/dist/pypy/lang/js/operations.py Thu May 24 20:45:07 2007 @@ -10,6 +10,9 @@ from pypy.rlib.rarithmetic import r_uint, intmask from constants import unescapedict, SLASH +import sys +import os + class Position(object): def __init__(self, lineno=-1, start=-1, end=-1): self.lineno = lineno @@ -224,60 +227,55 @@ class Comma(BinaryOp): - opcode = 'COMMA' - def eval(self, ctx): self.left.eval(ctx) return self.right.eval(ctx) + class Conditional(Expression): - opcode = 'CONDITIONAL' - - def __init__(self, pos, t): - self.logicalexpr = get_obj(t, '0') - self.trueop = get_obj(t, '1') - self.falseop = get_obj(t, '2') - + def __init__(self, pos, condition, truepart, falsepart): + self.pos = pos + self.condition = condition + self.truepart = truepart + self.falsepart = falsepart + def eval(self, ctx): - if self.logicalexpr.eval(ctx).GetValue().ToBoolean(): - return self.trueop.eval(ctx).GetValue() + if self.condition.eval(ctx).GetValue().ToBoolean(): + return self.truepart.eval(ctx).GetValue() else: - return self.falseop.eval(ctx).GetValue() + return self.falsepart.eval(ctx).GetValue() + class Member(BinaryOp): def eval(self, ctx): w_obj = self.left.eval(ctx).GetValue().ToObject() - name = self.right.eval(ctx).GetPropertyName() + name = self.right.eval(ctx).GetValue().ToString() return W_Reference(name, w_obj) - -class FunctionStatement(Statement): - def __init__(self, pos, name, body, params): - self.name = name - self.body = body - self.params = params - - def execute(self, ctx): - pass + +class MemberDot(BinaryOp): def eval(self, ctx): - #XXX this is wrong, should clone the function prototype - w_obj = W_Object(ctx=ctx, callfunc = self) - w_obj.Put('prototype', W_Object(ctx=ctx)) - return w_obj + w_obj = self.left.eval(ctx).GetValue().ToObject() + name = self.right.get_literal() + return W_Reference(name, w_obj) -class FunctionExpression(Expression): - def __init__(self, pos, name, body, params): +class FunctionStatement(Statement): + def __init__(self, pos, name, params, body): + self.pos = pos self.name = name self.body = body self.params = params - + def eval(self, ctx): #XXX this is wrong, should clone the function prototype w_obj = W_Object(ctx=ctx, callfunc = self) w_obj.Put('prototype', W_Object(ctx=ctx)) return w_obj + def execute(self, ctx): + return self.eval(ctx) + class Identifier(Expression): def __init__(self, pos, name): @@ -287,9 +285,13 @@ def eval(self, ctx): return ctx.resolve_identifier(self.name) + def get_literal(self): + return self.name + class This(Identifier): - opcode = "THIS" + pass + class If(Statement): def __init__(self, pos, condition, thenpart, elsepart=astundef): @@ -778,24 +780,29 @@ self.nodes = nodes def execute(self, ctx): - for var in self.var_decl: - ctx.variable.Put(var.name, w_Undefined) - for fun in self.func_decl: - ctx.variable.Put(fun.name, fun.eval(ctx)) + print self.var_decl + print ctx + print repr(ctx.variable) + for varname in self.var_decl: + ctx.variable.Put(varname, w_Undefined) + for funcname, funccode in self.func_decl.items(): + ctx.variable.Put(funcname, funccode.eval(ctx)) node = self - + try: last = w_Undefined for node in self.nodes: last = node.execute(ctx) + print repr(ctx.variable) return last except Exception, e: if isinstance(e, ExecutionReturned) and e.type == 'return': raise else: # TODO: proper exception handling - print "exception in line: %s, on: %s"%(node.pos.lineno, node) + sys.stderr.write("exception in line: %s, on: %s%s"%(node.pos.lineno, node, os.linesep)) raise + class Program(Statement): def __init__(self, pos, body): @@ -807,16 +814,16 @@ class Return(Statement): - opcode = 'RETURN' - - def __init__(self, pos, t): - self.expr = get_obj(t, 'value') - + def __init__(self, pos, expr): + self.pos = pos + self.expr = expr + def execute(self, ctx): if isinstance(self.expr, Undefined): raise ExecutionReturned('return', None, None) else: raise ExecutionReturned('return', self.expr.eval(ctx), None) + class Throw(Statement): def __init__(self, pos, exp): @@ -827,20 +834,13 @@ raise ThrowException(self.exp.eval(ctx).GetValue()) class Try(Statement): - opcode = 'TRY' - - def __init__(self, pos, t): - self.tryblock = get_obj(t, 'tryBlock') - self.finallyblock = get_obj(t, 'finallyBlock') - catch = get_tree_item(t, 'catchClauses') - if catch is not None: - #multiple catch clauses is a spidermonkey extension - self.catchblock = get_obj(catch, 'block') - self.catchparam = get_string(catch, 'varName') - else: - self.catchblock = None - self.catchparam = None - + def __init__(self, pos, tryblock, catchparam, catchblock, finallyblock): + self.pos = pos + self.tryblock = tryblock + self.catchparam = catchparam + self.catchblock = catchblock + self.finallyblock = finallyblock + def execute(self, ctx): e = None tryresult = w_Undefined @@ -850,7 +850,7 @@ e = excpt if self.catchblock is not None: obj = W_Object() - obj.Put(self.catchparam, e.exception) + obj.Put(self.catchparam.name, e.exception) ctx.push_object(obj) tryresult = self.catchblock.execute(ctx) ctx.pop_object() @@ -863,6 +863,7 @@ raise e return tryresult + class Typeof(UnaryOp): opcode = 'TYPEOF' @@ -880,10 +881,11 @@ self.expr = expr def eval(self, ctx): - # XXX take a look at the specs - v = self.identifier.eval(ctx) - if self.expr is not None: - v.PutValue(self.expr.eval(ctx).GetValue(), ctx) + name = self.identifier.get_literal() + if self.expr is None: + ctx.variable.Put(name, w_Undefined) + else: + ctx.variable.Put(name, self.expr.eval(ctx).GetValue()) class VariableDeclList(Expression): Modified: pypy/dist/pypy/lang/js/test/ecma/conftest.py ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/conftest.py (original) +++ pypy/dist/pypy/lang/js/test/ecma/conftest.py Thu May 24 20:45:07 2007 @@ -5,8 +5,6 @@ from py.__.test.outcome import Failed, ExceptionFailure import pypy.lang.js as js -js.jsobj.DEBUG = False - rootdir = py.magic.autopath().dirpath() exclusionlist = ['shell.js', 'browser.js'] Modified: pypy/dist/pypy/lang/js/test/ecma/shell.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/shell.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/shell.js Thu May 24 20:45:07 2007 @@ -47,7 +47,7 @@ * constant strings */ var GLOBAL = "[object global]"; -var PASSED = " PASSED!" +var PASSED = " PASSED!"; var FAILED = " FAILED! expected: "; var DEBUG = false; @@ -141,7 +141,7 @@ // } // } // stopTest(); - return testcases.length + return testcases.length; } /* @@ -196,7 +196,7 @@ function run_test(tc) { // try { - getTestCaseResult(testcases[tc].expect, testcases[tc].actual) + getTestCaseResult(testcases[tc].expect, testcases[tc].actual); testcases[tc].reason += ( testcases[tc].passed ) ? "passed" : "wrong value expected: " +testcases[tc].expect+" but got: "+ testcases[tc].actual; return testcases[tc].reason; Modified: pypy/dist/pypy/lang/js/test/test_interp.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_interp.py (original) +++ pypy/dist/pypy/lang/js/test/test_interp.py Thu May 24 20:45:07 2007 @@ -13,25 +13,10 @@ n1 = Number(Position(), 2.0) n2 = Number(Position(), 4.0) p = Plus(Position(), n1, n2) - assert p.eval(ExecutionContext()).GetValue().ToNumber() == 6.0 + assert p.eval(ExecutionContext([W_Object(),])).GetValue().ToNumber() == 6.0 l = [] interpreter.writer = l.append -def assert_prints(code, assval): - l = [] - interpreter.writer = l.append - js_int = interpreter.Interpreter() - try: - if isinstance(code, str): - js_int.run(interpreter.load_source(code)) - else: - for codepiece in code: - js_int.run(interpreter.load_source(codepiece)) - except ThrowException, excpt: - l.append("uncaught exception: "+str(excpt.exception.ToString())) - print l, assval - assert l == assval - def assertp(code, prints): l = [] interpreter.writer = l.append @@ -51,7 +36,7 @@ try: code_val = jsint.run(interpreter.load_source(code)).GetValue() except ThrowException, excpt: - code_val = excpt + code_val = excpt.exception print code_val, value if isinstance(value, W_Root): assert AEC(jsint.global_context, code_val, value) == True @@ -80,42 +65,39 @@ assertv('\"sss\";', 'sss') def test_string_concat(): - assert_prints('x="xxx"; y="yyy"; print(x+y);', ["xxxyyy"]) + assertp('x="xxx"; y="yyy"; print(x+y);', "xxxyyy") def test_string_num_concat(): - assert_prints('x=4; y="x"; print(x+y, y+x);', ["4x,x4"]) + assertp('x=4; y="x"; print(x+y, y+x);', ["4x,x4"]) def test_to_string(): - assert_prints("x={}; print(x);", ["[object Object]"]) + assertp("x={}; print(x);", ["[object Object]"]) def test_object_access(): - assert_prints("x={d:3}; print(x.d);", ["3"]) - assert_prints("x={d:3}; print(x.d.d);", ["undefined"]) - assert_prints("x={d:3, z:4}; print(x.d+x.z);", ["7"]) + yield assertp, "x={d:3}; print(x.d);", "3" + yield assertp, "x={d:3}; print(x.d.d);", "undefined" + yield assertp, "x={d:3, z:4}; print(x.d+x.z);", "7" def test_object_access_index(): - assert_prints('x={d:"x"}; print(x["d"]);', ["x"]) + assertp('x={d:"x"}; print(x["d"]);', 'x') def test_function_prints(): - py.test.skip("not ready yet") - assert_prints('x=function(){print(3);}; x();', ["3"]) + assertp('x=function(){print(3);}; x();', '3') def test_function_returns(): - py.test.skip("not ready yet") - assert_prints('x=function(){return 1;}; print(x()+x());', ["2"]) - assert_prints('function x() { return; };', []) + yield assertp, 'x=function(){return 1;}; print(x()+x());', '2' + yield assertp, 'function x() { return; };', [] def test_var_declaration(): yield assertv, 'var x = 3; x;', 3 yield assertv, 'var x = 3; x+x;', 6 def test_var_scoping(): - py.test.skip("not ready yet") - assert_prints(""" + assertp(""" var y; var p; p = 0; - x = function() { + function x () { var p; p = 1; y = 3; return y + z; @@ -125,94 +107,84 @@ """, ["5,3,0"]) def test_function_args(): - py.test.skip("not ready yet") - assert_prints(""" + assertv(""" x = function (t,r) { return t+r; }; - print(x(2,3)); - """, ["5"]) + x(2,3); + """, 5) def test_function_less_args(): - py.test.skip("not ready yet") - assert_prints(""" + assertp(""" x = function (t, r) { return t + r; }; print(x(2)); - """, ["NaN"]) + """, "NaN") def test_function_more_args(): - py.test.skip("not ready yet") - assert_prints(""" + assertv(""" x = function (t, r) { return t + r; }; - print(x(2,3,4)); - """, ["5"]) + x(2,3,4); + """, 5) def test_function_has_var(): - py.test.skip("not ready yet") - assert_prints(""" + assertv(""" x = function () { var t = 'test'; return t; }; - print(x()); - """, ["test"]) + x(); + """, 'test') def test_function_arguments(): - py.test.skip("not ready yet") - assert_prints(""" + assertv(""" x = function () { r = arguments[0]; t = arguments[1]; return t + r; }; - print(x(2,3)); - """, ["5"]) + x(2,3); + """, 5) def test_index(): - yield assertv, """ + assertv(""" x = {1:"test"}; x[1]; - """, 'test' + """, 'test') def test_array_initializer(): - assert_prints(""" + assertp(""" x = []; print(x); - """, [""]) + """, '') def test_throw(): - assert_prints("throw(3);", ["uncaught exception: 3"]) + assertp("throw(3);", "uncaught exception: 3") def test_group(): - assert_prints("print((2+1));", ["3"]) + assertv("(2+1);", 3) def test_comma(): - py.test.skip("not ready yet") - assert_prints("print((500,3));", ["3"]) + assertv("(500,3);", 3) -def test_try_catch(): - py.test.skip("not ready yet") - assert_prints(""" +def test_block(): + yield assertv, "{5;}", 5 + yield assertv, "{3; 5;}", 5 + +def test_try_catch_finally(): + yield assertp, """ try { throw(3); } catch (x) { print(x); } - """, ["3"]) - -def test_block(): - assertv("{5;}", W_Number(5)) - assertv("{3; 5;}", W_Number(5)) - -def test_try_catch_finally(): - py.test.skip("not ready yet") - assert_prints(""" + """, "3" + yield assertp, """ try { throw(3); } @@ -222,7 +194,7 @@ finally { print(5); } - """, ["3", "5"]) + """, ["3", "5"] def test_if_then(): assertp(""" @@ -280,59 +252,43 @@ """, "[object Object]" def test_var_decl(): - py.test.skip("not ready yet") - assert_prints("print(x); var x;", ["undefined"]) - assert_prints(""" + yield assertp, "print(x); var x;", "undefined" + yield assertp, """ try { print(z); } catch (e) { print(e); } - """, ["ReferenceError: z is not defined"]) + """, "ReferenceError: z is not defined" def test_function_name(): - py.test.skip("not ready yet") - assert_prints(""" + assertp(""" function x() { print("my name is x"); } x(); - """, ["my name is x"]) + """, "my name is x") def test_new_with_function(): - py.test.skip("not ready yet") c= """ x = function() {this.info = 'hello';}; o = new x(); - print(o.info); + o.info; """ - print c - assert_prints(c, ["hello"]) + assertv(c, "hello") def test_vars(): - assert_prints(""" + assertp(""" var x;x=3; print(x);""", ["3"]) def test_in(): - py.test.skip("not ready yet") - assert_prints(""" + assertp(""" x = {y:3}; print("y" in x); print("z" in x); """, ["true", "false"]) -def test_append_code(): - assert_prints([""" - var x; x=3; - """, """ - print(x); - z = 2; - """,""" - print(z); - """] - ,["3", "2"]) - def test_for(): assertp(""" i = 0; @@ -386,11 +342,8 @@ x;""", 2) def test_ternaryop(): - py.test.skip("not ready yet") - assert_prints([ - "( 1 == 1 ) ? print('yep') : print('nope');", - "( 1 == 0 ) ? print('yep') : print('nope');"], - ["yep","nope"]) + yield assertv, "( 1 == 1 ) ? true : false;", True + yield assertv, "( 1 == 0 ) ? true : false;", False def test_booleanliterals(): assertp(""" @@ -436,9 +389,8 @@ """, ['NaN', 'Infinity', 'undefined']) def test_strangefunc(): - py.test.skip("not ready yet") - assert_prints("""function f1() { var z; var t;}""", []) - assert_prints(""" "'t'"; """, []) + assertp("""function f1() { var z; var t;}""", []) + assertp(""" "'t'"; """, []) def test_null(): assertv("null;", w_Null) @@ -448,8 +400,7 @@ ["hello", "undefined"]) def test_activationprob(): - py.test.skip("not ready yet") - assert_prints( """ + assertp( """ function intern (int1){ print(int1); return int1; @@ -464,16 +415,17 @@ """, ['1','1', '1']) def test_array_acess(): - py.test.skip("not ready yet") - assert_prints(""" + assertp(""" var x = new Array(); x[0] = 1; + print(x[0]); x[x[0]] = 2; + print(x[1]); x[2] = x[0]+x[1]; for(i=0; i<3; i++){ print(x[i]); } - """, ['1', '2', '3']) + """, ['1','2', '1', '2', '3']) def test_array_length(): assertp(""" @@ -498,7 +450,7 @@ def test_forin(): py.test.skip("not ready yet") - assert_prints(""" + assertp(""" var x = {a:5}; for(y in x){ print(y); @@ -513,7 +465,7 @@ def test_with(): py.test.skip("not ready yet") - assert_prints(""" + assertp(""" var mock = {x:2}; var x=4; print(x); @@ -535,31 +487,36 @@ yield assertv, "2 & 3;", 2 yield assertv, "2 | 3;", 3 -def test_for_strange(): +def test_for_vararg(): py.test.skip("not ready yet") - assert_prints(""" + assertp(""" for (var arg = "", i = 0; i < 2; i++) { print(i);} """, ['0', '1']) def test_recursive_call(): - py.test.skip("not ready yet") - assert_prints(""" + assertv(""" function fact(x) { if (x == 0) { return 1; } else { return fact(x-1)*x; }} - print(fact(3)); - """, ['6',]) + fact(3); + """, 6) def test_function_prototype(): - py.test.skip("not ready yet") - assert_prints(""" + assertp(""" function foo() {}; foo.prototype.bar = function() {}; """, []) def test_function_this(): - py.test.skip("not ready yet") - assert_prints(""" + assertp(""" function foo() {print("debug");this.bar = function() {};}; var f = new foo(); f.bar(); - """, ['debug',]) + """, 'debug') +def test_switch(): + py.test.skip("not ready yet") + assertv(""" + x = 1; + switch(x){ + case 1: 15; break; + default: 30; + };""", 15) From afa at codespeak.net Fri May 25 09:17:01 2007 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 25 May 2007 09:17:01 +0200 (CEST) Subject: [pypy-svn] r43611 - pypy/dist/pypy/interpreter Message-ID: <20070525071701.34B37809D@code0.codespeak.net> Author: afa Date: Fri May 25 09:16:59 2007 New Revision: 43611 Modified: pypy/dist/pypy/interpreter/callmethod.py Log: Correct an import changed since r43585. Translations using --objspace-opcodes-CALL_METHOD (and --allopts) were broken. Modified: pypy/dist/pypy/interpreter/callmethod.py ============================================================================== --- pypy/dist/pypy/interpreter/callmethod.py (original) +++ pypy/dist/pypy/interpreter/callmethod.py Fri May 25 09:16:59 2007 @@ -11,7 +11,7 @@ """ from pypy.interpreter import pyframe, function -from pypy.rlib.objectmodel import we_are_jitted +from pypy.rlib.jit import we_are_jitted from pypy.interpreter.argument import Arguments From afa at codespeak.net Fri May 25 13:50:19 2007 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 25 May 2007 13:50:19 +0200 (CEST) Subject: [pypy-svn] r43617 - pypy/dist/pypy/translator/jvm/test Message-ID: <20070525115019.D75AE8082@code0.codespeak.net> Author: afa Date: Fri May 25 13:50:19 2007 New Revision: 43617 Modified: pypy/dist/pypy/translator/jvm/test/test_class.py Log: untabify Modified: pypy/dist/pypy/translator/jvm/test/test_class.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_class.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_class.py Fri May 25 13:50:19 2007 @@ -22,7 +22,7 @@ a = A() b = B() return call(a, x) + call(b, x) - assert self.interpret(fn, [0]) == 3 + assert self.interpret(fn, [0]) == 3 def test_abstract_method2(self): class Root(object): From fijal at codespeak.net Fri May 25 14:39:24 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 25 May 2007 14:39:24 +0200 (CEST) Subject: [pypy-svn] r43625 - pypy/branch/kill-ctypes/pypy/translator/c Message-ID: <20070525123924.348BA8082@code0.codespeak.net> Author: fijal Date: Fri May 25 14:39:22 2007 New Revision: 43625 Modified: pypy/branch/kill-ctypes/pypy/translator/c/funcgen.py Log: * add OP_FLAVORED_MALLOC_VARSIZE. I need to find tests for it, otherwise I know it's working just for a very basic case, very much XXX by now * remove unused variable Modified: pypy/branch/kill-ctypes/pypy/translator/c/funcgen.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/translator/c/funcgen.py (original) +++ pypy/branch/kill-ctypes/pypy/translator/c/funcgen.py Fri May 25 14:39:22 2007 @@ -552,7 +552,6 @@ elength, itemtype, eresult) - tail = '\n}' esize = 'sizeof(%s)-sizeof(%s)+%s*sizeof(%s)' % ( cdecl(typename, ''), itemtype, @@ -566,8 +565,6 @@ result += '\n}' return result - OP_ZERO_MALLOC_VARSIZE = OP_MALLOC_VARSIZE - def OP_RAW_MALLOC(self, op): eresult = self.expr(op.result) esize = self.expr(op.args[0]) @@ -590,7 +587,61 @@ else: raise NotImplementedError - OP_FLAVORED_MALLOC_VARSIZE = OP_FLAVORED_MALLOC + def OP_FLAVORED_MALLOC_VARSIZE(self, op): + # XXX I know this working in just one case, probably makes + # sense to assert it here, rest is just copied + flavor = op.args[0].value + assert flavor == 'raw' + TYPE = self.lltypemap(op.result).TO + assert isinstance(TYPE, Array) + assert TYPE._hints.get('nolength', False) + # + typename = self.db.gettype(TYPE) + lenfld = 'length' + nodedef = self.db.gettypedefnode(TYPE) + if isinstance(TYPE, Struct): + arfld = TYPE._arrayfld + lenfld = "%s.length" % nodedef.c_struct_field_name(arfld) + VARPART = TYPE._flds[TYPE._arrayfld] + else: + VARPART = TYPE + assert isinstance(VARPART, Array) + itemtypename = self.db.gettype(VARPART.OF) + elength = self.expr(op.args[2]) + eresult = self.expr(op.result) + erestype = cdecl(typename, '*') + if VARPART.OF is Void: # strange + esize = 'sizeof(%s)' % (cdecl(typename, ''),) + result = '{\n' + else: + itemtype = cdecl(itemtypename, '') + result = 'IF_VARSIZE_OVERFLOW(%s, %s, %s)\nelse {\n' % ( + elength, + itemtype, + eresult) + esize = 'sizeof(%s)-sizeof(%s)+%s*sizeof(%s)' % ( + cdecl(typename, ''), + itemtype, + elength, + itemtype) + + # ctypes Arrays have no length field + if not VARPART._hints.get('nolength', False): + result += '\nif(%s) %s->%s = %s;' % (eresult, eresult, lenfld, elength) + if flavor == "raw": + result += "OP_RAW_MALLOC(%s, %s, %s);" % (esize, eresult, erestype) + elif flavor == "stack": + result += "OP_STACK_MALLOC(%s, %s, %s);" % (esize, eresult, erestype) + elif flavor == "cpy": + xxx # this will never work, as I don't know which arg it would be + # tests, tests, tests.... + cpytype = self.expr(op.args[2]) + result += "OP_CPY_MALLOC(%s, %s, %s);" % (cpytype, eresult, erestype) + else: + raise NotImplementedError + + result += '\n}' + return result def OP_FLAVORED_FREE(self, op): flavor = op.args[0].value From fijal at codespeak.net Fri May 25 14:40:28 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 25 May 2007 14:40:28 +0200 (CEST) Subject: [pypy-svn] r43626 - in pypy/branch/kill-ctypes/pypy/rpython/lltypesystem: . test Message-ID: <20070525124028.F39528082@code0.codespeak.net> Author: fijal Date: Fri May 25 14:40:27 2007 New Revision: 43626 Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/lloperation.py pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rffi.py Log: Next test (involving char**, null terminated) passes. This is just enough to write down execv Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/lloperation.py Fri May 25 14:40:27 2007 @@ -314,6 +314,7 @@ 'zero_malloc_varsize': LLOp(canraise=(MemoryError,), canunwindgc=True), 'zero_gc_pointers_inside': LLOp(), 'flavored_malloc': LLOp(canraise=(MemoryError,)), + 'flavored_malloc_varsize' : LLOp(canraise=(MemoryError,)), 'flavored_free': LLOp(), 'getfield': LLOp(sideeffects=False, canrun=True), 'getarrayitem': LLOp(sideeffects=False, canrun=True), Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py Fri May 25 14:40:27 2007 @@ -1,5 +1,6 @@ from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem.lloperation import llop def llexternal(name, args, result, sources=[], includes=[]): ext_type = lltype.FuncType(args, result) @@ -15,5 +16,24 @@ array = lltype.malloc(CCHARP, len(s) + 1, flavor='raw') for i in range(len(s)): array[i] = s[i] - array[len(s)] = chr(0) + array[len(s)] = '\x00' return array + +CCHARPP = lltype.Array(lltype.Ptr(CCHARP), hints={'nolength': True}) +# list[str] -> char**, NULL terminated +def liststr2charpp(l): + array = lltype.malloc(CCHARPP, len(l) + 1, flavor='raw') + for i in range(len(l)): + array[i] = str2charp(l[i]) + array[len(l)] = lltype.nullptr(CCHARP) + return array + +# frees list of char** +def free_charpp(ref): + next = ref + i = 0 + while next[i]: + lltype.free(next[i], flavor='raw') + i += 1 + lltype.free(ref, flavor='raw') + Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rffi.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rffi.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rffi.py Fri May 25 14:40:27 2007 @@ -1,6 +1,6 @@ #import py -from pypy.rpython.lltypesystem.rffi import llexternal, str2charp, CCHARP +from pypy.rpython.lltypesystem.rffi import * from pypy.translator.c.test.test_genc import compile from pypy.rpython.lltypesystem.lltype import Signed, Ptr, Char, malloc from pypy.rpython.lltypesystem import lltype @@ -40,7 +40,7 @@ assert xf() == 8+3 def test_string(): - z = llexternal('strlen', [Ptr(CCHARP)], Signed, includes=['stdio.h']) + z = llexternal('strlen', [Ptr(CCHARP)], Signed, includes=['string.h']) def f(): s = str2charp("xxx") @@ -50,3 +50,31 @@ xf = compile(f, [], backendopt=False) assert xf() == 3 + +def test_stringstar(): + c_source = """ + #include + + int f(char *args[]) { + char **p = args; + int l = 0; + while (*p) { + l += strlen(*p); + p++; + } + return (l); + } + """ + c_file = udir.join("stringstar.c") + c_file.write(c_source) + z = llexternal('f', [Ptr(CCHARPP)], Signed, sources=[str(c_file)]) + + def f(): + l = ["xxx", "x", "xxxx"] + ss = liststr2charpp(l) + result = z(ss) + free_charpp(ss) + return result + + xf = compile(f, [], backendopt=False) + assert xf() == 8 From fijal at codespeak.net Fri May 25 14:52:23 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 25 May 2007 14:52:23 +0200 (CEST) Subject: [pypy-svn] r43628 - pypy/branch/kill-ctypes/pypy/rpython/lltypesystem Message-ID: <20070525125223.49C078082@code0.codespeak.net> Author: fijal Date: Fri May 25 14:52:12 2007 New Revision: 43628 Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py Log: Move comments to docstrings Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py Fri May 25 14:52:12 2007 @@ -8,28 +8,34 @@ sources=tuple(sources), includes=tuple(includes)) +# char * CCHARP = lltype.Array(lltype.Char, hints={'nolength': True}) # various type mapping -# str -> char* def str2charp(s): + """ str -> char* + """ array = lltype.malloc(CCHARP, len(s) + 1, flavor='raw') for i in range(len(s)): array[i] = s[i] array[len(s)] = '\x00' return array +# char** CCHARPP = lltype.Array(lltype.Ptr(CCHARP), hints={'nolength': True}) -# list[str] -> char**, NULL terminated + def liststr2charpp(l): + """ list[str] -> char**, NULL terminated + """ array = lltype.malloc(CCHARPP, len(l) + 1, flavor='raw') for i in range(len(l)): array[i] = str2charp(l[i]) array[len(l)] = lltype.nullptr(CCHARP) return array -# frees list of char** def free_charpp(ref): + """ frees list of char**, NULL terminated + """ next = ref i = 0 while next[i]: From cfbolz at codespeak.net Fri May 25 15:33:04 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 25 May 2007 15:33:04 +0200 (CEST) Subject: [pypy-svn] r43630 - pypy/dist/pypy/doc/statistic Message-ID: <20070525133304.C077E8082@code0.codespeak.net> Author: cfbolz Date: Fri May 25 15:33:04 2007 New Revision: 43630 Modified: pypy/dist/pypy/doc/statistic/release_dates.dat Log: remove double entry Modified: pypy/dist/pypy/doc/statistic/release_dates.dat ============================================================================== --- pypy/dist/pypy/doc/statistic/release_dates.dat (original) +++ pypy/dist/pypy/doc/statistic/release_dates.dat Fri May 25 15:33:04 2007 @@ -5,6 +5,5 @@ 2005-08-28,"PyPy 0.7" 2005-11-03,"PyPy 0.8" 2006-06-25,"PyPy 0.9" -2006-06-25,"PyPy 0.9" 2007-02-17,"PyPy 0.99" 2007-03-27,"PyPy 1.0" From afa at codespeak.net Fri May 25 15:42:40 2007 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 25 May 2007 15:42:40 +0200 (CEST) Subject: [pypy-svn] r43632 - pypy/dist/pypy/translator/jvm Message-ID: <20070525134240.E28398082@code0.codespeak.net> Author: afa Date: Fri May 25 15:42:40 2007 New Revision: 43632 Modified: pypy/dist/pypy/translator/jvm/genjvm.py Log: Adapt genjvm for win32: - split long arguments lists to build smaller command lines - shell=True is needed to find the jasmin.bat script Modified: pypy/dist/pypy/translator/jvm/genjvm.py ============================================================================== --- pypy/dist/pypy/translator/jvm/genjvm.py (original) +++ pypy/dist/pypy/translator/jvm/genjvm.py Fri May 25 15:42:40 2007 @@ -92,7 +92,8 @@ def _invoke(self, args, allow_stderr): subp = subprocess.Popen( - args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + shell=True, universal_newlines=True) stdout, stderr = subp.communicate() res = subp.wait() if res or (not allow_stderr and stderr): @@ -108,7 +109,7 @@ if not os.path.exists(str(pypycls)): tocompile.append(clsnm) if tocompile: - sl = __file__.rindex('/') + sl = __file__.rindex(os.path.sep) javasrcs = [__file__[:sl]+("/src/pypy/%s.java" % clsnm) for clsnm in tocompile] self._invoke([getoption('javac'), @@ -124,9 +125,32 @@ """ jascmd = [getoption('jasmin'), '-d', str(self.javadir)] - print "Invoking jasmin on %s" % self.jasmin_files - self._invoke(jascmd+list(self.jasmin_files), False) - print "... completed!" + def split_list(files): + "Split the files list into manageable pieces" + + # - On Windows 2000, commands in .bat are limited to 2047 chars. + # - But the 'jasmin' script contains a line like + # path_to_jre/java -jar path_to_jasmin/jasmin.jar $* + # So we limit the length of arguments files to: + MAXLINE = 1500 + + chunk = [] + chunklen = 0 + for f in files: + # Account for the space between items + chunklen += len(f) + 1 + if chunklen > MAXLINE: + yield chunk + chunk = [] + chunklen = len(f) + chunk.append(f) + if chunk: + yield chunk + + for files in split_list(self.jasmin_files): + print "Invoking jasmin on %s" % files + self._invoke(jascmd + files, False) + print "... completed!" self.compiled = True self._compile_helper(('Callback', From afa at codespeak.net Fri May 25 16:22:33 2007 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 25 May 2007 16:22:33 +0200 (CEST) Subject: [pypy-svn] r43638 - pypy/dist/pypy/translator/jvm Message-ID: <20070525142233.EB1628082@code0.codespeak.net> Author: afa Date: Fri May 25 16:22:33 2007 New Revision: 43638 Modified: pypy/dist/pypy/translator/jvm/generator.py pypy/dist/pypy/translator/jvm/genjvm.py Log: Use py.path to manipulate file names: code is simpler and more portable Modified: pypy/dist/pypy/translator/jvm/generator.py ============================================================================== --- pypy/dist/pypy/translator/jvm/generator.py (original) +++ pypy/dist/pypy/translator/jvm/generator.py Fri May 25 16:22:33 2007 @@ -1,4 +1,3 @@ -import os # from pypy.objspace.flow import model as flowmodel from pypy.translator.oosupport.metavm import Generator from pypy.rpython.ootypesystem import ootype @@ -1246,14 +1245,11 @@ iclassnm = self.current_type().descriptor.int_class_name() isuper = self.curclass.superclass_type.descriptor.int_class_name() - jfile = "%s/%s.j" % (self.outdir, iclassnm) + jfile = self.outdir.join("%s.j" % iclassnm) - try: - jdir = jfile[:jfile.rindex('/')] - os.makedirs(jdir) - except OSError: pass - self.curclass.file = open(jfile, 'w') - self.db.add_jasmin_file(jfile) + jfile.dirpath().ensure(dir=True) + self.curclass.file = jfile.open('w') + self.db.add_jasmin_file(str(jfile)) # Determine the "declaration string" if interface: decl_str = "interface" Modified: pypy/dist/pypy/translator/jvm/genjvm.py ============================================================================== --- pypy/dist/pypy/translator/jvm/genjvm.py (original) +++ pypy/dist/pypy/translator/jvm/genjvm.py Fri May 25 16:22:33 2007 @@ -2,7 +2,7 @@ Backend for the JVM. """ -import os, os.path, sys +import sys import py from py.compat import subprocess @@ -106,11 +106,11 @@ tocompile = [] for clsnm in clsnms: pypycls = self.classdir.join(clsnm + '.class') - if not os.path.exists(str(pypycls)): + if not pypycls.check(): tocompile.append(clsnm) if tocompile: - sl = __file__.rindex(os.path.sep) - javasrcs = [__file__[:sl]+("/src/pypy/%s.java" % clsnm) for + thisdir = py.magic.autopath().dirpath() + javasrcs = [str(thisdir.join('src/pypy', clsnm + '.java')) for clsnm in tocompile] self._invoke([getoption('javac'), '-nowarn', From cfbolz at codespeak.net Fri May 25 18:08:01 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 25 May 2007 18:08:01 +0200 (CEST) Subject: [pypy-svn] r43639 - pypy/dist/pypy/lang/prolog/interpreter/test Message-ID: <20070525160801.CBBEC8092@code0.codespeak.net> Author: cfbolz Date: Fri May 25 18:08:00 2007 New Revision: 43639 Modified: pypy/dist/pypy/lang/prolog/interpreter/test/test_arithmetic.py Log: more tests Modified: pypy/dist/pypy/lang/prolog/interpreter/test/test_arithmetic.py ============================================================================== --- pypy/dist/pypy/lang/prolog/interpreter/test/test_arithmetic.py (original) +++ pypy/dist/pypy/lang/prolog/interpreter/test/test_arithmetic.py Fri May 25 18:08:00 2007 @@ -81,6 +81,17 @@ assert_true("X is float_integer_part(-0.6), X = 0.") assert_true("X is float_integer_part(-1), X = -1.") + assert_true("X is float_fractional_part(1), X = 0.") + assert_true("X is float_fractional_part(2), X = 0.") + assert_true("X is float_fractional_part(-1), X = 0.") + assert_true("X is float_fractional_part(1.2), Y is 1.2 - 1, X = Y.") + assert_true("X is float_fractional_part(1.4), Y is 1.4 - 1, X = Y.") + assert_true("X is float_fractional_part(1.6), Y is 1.6 - 1, X = Y.") + assert_true("X is float_fractional_part(-1.2), X is -1.2 + 1, X = Y.") + assert_true("X is float_fractional_part(-1.4), X is -1.4 + 1, X = Y.") + assert_true("X is float_fractional_part(-1.6), X is -1.6 + 1, X = Y.") + + assert_true("X is 2 ** 4, X = 16.") def test_comparison(): assert_true("1 =:= 1.0.") From cfbolz at codespeak.net Fri May 25 18:11:45 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 25 May 2007 18:11:45 +0200 (CEST) Subject: [pypy-svn] r43640 - in pypy/dist/pypy/lang/prolog: builtin interpreter interpreter/test Message-ID: <20070525161145.6288A8092@code0.codespeak.net> Author: cfbolz Date: Fri May 25 18:11:45 2007 New Revision: 43640 Modified: pypy/dist/pypy/lang/prolog/builtin/arithmeticbuiltin.py pypy/dist/pypy/lang/prolog/builtin/formatting.py pypy/dist/pypy/lang/prolog/builtin/register.py pypy/dist/pypy/lang/prolog/interpreter/arithmetic.py pypy/dist/pypy/lang/prolog/interpreter/engine.py pypy/dist/pypy/lang/prolog/interpreter/helper.py pypy/dist/pypy/lang/prolog/interpreter/portal.py pypy/dist/pypy/lang/prolog/interpreter/term.py pypy/dist/pypy/lang/prolog/interpreter/test/test_jit.py Log: attempt to jit some of the integer arithmetic functions. requires some rewriting. Modified: pypy/dist/pypy/lang/prolog/builtin/arithmeticbuiltin.py ============================================================================== --- pypy/dist/pypy/lang/prolog/builtin/arithmeticbuiltin.py (original) +++ pypy/dist/pypy/lang/prolog/builtin/arithmeticbuiltin.py Fri May 25 18:11:45 2007 @@ -27,6 +27,7 @@ def impl_is(engine, var, num): var.unify(num, engine.heap) +impl_is._look_inside_me_ = True expose_builtin(impl_is, "is", unwrap_spec=["raw", "arithmetic"]) for ext, prolog, python in [("eq", "=:=", "=="), @@ -43,7 +44,7 @@ eq = num1.num %s num2.num elif isinstance(num1, term.Float): if isinstance(num2, term.Float): - eq = num1.num %s num2.num + eq = num1.floatval %s num2.floatval if not eq: raise error.UnificationFailed()""" % (ext, python, python)).compile() expose_builtin(globals()["impl_arith_%s" % (ext, )], prolog, Modified: pypy/dist/pypy/lang/prolog/builtin/formatting.py ============================================================================== --- pypy/dist/pypy/lang/prolog/builtin/formatting.py (original) +++ pypy/dist/pypy/lang/prolog/builtin/formatting.py Fri May 25 18:11:45 2007 @@ -73,7 +73,7 @@ return str(num.num) def format_float(self, num): - return str(num.num) + return str(num.floatval) def format_var(self, var): return "_G%s" % (var.index, ) Modified: pypy/dist/pypy/lang/prolog/builtin/register.py ============================================================================== --- pypy/dist/pypy/lang/prolog/builtin/register.py (original) +++ pypy/dist/pypy/lang/prolog/builtin/register.py Fri May 25 18:11:45 2007 @@ -87,7 +87,7 @@ signature = "%s/%s" % (name, len(unwrap_spec)) b = Builtin(miniglobals[funcname]) builtins[signature] = b - if signature == ",/2": + if signature in [",/2", "is/2"]: builtins_list.insert(0, (signature, b)) else: builtins_list.append((signature, b)) Modified: pypy/dist/pypy/lang/prolog/interpreter/arithmetic.py ============================================================================== --- pypy/dist/pypy/lang/prolog/interpreter/arithmetic.py (original) +++ pypy/dist/pypy/lang/prolog/interpreter/arithmetic.py Fri May 25 18:11:45 2007 @@ -4,10 +4,12 @@ from pypy.lang.prolog.interpreter import engine, helper, term, error from pypy.lang.prolog.interpreter.error import UnificationFailed, FunctionNotFound from pypy.rlib.rarithmetic import intmask +from pypy.rlib.jit import we_are_jitted, hint +from pypy.rlib.unroll import unrolling_iterable arithmetic_functions = {} +arithmetic_functions_list = [] -pattern_to_function = {} class CodeCollector(object): def __init__(self): @@ -85,29 +87,15 @@ code.end_block("def") miniglobals = globals().copy() exec py.code.Source(code.tostring()).compile() in miniglobals - return miniglobals["prolog_" + name] + result = miniglobals["prolog_" + name] + result._look_inside_me_ = True + return result wrap_builtin_operation._annspecialcase_ = 'specialize:memo' def eval_arithmetic(engine, query): - query = query.getvalue(engine.heap) - if isinstance(query, term.Number): - return query - if isinstance(query, term.Float): - return norm_float(query) - if isinstance(query, term.Atom): - #XXX beautify that - if query.name == "pi": - return term.Float(math.pi) - if query.name == "e": - return term.Float(math.e) - error.throw_type_error("evaluable", query.get_prolog_signature()) - if isinstance(query, term.Term): - func = arithmetic_functions.get(query.signature, None) - if func is None: - error.throw_type_error("evaluable", query.get_prolog_signature()) - return func(engine, query) - raise error.UncatchableError("not implemented") + return query.eval_arithmetic(engine) +eval_arithmetic._look_inside_me_ = True def norm_float(obj): v = obj.floatval @@ -121,7 +109,7 @@ ("-", ["expr", "expr"], "v0 - v1", True, True), ("*", ["expr", "expr"], "v0 * v1", True, True), ("//", ["int", "int"], "v0 / v1", True, False), - ("**", ["expr", "expr"], "v0 ** v1", True, True), + ("**", ["expr", "expr"], "float(v0) ** float(v1)", True, False), (">>", ["int", "int"], "v0 >> v1", False, False), ("<<", ["int", "int"], "intmask(v0 << v1)", False, False), @@ -134,9 +122,9 @@ ("max", ["expr", "expr"], "max(v0, v1)", False, True), ("min", ["expr", "expr"], "min(v0, v1)", False, True), ("round", ["expr"], "int(v0 + 0.5)", False, False), - ("floor", ["expr"], "math.floor(v0)", False, True), #XXX - ("ceiling", ["expr"], "math.ceil(v0)", False, True), #XXX - ("float_fractional_part", ["expr"], "v0 - int(v0)", False, True), #XXX + ("floor", ["expr"], "math.floor(v0)", False, False), #XXX + ("ceiling", ["expr"], "math.ceil(v0)", False, False), #XXX + ("float_fractional_part", ["expr"], "v0 - int(v0)", False, False), #XXX ("float_integer_part", ["expr"], "int(v0)", False, True), ] @@ -151,3 +139,6 @@ intversion) signature = "%s/%s" % (prolog_name, len(unwrap_spec)) arithmetic_functions[signature] = f + arithmetic_functions_list.append((signature, f)) + +arithmetic_functions_list = unrolling_iterable(arithmetic_functions_list) Modified: pypy/dist/pypy/lang/prolog/interpreter/engine.py ============================================================================== --- pypy/dist/pypy/lang/prolog/interpreter/engine.py (original) +++ pypy/dist/pypy/lang/prolog/interpreter/engine.py Fri May 25 18:11:45 2007 @@ -334,7 +334,7 @@ continuation) raise else: - inline = False #XXX rule.body is None # inline facts + inline = rule.body is None # inline facts try: # for the last rule (rulechain is None), this will always # return, because choice_point is False Modified: pypy/dist/pypy/lang/prolog/interpreter/helper.py ============================================================================== --- pypy/dist/pypy/lang/prolog/interpreter/helper.py (original) +++ pypy/dist/pypy/lang/prolog/interpreter/helper.py Fri May 25 18:11:45 2007 @@ -3,8 +3,10 @@ from pypy.lang.prolog.interpreter import term from pypy.lang.prolog.interpreter import error +emptylist = term.Atom.newatom("[]") + def wrap_list(python_list): - curr = term.Atom.newatom("[]") + curr = emptylist for i in range(len(python_list) - 1, -1, -1): curr = term.Term(".", [python_list[i], curr]) return curr @@ -23,6 +25,7 @@ def is_callable(var, engine): return isinstance(var, term.Callable) +is_callable._look_inside_me_ = True def ensure_callable(var): if isinstance(var, term.Var): @@ -31,6 +34,7 @@ return var else: error.throw_type_error("callable", var) +ensure_callable._look_inside_me_ = True def unwrap_int(obj): if isinstance(obj, term.Number): @@ -45,6 +49,7 @@ if isinstance(obj, term.Atom): return obj.name error.throw_type_error('atom', obj) +unwrap_atom._look_inside_me_ = True def unwrap_predicate_indicator(predicate): if not isinstance(predicate, term.Term): Modified: pypy/dist/pypy/lang/prolog/interpreter/portal.py ============================================================================== --- pypy/dist/pypy/lang/prolog/interpreter/portal.py (original) +++ pypy/dist/pypy/lang/prolog/interpreter/portal.py Fri May 25 18:11:45 2007 @@ -1,5 +1,5 @@ from pypy.jit.hintannotator.policy import ManualGraphPolicy -from pypy.lang.prolog.interpreter import term, engine +from pypy.lang.prolog.interpreter import term, engine, helper from pypy.translator.translator import graphof from pypy.annotation.specialize import getuniquenondirectgraph @@ -27,7 +27,7 @@ def fill_timeshift_graphs(self, t, portal_graph): import pypy - for cls in [term.Var, term.Term, term.Number, term.Float, term.Atom]: + for cls in [term.Var, term.Term, term.Number, term.Atom]: self.seegraph(cls.copy) self.seegraph(cls.__init__) self.seegraph(cls.copy_and_unify) @@ -37,6 +37,7 @@ self.seegraph(cls.copy_and_basic_unify) for cls in [term.Var, term.Term, term.Number, term.Atom]: self.seegraph(cls.get_unify_hash) + self.seegraph(cls.eval_arithmetic) for cls in [term.Callable, term.Atom, term.Term]: self.seegraph(cls.get_prolog_signature) self.seegraph(cls.unify_hash_of_children) @@ -58,6 +59,8 @@ for cls in [engine.Continuation, engine.LimitedScopeContinuation, pypy.lang.prolog.builtin.control.AndContinuation]: self.seegraph(cls._call) + for function in "".split(): + self.seegraph(getattr(helper, function)) def get_portal(drv): t = drv.translator Modified: pypy/dist/pypy/lang/prolog/interpreter/term.py ============================================================================== --- pypy/dist/pypy/lang/prolog/interpreter/term.py (original) +++ pypy/dist/pypy/lang/prolog/interpreter/term.py Fri May 25 18:11:45 2007 @@ -2,8 +2,10 @@ from pypy.rlib.objectmodel import we_are_translated, UnboxedValue from pypy.rlib.rarithmetic import intmask from pypy.lang.prolog.interpreter.error import UnificationFailed, UncatchableError +from pypy.lang.prolog.interpreter import error from pypy.rlib.jit import hint from pypy.rlib.objectmodel import specialize +from pypy.rlib.jit import we_are_jitted, hint, purefunction DEBUG = False @@ -71,6 +73,8 @@ # for testing return not (self == other) + def eval_arithmetic(self, engine): + error.throw_type_error("evaluable", self) class Var(PrologObject): TAG = 0 @@ -169,13 +173,20 @@ return (self.__class__ == other.__class__ and self.index == other.index) + @staticmethod + @purefunction def newvar(index): result = Var.cache.get(index, None) if result is not None: return result Var.cache[index] = result = Var(index) return result - newvar = staticmethod(newvar) + + def eval_arithmetic(self, engine): + self = self.dereference(engine.heap) + if isinstance(self, Var): + error.throw_instantiation_error() + return self.eval_arithmetic(engine) class NonVar(PrologObject): @@ -271,13 +282,22 @@ def get_prolog_signature(self): return Term("/", [self, NUMBER_0]) + @staticmethod + @purefunction def newatom(name): result = Atom.cache.get(name, None) if result is not None: return result Atom.cache[name] = result = Atom(name) return result - newatom = staticmethod(newatom) + + def eval_arithmetic(self, engine): + #XXX beautify that + if self.name == "pi": + return Float.pi + if self.name == "e": + return Float.e + error.throw_type_error("evaluable", self.get_prolog_signature()) class Number(NonVar): @@ -312,6 +332,9 @@ def get_unify_hash(self, heap): return intmask(self.num << TAGBITS | self.TAG) + def eval_arithmetic(self, engine): + return self + NUMBER_0 = Number(0) class Float(NonVar): @@ -349,6 +372,14 @@ def __repr__(self): return "Float(%r)" % (self.floatval, ) + def eval_arithmetic(self, engine): + from pypy.lang.prolog.interpreter.arithmetic import norm_float + return norm_float(self) + +Float.e = Float(math.e) +Float.pi = Float(math.pi) + + class BlackBox(NonVar): # meant to be subclassed TAG = tag() @@ -376,6 +407,7 @@ return intmask(id(self) << TAGBITS | self.TAG) + # helper functions for various Term methods def _clone(obj, offset): @@ -490,6 +522,20 @@ return True return False + def eval_arithmetic(self, engine): + from pypy.lang.prolog.interpreter.arithmetic import arithmetic_functions + from pypy.lang.prolog.interpreter.arithmetic import arithmetic_functions_list + if we_are_jitted(): + signature = hint(self.signature, promote=True) + func = None + for sig, func in arithmetic_functions_list: + if sig == signature: + break + else: + func = arithmetic_functions.get(self.signature, None) + if func is None: + error.throw_type_error("evaluable", self.get_prolog_signature()) + return func(engine, self) class Rule(object): _immutable_ = True Modified: pypy/dist/pypy/lang/prolog/interpreter/test/test_jit.py ============================================================================== --- pypy/dist/pypy/lang/prolog/interpreter/test/test_jit.py (original) +++ pypy/dist/pypy/lang/prolog/interpreter/test/test_jit.py Fri May 25 18:11:45 2007 @@ -133,23 +133,26 @@ def test_loop(self): e = get_engine(""" - f(X) :- h(X). - f(a). - h(X) :- h(X). + f(X) :- h(X, _). + f(50). + h(0, _). + h(X, Y) :- Y is X - 1, h(Y, _). """) - X = e.heap.newvar() + num = term.Number(50) def main(n): e.heap.reset() if n == 0: - e.call(term.Term("f", [X])) - return isinstance(X.dereference(e.heap), term.Atom) + e.call(term.Term("f", [num])) + return True else: return False + res = main(0) + assert res res = self.timeshift_from_portal(main, portal.PORTAL, [0], policy=POLICY, backendoptimize=True, inline=0.0) - assert res == True + assert res From cfbolz at codespeak.net Fri May 25 19:09:32 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 25 May 2007 19:09:32 +0200 (CEST) Subject: [pypy-svn] r43641 - pypy/dist/pypy/rlib/test Message-ID: <20070525170932.405548082@code0.codespeak.net> Author: cfbolz Date: Fri May 25 19:09:31 2007 New Revision: 43641 Added: pypy/dist/pypy/rlib/test/test_jit.py (contents, props changed) Modified: pypy/dist/pypy/rlib/test/test_objectmodel.py Log: move jit rlib tests into their own file. fixes them, too, because the imports were wrong. Added: pypy/dist/pypy/rlib/test/test_jit.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/rlib/test/test_jit.py Fri May 25 19:09:31 2007 @@ -0,0 +1,31 @@ +import py +from pypy.rlib.jit import hint, _is_early_constant +from pypy.translator.translator import TranslationContext, graphof +from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin + +class TestJIT(BaseRtypingTest, LLRtypeMixin): + def test_hint(self): + def f(): + x = hint(5, hello="world") + return x + res = self.interpret(f, []) + assert res == 5 + + def test_is_early_constant(self): + def f(x): + if _is_early_constant(x): + return 42 + return 0 + + assert f(3) == 0 + res = self.interpret(f, [5]) + assert res == 0 + + def g(): + return f(88) + + res = self.interpret(g, []) + assert res == 42 + + + Modified: pypy/dist/pypy/rlib/test/test_objectmodel.py ============================================================================== --- pypy/dist/pypy/rlib/test/test_objectmodel.py (original) +++ pypy/dist/pypy/rlib/test/test_objectmodel.py Fri May 25 19:09:31 2007 @@ -269,14 +269,6 @@ res = self.interpret(g, [3]) assert res == 77 - def test_hint(self): - from pypy.rlib import objectmodel - def f(): - x = objectmodel.hint(5, hello="world") - return x - res = self.interpret(f, []) - assert res == 5 - class TestLLtype(BaseTestObjectModel, LLRtypeMixin): def test_cast_to_and_from_weakaddress(self): @@ -312,23 +304,6 @@ res = self.interpret(f, []) assert res == 1 - def test_is_early_constant(self): - from pypy.rlib import objectmodel - def f(x): - if objectmodel._is_early_constant(x): - return 42 - return 0 - - assert f(3) == 0 - res = self.interpret(f, [5]) - assert res == 0 - - def g(): - return f(88) - - res = self.interpret(g, []) - assert res == 42 - class TestOOtype(BaseTestObjectModel, OORtypeMixin): pass From cfbolz at codespeak.net Fri May 25 19:11:27 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 25 May 2007 19:11:27 +0200 (CEST) Subject: [pypy-svn] r43642 - pypy/dist/pypy/translator/c/test Message-ID: <20070525171127.9D16C8082@code0.codespeak.net> Author: cfbolz Date: Fri May 25 19:11:27 2007 New Revision: 43642 Modified: pypy/dist/pypy/translator/c/test/test_symbolic.py Log: fix import Modified: pypy/dist/pypy/translator/c/test/test_symbolic.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_symbolic.py (original) +++ pypy/dist/pypy/translator/c/test/test_symbolic.py Fri May 25 19:11:27 2007 @@ -112,9 +112,9 @@ assert res == 42 def test_is_early_constant(): - from pypy.rlib import objectmodel + from pypy.rlib import jit def f(x): - if objectmodel._is_early_constant(x): + if jit._is_early_constant(x): return 42 return 0 From cfbolz at codespeak.net Fri May 25 19:12:50 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 25 May 2007 19:12:50 +0200 (CEST) Subject: [pypy-svn] r43643 - pypy/dist/pypy/translator/js/test Message-ID: <20070525171250.CDB508082@code0.codespeak.net> Author: cfbolz Date: Fri May 25 19:12:50 2007 New Revision: 43643 Modified: pypy/dist/pypy/translator/js/test/test_rsnippet.py Log: fix another import Modified: pypy/dist/pypy/translator/js/test/test_rsnippet.py ============================================================================== --- pypy/dist/pypy/translator/js/test/test_rsnippet.py (original) +++ pypy/dist/pypy/translator/js/test/test_rsnippet.py Fri May 25 19:12:50 2007 @@ -443,9 +443,9 @@ assert fn() == 2 def test_is_early_constant(): - from pypy.rlib import objectmodel + from pypy.rlib import jit def f(x): - if objectmodel._is_early_constant(x): + if jit._is_early_constant(x): return 42 return 0 fn = compile_function(f, [int]) From fijal at codespeak.net Fri May 25 20:51:10 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 25 May 2007 20:51:10 +0200 (CEST) Subject: [pypy-svn] r43648 - in pypy/branch/kill-ctypes/pypy/rpython/module: . test Message-ID: <20070525185110.3E0578092@code0.codespeak.net> Author: fijal Date: Fri May 25 20:51:09 2007 New Revision: 43648 Modified: pypy/branch/kill-ctypes/pypy/rpython/module/ll_os.py pypy/branch/kill-ctypes/pypy/rpython/module/test/test_ll_os.py Log: WHACK! Now execv has some saner implementation (but test has gone, since it become identical to the one in translator/c :-/ Modified: pypy/branch/kill-ctypes/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/module/ll_os.py Fri May 25 20:51:09 2007 @@ -27,30 +27,25 @@ import ctypes import pypy.rpython.rctypes.implementation from pypy.rpython.rctypes.tool.libc import libc +from pypy.rpython.lltypesystem import rffi +from pypy.rpython.lltypesystem import lltype from pypy.rpython.rctypes.aerrno import geterrno if hasattr(os, 'execv'): if os.name == 'nt': - os_execv = libc._execv + name = '_execv' else: - os_execv = libc.execv - os_execv.argtypes = [ctypes.c_char_p, ctypes.POINTER(ctypes.c_char_p)] - os_execv.restype = ctypes.c_int + name = 'execv' + os_execv = rffi.llexternal(name, [lltype.Ptr(rffi.CCHARP), + lltype.Ptr(rffi.CCHARPP)], lltype.Signed) def execv_lltypeimpl(path, args): - # XXX incredible code to work around rctypes limitations - length = len(args) + 1 - num_bytes = ctypes.sizeof(ctypes.c_char_p) * length - buffer = ctypes.create_string_buffer(num_bytes) - array = ctypes.cast(buffer, ctypes.POINTER(ctypes.c_char_p)) - buffer_addr = ctypes.cast(buffer, ctypes.c_void_p).value - for num in range(len(args)): - adr1 = buffer_addr + ctypes.sizeof(ctypes.c_char_p) * num - ptr = ctypes.c_void_p(adr1) - arrayitem = ctypes.cast(ptr, ctypes.POINTER(ctypes.c_char_p)) - arrayitem[0] = args[num] - os_execv(path, array) + l_path = rffi.str2charp(path) + l_args = rffi.liststr2charpp(args) + os_execv(l_path, l_args) + rffi.free_charpp(l_args) + lltype.free(l_path, flavor='raw') raise OSError(geterrno(), "execv failed") register_external(os.execv, [str, [str]], s_ImpossibleValue, llimpl= Modified: pypy/branch/kill-ctypes/pypy/rpython/module/test/test_ll_os.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/module/test/test_ll_os.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/module/test/test_ll_os.py Fri May 25 20:51:09 2007 @@ -116,25 +116,6 @@ compared_with.sort() assert result == compared_with -if hasattr(os, 'execv'): - from pypy.rpython.extregistry import lookup - os_execv = lookup(os.execv).lltypeimpl - - def test_execv(): - filename = str(udir.join('test_execv_ctypes.txt')) - - progname = str(sys.executable) - l = ['', ''] - l[0] = progname - l[1] = "-c" - l.append('open("%s","w").write("1")' % filename) - pid = os.fork() - if pid == 0: - os_execv(progname, l) - else: - os.waitpid(pid, 0) - assert open(filename).read() == "1" - def test_dup(): from pypy.rpython.extregistry import lookup os_dup = lookup(os.dup).lltypeimpl From fijal at codespeak.net Fri May 25 20:51:32 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 25 May 2007 20:51:32 +0200 (CEST) Subject: [pypy-svn] r43649 - pypy/branch/kill-ctypes/pypy/translator/c/test Message-ID: <20070525185132.B59B68093@code0.codespeak.net> Author: fijal Date: Fri May 25 20:51:32 2007 New Revision: 43649 Modified: pypy/branch/kill-ctypes/pypy/translator/c/test/test_extfunc.py Log: Test might be a bit saner now. Modified: pypy/branch/kill-ctypes/pypy/translator/c/test/test_extfunc.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/translator/c/test/test_extfunc.py (original) +++ pypy/branch/kill-ctypes/pypy/translator/c/test/test_extfunc.py Fri May 25 20:51:32 2007 @@ -708,16 +708,13 @@ filename = str(udir.join('test_execv.txt')) def does_stuff(): progname = str(sys.executable) - l = ['', ''] - l[0] = progname - l[1] = "-c" - l.append('open("%s","w").write("1")' % filename) + l = [progname, '-c', 'open("%s","w").write("1")' % filename] pid = os.fork() if pid == 0: os.execv(progname, l) else: os.waitpid(pid, 0) - func = compile(does_stuff, []) + func = compile(does_stuff, [], backendopt=False) func() assert open(filename).read() == "1" From fijal at codespeak.net Fri May 25 21:02:13 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 25 May 2007 21:02:13 +0200 (CEST) Subject: [pypy-svn] r43650 - in pypy/branch/kill-ctypes/pypy/rpython: lltypesystem lltypesystem/test module Message-ID: <20070525190213.9EE448092@code0.codespeak.net> Author: fijal Date: Fri May 25 21:02:13 2007 New Revision: 43650 Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rffi.py pypy/branch/kill-ctypes/pypy/rpython/module/ll_os.py Log: (fijal, arigo) slight renaming to give saner names Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py Fri May 25 21:02:13 2007 @@ -9,28 +9,28 @@ includes=tuple(includes)) # char * -CCHARP = lltype.Array(lltype.Char, hints={'nolength': True}) +CCHARP = lltype.Ptr(lltype.Array(lltype.Char, hints={'nolength': True})) # various type mapping def str2charp(s): """ str -> char* """ - array = lltype.malloc(CCHARP, len(s) + 1, flavor='raw') + array = lltype.malloc(CCHARP.TO, len(s) + 1, flavor='raw') for i in range(len(s)): array[i] = s[i] array[len(s)] = '\x00' return array # char** -CCHARPP = lltype.Array(lltype.Ptr(CCHARP), hints={'nolength': True}) +CCHARPP = lltype.Ptr(lltype.Array(CCHARP, hints={'nolength': True})) def liststr2charpp(l): """ list[str] -> char**, NULL terminated """ - array = lltype.malloc(CCHARPP, len(l) + 1, flavor='raw') + array = lltype.malloc(CCHARPP.TO, len(l) + 1, flavor='raw') for i in range(len(l)): array[i] = str2charp(l[i]) - array[len(l)] = lltype.nullptr(CCHARP) + array[len(l)] = lltype.nullptr(CCHARP.TO) return array def free_charpp(ref): Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rffi.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rffi.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rffi.py Fri May 25 21:02:13 2007 @@ -40,7 +40,7 @@ assert xf() == 8+3 def test_string(): - z = llexternal('strlen', [Ptr(CCHARP)], Signed, includes=['string.h']) + z = llexternal('strlen', [CCHARP], Signed, includes=['string.h']) def f(): s = str2charp("xxx") @@ -67,7 +67,7 @@ """ c_file = udir.join("stringstar.c") c_file.write(c_source) - z = llexternal('f', [Ptr(CCHARPP)], Signed, sources=[str(c_file)]) + z = llexternal('f', [CCHARPP], Signed, sources=[str(c_file)]) def f(): l = ["xxx", "x", "xxxx"] Modified: pypy/branch/kill-ctypes/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/module/ll_os.py Fri May 25 21:02:13 2007 @@ -37,8 +37,8 @@ name = '_execv' else: name = 'execv' - os_execv = rffi.llexternal(name, [lltype.Ptr(rffi.CCHARP), - lltype.Ptr(rffi.CCHARPP)], lltype.Signed) + os_execv = rffi.llexternal(name, [rffi.CCHARP, rffi.CCHARPP], + lltype.Signed) def execv_lltypeimpl(path, args): l_path = rffi.str2charp(path) From fijal at codespeak.net Fri May 25 21:38:29 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 25 May 2007 21:38:29 +0200 (CEST) Subject: [pypy-svn] r43652 - in pypy/branch/kill-ctypes/pypy/rpython/module: . test Message-ID: <20070525193829.AFF8D8092@code0.codespeak.net> Author: fijal Date: Fri May 25 21:38:29 2007 New Revision: 43652 Modified: pypy/branch/kill-ctypes/pypy/rpython/module/ll_os.py pypy/branch/kill-ctypes/pypy/rpython/module/test/test_ll_os.py Log: That one was easy Modified: pypy/branch/kill-ctypes/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/module/ll_os.py Fri May 25 21:38:29 2007 @@ -52,11 +52,10 @@ execv_lltypeimpl, export_name="ll_os.ll_os_execv") if os.name == 'nt': - os_dup = libc._dup + name = '_dup' else: - os_dup = libc.dup -os_dup.argtypes = [ctypes.c_int] -os_dup.restype = ctypes.c_int + name = 'dup' +os_dup = rffi.llexternal(name, [lltype.Signed], lltype.Signed) def dup_lltypeimpl(fd): newfd = os_dup(fd) @@ -67,11 +66,10 @@ export_name="ll_os.ll_os_dup") if os.name == 'nt': - os_dup2 = libc._dup2 + name = '_dup2' else: - os_dup2 = libc.dup2 -os_dup2.argtypes = [ctypes.c_int, ctypes.c_int] -os_dup2.restype = ctypes.c_int + name = 'dup2' +os_dup2 = rffi.llexternal(name, [lltype.Signed, lltype.Signed], lltype.Signed) def dup2_lltypeimpl(fd, newfd): error = os_dup2(fd, newfd) Modified: pypy/branch/kill-ctypes/pypy/rpython/module/test/test_ll_os.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/module/test/test_ll_os.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/module/test/test_ll_os.py Fri May 25 21:38:29 2007 @@ -115,19 +115,3 @@ compared_with = os.listdir(dirname) compared_with.sort() assert result == compared_with - -def test_dup(): - from pypy.rpython.extregistry import lookup - os_dup = lookup(os.dup).lltypeimpl - testf = udir.join('test.txt') - testf.write("foo") - path = testf.strpath - - def ff(fi): - g = os_dup(fi) - return g - fi = os.open(path,os.O_RDONLY,0755) - g = ff(fi) - assert os.fstat(g) == os.fstat(fi) - - From cfbolz at codespeak.net Fri May 25 22:30:24 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 25 May 2007 22:30:24 +0200 (CEST) Subject: [pypy-svn] r43653 - pypy/dist/pypy/translator/c/src Message-ID: <20070525203024.E0F95808E@code0.codespeak.net> Author: cfbolz Date: Fri May 25 22:30:23 2007 New Revision: 43653 Modified: pypy/dist/pypy/translator/c/src/stack.h Log: USE_STACKLESS does no longer exist Modified: pypy/dist/pypy/translator/c/src/stack.h ============================================================================== --- pypy/dist/pypy/translator/c/src/stack.h (original) +++ pypy/dist/pypy/translator/c/src/stack.h Fri May 25 22:30:23 2007 @@ -13,11 +13,7 @@ void LL_stack_unwind(void) { -#ifdef USE_STACKLESS - LL_stackless_stack_unwind(); -#else RPyRaiseSimpleException(PyExc_RuntimeError, "Recursion limit exceeded"); -#endif } long _LL_stack_growing_direction(char *parent) From arigo at codespeak.net Fri May 25 22:33:37 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 25 May 2007 22:33:37 +0200 (CEST) Subject: [pypy-svn] r43654 - pypy/extradoc/talk/dyla2007 Message-ID: <20070525203337.532D6808E@code0.codespeak.net> Author: arigo Date: Fri May 25 22:33:31 2007 New Revision: 43654 Added: pypy/extradoc/talk/dyla2007/section3.txt (contents, props changed) Log: Starting a new file with the "Metaprogramming is Good" section. For now it seems to make sense to have subsections corresponding to the top-level points of the outline. Added: pypy/extradoc/talk/dyla2007/section3.txt ============================================================================== --- (empty file) +++ pypy/extradoc/talk/dyla2007/section3.txt Fri May 25 22:33:31 2007 @@ -0,0 +1,156 @@ + +Metaprogramming Is Good +======================= + +The present paper proposes to approach the implementation of dynamic +languages from a meta-level: virtual machines for such languages should +not be written by hand, but generated automatically "around" a +description of the language in the form of an interpreter for it. We +argue that this approach gives many of the benefits usually expected by +an implementer when he decides to target an existing object-oriented +virtual machine. It also gives other benefits that we will describe - +mostly in term of flexibility. But most importantly, it lets a +community write a single source implementation of the language, avoiding +the time-consuming task of keeping multiple ones in sync. The single +source can be used to generate either custom VMs for C-like +environments, or interpreters running on top of OO VMs. It makes it +practical to experiment with large changes to the language and with +entirely new languages, like domain-specific languages, while at any +time being able to run the implemented language in a variety of +environments, from C/Posix to the JVM to .NET. + +PyPy architecture +----------------- + +We implemented this idea in the PyPy project. The dynamic language for +which we wrote an interpreter is Python. It is a language which, +because of its size and rather intricate semantics, is a good target for +our approach, in the following sense: its previous reimplementation +(Jython for the JVM and IronPython for .NET) have each proved to be very +time-consuming to maintain. Our implementation is by construction +easier to maintain, and extremely portable (including to C/Posix, to the +JVM and to .NET). + +In metaprogramming terms, the PyPy architecture is as follows: + +* we use a very expressive *object language* (RPython - an analyzable + subset of Python) as the language in which the complete Python + interpreter is written, together with the implementation of its + built-in types. The language is still close to Python, e.g. it is + object-oriented, provides rich built-in types and has automatic memory + management. In other words, the source code of our complete Python + interpreter is mostly free of low-level details - no explicit memory + management, no pieces of C or C-level code. + +* we use a very expressive metalanguage (namely regular Python) to + perform the analysis of RPython code (control flow and data flow + construction, type inference, etc.) and its successive + transformations. + +* this metaprogramming component of PyPy is called the *translation + framework,* as it translates RPython source code (i.e. the full Python + interpreter) into lower-level code. Its purpose is to add aspects to + and specialize the interpreter to fit a selectable virtual or hardware + runtime environment. This either turns the interpreter into a + standalone virtual machine, or integrates it into an existing OO VM. + The necessary support code - e.g. the garbage collector when + targetting C - is itself written in RPython in much the same spirit + that the Jikes RVM's GCs are written in Java [JIKES]; as needed, it is + translated together with the interpreter to form the final custom VM. + +A detailed description of this translation process is beyond the scope +of the present paper; it can be found in [VMC]. The actual Python +interpreter of PyPy and the results we achieved by translating it to C, +LLVM [LLVM] and .NET are described in [XXX]. These results show that +the approach is practical and gives results whose performance is within +the same order of magnitude (within a factor of 2 and improving) of the +hand-written, well-tuned CPython, the C reference implementation. + +A single source +--------------- + +Our approach - a single "meta-written" implementation - naturally leads +to language implementations that have various advantages over the +"hand-written" implementations. First of all, it is a single-source +approach - we explicitly seek to solve the problem of proliferation of +implementation. In the sequel we will show more precise evidence that +this can be done in a practical way with no major drawback. By itself +this would already be a valid argument against the need for +standardization on a single OO VM. But there are also other advantages +in generating language implementations - which are in our opinion very +significant, to the extent that it hints that metaprogramming, though +not widely used in general-purpose programming, is an essential tool in +a language implementer's toolbox. + +Writing the interpreter is easier +--------------------------------- + +A first point is that it makes interpreters easy to write, update and +generally experiment with. More expressiveness helps at all levels: our +Python interpreter is written in RPython as a relatively simple +interpreter, in some respects easier to understand than CPython. We are +using its high level and flexibility to quickly experiment with features +or implementation techniques in ways that would, in a traditional +approach, require pervasive changes to the source code. For example, +PyPy's Python interpreter can optionally provide lazily computed objects +- a 150-lines extension in PyPy that would require global changes in +CPython. Further examples can be found in our technical reports; we +should notably mention an extension adding a state-of-the-art security +model for Python based on data flow tacking [XXX], and general +performance improvements found by extensive experimentation [XXX], some +of which were backported to CPython. + +If we compare with hand-writing an implementation for a specific OO VM, +then the latter requires not only good knowledge of the OO VM in +question and its object model - it requires the language implementer to +fit the language into the imposed models. For example, it is natural to +map classes and instances of a dynamic object-oriented language to the +OO VM's notion of classes and instances, but this might not be a simple +task at all if the models are substantially different and/or if the OO +VM is essentially less dynamic than the language to implement. In our +approach, this efforts is done at two levels: in a first step, while +writing the interpreter, the implementer does not need to worry about +integration with an OO VM's object model. Of course, the integration +effort does not simply vanish - indeed, a simple translation of such an +interpreter to a given OO VM would give an interpreter for the dynamic +language which is unable to communicate with the host VM (which might +already be interesting in specific cases, but not in general). +Integration comes as a second step, and occurs at a different level, by +introducing mappings between the relevant classes of the interpreter and +the corresponding classes of the OO VM. As of yet we have no evidence +that this makes the total integration effort much lower or higher; the +point is that it has proven possible to take the single, OO +VM-independent source code of PyPy's Python interpreter, and produce +from it a version that runs in and integrates with the rest of the .NET +environment - by writing *orthogonal* code. + +At the level of the translation framework, the ability to change or +insert new whole-program transformations makes some aspects of the +interpreter easier to deal with. By "aspect" we mean, in the original +AOP sense, a feature that is added to an object program by a +meta-program. The most obvious example in our context is the garbage +collector for the target environments that lack it. The hand-written C +source of CPython is littered with macro calls that increment and +decrement reference counters. Our translation framework can insert such +macro calls automatically - in fact, we have a choice of GCs, and +reference counting is only one of them (not a particularly efficient +one, either). Some GCs require different transformations of the code. +By contrast, supporting more than one GC in CPython is close to +impossible without forking the whole source code base. + +In fact, there is a well-known example of a CPython fork: Stackless +Python [XXX], which adds support for coroutines. This is very hard to +do in CPython because the interpreter is written in a highly recursive +style. Stackless Python required large-scale changes; it is not merged +back into CPython due to the pervasive increase in complexity that it +requires. In PyPy, though, an optional "stackless" transformation is +able to turn the Python interpreter - also written in a simple highly +recursive style - into an efficient variant of continuation-passing +style (CPS), enabling the usage of coroutines in the translated +interpreter. For more details and other examples of translation-level +transformations, see [XXX]. + +Getting good GCs and tools is possible +-------------------------------------- + +xxx From cfbolz at codespeak.net Fri May 25 22:47:28 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 25 May 2007 22:47:28 +0200 (CEST) Subject: [pypy-svn] r43655 - pypy/extradoc/talk/dyla2007 Message-ID: <20070525204728.DFABE8082@code0.codespeak.net> Author: cfbolz Date: Fri May 25 22:47:28 2007 New Revision: 43655 Modified: pypy/extradoc/talk/dyla2007/section3.txt Log: typos, trying to make a clunky sentences clearer Modified: pypy/extradoc/talk/dyla2007/section3.txt ============================================================================== --- pypy/extradoc/talk/dyla2007/section3.txt (original) +++ pypy/extradoc/talk/dyla2007/section3.txt Fri May 25 22:47:28 2007 @@ -1,6 +1,6 @@ -Metaprogramming Is Good -======================= +Meta-Programming Is Good +======================== The present paper proposes to approach the implementation of dynamic languages from a meta-level: virtual machines for such languages should @@ -31,7 +31,7 @@ easier to maintain, and extremely portable (including to C/Posix, to the JVM and to .NET). -In metaprogramming terms, the PyPy architecture is as follows: +In meta-programming terms, the PyPy architecture is as follows: * we use a very expressive *object language* (RPython - an analyzable subset of Python) as the language in which the complete Python @@ -47,14 +47,14 @@ construction, type inference, etc.) and its successive transformations. -* this metaprogramming component of PyPy is called the *translation +* this meta-programming component of PyPy is called the *translation framework,* as it translates RPython source code (i.e. the full Python interpreter) into lower-level code. Its purpose is to add aspects to and specialize the interpreter to fit a selectable virtual or hardware runtime environment. This either turns the interpreter into a standalone virtual machine, or integrates it into an existing OO VM. The necessary support code - e.g. the garbage collector when - targetting C - is itself written in RPython in much the same spirit + targeting C - is itself written in RPython in much the same spirit that the Jikes RVM's GCs are written in Java [JIKES]; as needed, it is translated together with the interpreter to form the final custom VM. @@ -73,12 +73,12 @@ to language implementations that have various advantages over the "hand-written" implementations. First of all, it is a single-source approach - we explicitly seek to solve the problem of proliferation of -implementation. In the sequel we will show more precise evidence that +implementations. In the sequel we will show more precise evidence that this can be done in a practical way with no major drawback. By itself this would already be a valid argument against the need for standardization on a single OO VM. But there are also other advantages -in generating language implementations - which are in our opinion very -significant, to the extent that it hints that metaprogramming, though +in generating language implementations. In our opinion these are very +significant, to the extent that it hints that meta-programming, though not widely used in general-purpose programming, is an essential tool in a language implementer's toolbox. @@ -98,7 +98,7 @@ should notably mention an extension adding a state-of-the-art security model for Python based on data flow tacking [XXX], and general performance improvements found by extensive experimentation [XXX], some -of which were backported to CPython. +of which were back-ported to CPython. If we compare with hand-writing an implementation for a specific OO VM, then the latter requires not only good knowledge of the OO VM in From cfbolz at codespeak.net Fri May 25 23:35:09 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Fri, 25 May 2007 23:35:09 +0200 (CEST) Subject: [pypy-svn] r43656 - pypy/dist/pypy/annotation Message-ID: <20070525213509.D0E8F808D@code0.codespeak.net> Author: cfbolz Date: Fri May 25 23:35:08 2007 New Revision: 43656 Modified: pypy/dist/pypy/annotation/builtin.py Log: more than two years ago, this was once the annotation of getrefcount (!) Modified: pypy/dist/pypy/annotation/builtin.py ============================================================================== --- pypy/dist/pypy/annotation/builtin.py (original) +++ pypy/dist/pypy/annotation/builtin.py Fri May 25 23:35:08 2007 @@ -269,9 +269,6 @@ pass -def count(s_obj): - return SomeInteger() - def conf(): return SomeString() From tismer at codespeak.net Sat May 26 00:08:46 2007 From: tismer at codespeak.net (tismer at codespeak.net) Date: Sat, 26 May 2007 00:08:46 +0200 (CEST) Subject: [pypy-svn] r43657 - pypy/extradoc/talk/ep2007 Message-ID: <20070525220846.A10E18078@code0.codespeak.net> Author: tismer Date: Sat May 26 00:08:45 2007 New Revision: 43657 Added: pypy/extradoc/talk/ep2007/stackless-essentials.txt Log: the just submitted talk for EP2007, similar but enhanced PyCon version Added: pypy/extradoc/talk/ep2007/stackless-essentials.txt ============================================================================== --- (empty file) +++ pypy/extradoc/talk/ep2007/stackless-essentials.txt Sat May 26 00:08:45 2007 @@ -0,0 +1,78 @@ +Abstract ID: 62 +Abstract content: + +This is a re-worked, actualized and improved version of +my talk at PyCon 2007. Repeating the abstract: + +As a surprise for people who think they know Stackless, +we present the new Stackless implementation For PyPy, +which has led to a significant amount of new insight +about parallel programming and its possible implementations. +We will isolate the known Stackless as a special case of +a general concept. + +This is a Stackless, not a PyPy talk. But the insights presented +here would not exist without PyPy's existance. + +Summary: + +Stackless has been around for a long time now. After several +versions with different goals in mind, the basic concepts of +channels and tasklets turned out to be useful abstractions, +and since many versions, Stackless is only ported from version +to version, without fundamental changes to the principles. + +As some spin-off, Armin Rigo invented Greenlets at a Stackless +sprint. They are some kind of coroutines and a bit of special +semantics. The major benefit is that Greenlets can run on +unmodified CPython. + +In parallel to that, the PyPy project is in its fourth year +now, and one of its goals was Stackless integration as an option. +And of course, Stackless has been integrated into PyPy in a very +nice and elegant way, much nicer than expected. During the design +of the Stackless extension to PyPy, it turned out, that tasklets, +greenlets and coroutines are not that different in principle, +and it was possible to base all known parallel paradigms on +one simple coroutine layout, which is as minimalistic as possible. + +It is a side effect of PyPy's simplicity, that even led to a +pretty new concept for Stackless, that allows all the different +switching paradigms to co-exist without interference. Users +could go further and implement their own concurrency model, +and it would neither interfere with others nor cost performance. + +Today's Stackless can be seen as a special case of the more general +implementation given for PyPy. This implementation can also be taken +as a reference about how Stackless is meant to be implemented. +This reference implementation is completely written in Python. + +The talk will try to isolate the crucial design decisions in +Stackless from implementation details. The reduced concepts +are together the essentials of Stackless. + +Special emphasis is given to interactive examples, simple use-cases, +and an animation that visually explains the new concept of +composability. + +Primary authors: TISMER, Christian (tismerysoft GmbH) +Co-Authors: -- none -- +Presenters: TISMER, Christian +Track classification: Python Language and Libraries +Submitted by: TISMER, Christian +Submitted on: 25 May 2007 23:56 +Last modified on: 25 May 2007 23:56 +Status: SUBMITTED +Comments : + + +As this is a re-worked version of my PyCon 2007 talk, I would like +to improve it by giving more examples, including some of +Andrew Dahlke's PyCon talk. +The original was quite short in explaining concepts, and moving +over to the new concept of composability, quickly. +I would lke to give more space to introducing the basic +concepts this time, having more visualisation, iteractive sessions, +and short repetitions and summaries. +This talk intends to give some insight both to the novice and also +to experienced stackless users. \ No newline at end of file From cfbolz at codespeak.net Sat May 26 02:02:29 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 26 May 2007 02:02:29 +0200 (CEST) Subject: [pypy-svn] r43658 - pypy/extradoc/talk/dyla2007 Message-ID: <20070526000229.35085808D@code0.codespeak.net> Author: cfbolz Date: Sat May 26 02:02:28 2007 New Revision: 43658 Added: pypy/extradoc/talk/dyla2007/section2.txt (contents, props changed) Modified: pypy/extradoc/talk/dyla2007/outline.txt Log: slightly messy and sketchy first attempt at the second section of the paper. Modified: pypy/extradoc/talk/dyla2007/outline.txt ============================================================================== --- pypy/extradoc/talk/dyla2007/outline.txt (original) +++ pypy/extradoc/talk/dyla2007/outline.txt Sat May 26 02:02:28 2007 @@ -40,12 +40,6 @@ - People eventually ask for better integration with successful OO VMs, mere bridges are temporary hacks => more implementations -- Proliferation of implementations, division of the effort, troublesome - if resources are limited. Implementations written for a specific OO VM - have advantages over custom VMs in C, so people propose as a solution - to standardize on one OO VM and just have this one implementation of - the dyn language. - - Let's first explore the alledged advantages of writing dynamic language implementations in an OO VM over writing custom VMs in C: * more interoperability than the C level @@ -75,6 +69,13 @@ other low-level aspects, and of course interoperability with the rest of the VM + +- Proliferation of implementations, division of the effort, troublesome + if resources are limited. Implementations written for a specific OO VM + have advantages over custom VMs in C, so people propose as a solution + to standardize on one OO VM and just have this one implementation of + the dyn language. + - A single best base OO VM for everyone would help reducing the proliferation of implementations; but that's unlikely to occur, and it would come with trade-offs too Added: pypy/extradoc/talk/dyla2007/section2.txt ============================================================================== --- (empty file) +++ pypy/extradoc/talk/dyla2007/section2.txt Sat May 26 02:02:28 2007 @@ -0,0 +1,236 @@ + +XXX maybe drop something like that we want to look at the "life cycle" of a +successful open source dynamic language + +XXX define OO VM somewhere: common effort, usually backed by a large group of +people or even company. lots of engineering effort, high tuning, typical +examples JVM, CLR (maybe parrot?) + +XXX describe open source and research general conditions somewhere + - low amount of workforce + - volunteers (in the case of open source) + + - due to complicated semantics: simple bytecode oder AST interpreter. no + threaded code (ref) or other advanced interpreter technology or even dynamic + compilation + + +Dynamic languages are traditionally implemented by writing a virtual +machine for them in a low-level language like C, or in a language that +can relatively easily be turned into C. The machine implements an +object model supporting the high level dynamic language's objects. It +typically provides features like automatic garbage collection. Recent +languages like Python, Ruby, Perl and JavaScript have complicated +semantics which are most easily mapped to a simple interpreter operating +on syntax trees or bytecode. Simpler languages like Lisp and Self +typically have more efficient implementations based on just-in-time code +generation. + +There are many good reasons for choosing C as an implementation language. It +gives a decent base performance, is highly portable across different hardware +architectures and operating system if some care is taken and it allows the reuse +of many existing libraries. + +On the other hand the use of C comes with a set of problems. Since C has no ways +to do automatic memory management and since the implemented languages are +garbage collected, the VM needs to contain a garbage collector. Implementing a +high-performance GC is not a simple task and so many implementations resort to +relatively simple algorithms like mark-and-sweep or reference counting. Also the +choice of garbage collector is fixed relatively early in the life of the project +and then manifests itself throughout the complete VM source code. Thus it +becomes extremely tedious to change it later. + +A similar problem is the threading model the VM uses. XXX expand a bit + - simpler threading models + - don't make single-threaded apps slower due to locks + - global locks + - green threading + + - C-level implementations of dyn language VMs have issues and trade-offs: + - reduced flexibility + - pressure to keep the implementation simple + + - wrapping with C libraries simple but tedious + +These listed limitations of a C-based implementation often lead to more +implementations of the language. For example in addition to the main C +implementation of Python ("CPython") there is a number of additional +implementations that want to fix some of the limitations of CPython. One of them +is "Stackless Python" which is a fork of CPython that adds micro-threading +capabilities to Python. One of the reasons for not incorporating back into +CPython was that it was felt that they would make the implementation too +complex. Another implementation of the Python language is the Psyco project which adds a JIT-compiler to CPython. +XXX maybe drop something like that we want to look at the "life cycle" of a +successful open source dynamic language + +XXX define OO VM somewhere: common effort, usually backed by a large group of +people or even company. lots of engineering effort, high tuning, typical +examples JVM, CLR (maybe parrot?) + +XXX describe open source and research general conditions somewhere + - low amount of workforce + - volunteers (in the case of open source) + + - due to complicated semantics: simple bytecode oder AST interpreter. no + threaded code (ref) or other advanced interpreter technology or even dynamic + compilation + + +Section 2 Title +=============== + +Dynamic languages are traditionally implemented by writing a virtual +machine for them in a low-level language like C, or in a language that +can relatively easily be turned into C. The machine implements an +object model supporting the high level dynamic language's objects. It +typically provides features like automatic garbage collection. Recent +languages like Python, Ruby, Perl and JavaScript have complicated +semantics which are most easily mapped to a simple interpreter operating +on syntax trees or bytecode. Simpler languages like Lisp and Self +typically have more efficient implementations based on just-in-time code +generation. + +There are many good reasons for choosing C as an implementation language. It +gives a decent base performance, is highly portable across different hardware +architectures and operating system if some care is taken and it allows the reuse +of many existing libraries. + +On the other hand the use of C comes with a set of problems. Since C has no ways +to do automatic memory management and since the implemented languages are +garbage collected, the VM needs to contain a garbage collector. Implementing a +high-performance GC is not a simple task and so many implementations resort to +relatively simple algorithms like mark-and-sweep or reference counting. Also the +choice of garbage collector is fixed relatively early in the life of the project +and then manifests itself throughout the complete VM source code. Thus it +becomes extremely tedious to change it later. + +A similar problem is the threading model the VM uses. XXX expand a bit + - simpler threading models + - don't make single-threaded apps slower due to locks + - global locks + - green threading + + - C-level implementations of dyn language VMs have issues and trade-offs: + - reduced flexibility + - pressure to keep the implementation simple + + - wrapping with C libraries simple but tedious + +These listed limitations of a C-based implementation often lead to more +implementations of the language. For example in addition to the main C +implementation of Python ("CPython") there is a number of additional +implementations that want to fix some of the limitations of CPython. One of them +is "Stackless Python" which is a fork of CPython that adds micro-threading +capabilities to Python. One of the reasons for not incorporating back into +CPython was that it was felt that they would make the implementation too +complex. Another implementation of the Python language is the Psyco project +which adds a JIT-compiler to CPython. As all other reimplementations, Psyco +contains the Pythonsemantics and needs to be kept synchronized with CPython +manually. + + - free threading attempts for Python + +As a language becomes more popular, yet another reason for different +implementations emerge: People want to use it together with Java and C# +libraries and tools. Although it is in principle possible to do that by +bridging the C-VM with the respective VMs of these languages (XXX examples) +this is an incomplete solution since it prevents many useful use-cases. +Therefore in the end implementations of the dynamic language on top of the +object-oriented VMs are created to properly integrate with the complete +environment. + + +Assessing the Advantages of Implementing a Language on Top of OO VMs +-------------------------------------------------------------------- + +Let's now take a look at the advantages that are usually cited for basing a +language implementation of a dynamic language on a standard object oriented virtual +machine, for example the JVM or the CLR: + +*Better interoperability than the C level:* Since the VM offers a standard +object model and all the languages implemented on top of it are using it, it +is easier to integrate the languages on top of the VM. This allows reuse of +libraries between all the implemented languages. This is typically the most +important reason for wanting an implementation on the VM in the first place. + +*Cross-platform portability:* Only the underlying VM has to be ported to +various hardware architectures and operating systems. The languages +implemented on top can then be run without change in various environments. + +*Better tools:* XXX + +*Better implementation of low-level issues like garbage collection, threading:* +XXX + +*Better performance:* Object-oriented VMs usually come with a highly tuned +just-in-time compiler to make them perform well without requiring ahead-of-time +compilation to machine language. This in addition with the previous point leads +to much better performance of the languages running on top of the VM. + +*Ease of implementation:* The implementation of a language on top of an OO VM is +easier because it starts of at a higher level than C. Usually a high-level +language like Java or C# is used for the language implementation, which both +offer the language implementer a much higher level of abstraction. XXX expand + +*A single unified implementation base:* XXX + + +At a closer look, some of these advantages are only partially true in practice. + +*Better performance:* So far it seems like performance of highly dynamic +languages is not actually significantly improved on OO VMs. Psyco is around 5 +times slower than CPython, for IronPython the figures vary but it is mostly +within the same order of magnitude as CPython. The most important reason for +this that the VM's JIT compilers are not prepared to deal with the highly +dynamic behaviour and the complex dispatch semantics of dynamic languages. (XXX +expand) + +*Better GCs:* XXX large memory overhead of OO VMs + +*Cross-platform portability:* While this is true to some extend, the situation +with regard to portability is not significantly improved compared to e.g. +C/Posix, which is relatively portable too. Also portability sometimes comes at +the price of performance, because even if the OO VM is running on a particular +hardware architecture it is not clear that the JIT is tuned for this +architecture too or working at all, which leads to significantly less +performance. + +*Ease of implementation:* This point is disputable. While it is true that OO VMs +typically allow the language implementor to start at a higher level, they also +enforce a specific object and execution model. This means that the concepts of +the implemented language need to be mapped to the execution model of the +underlying VM, which may be easy or not, depending very much on the language in +question. XXX give example where it works well. + +An example where this mapping does not work too well is Prolog. While there +exist several implementations of Prolog on top of the JVM and also one on .NET, +they are not particular efficient, especially when compared to good Prolog VMs +in written in C. This is mostly because the Prolog execution model, which +involves backtracking and deep recursion does not fit the JVM and .NET very +well. XXX some more + + +On the other hand some of the benefits are real and very useful, the most +prominent being the easy interaction with the rest of the VM. Furthermore there +is better tool support and better GCs. XXX expand + + +The Cost of Implementation-Proliferation +----------------------------------------- + +The described proliferation of language implementations is a big problem for +language communities. Although most individual implementations exist for good +reasons, the sum of all of them and the need to keep them synchronized with the +reference implementations lead to a lot of duplicated work and division of +efforts. This is especially true for open source languages which tend to evolve +quickly. At any one point in time some of the implementations will lag behind +which makes writing code which can work on all of the implementations harder. + +Implementing a language on top of a OO VM has many advantages, so some +people propose the solution of standardizing on one particular OO VM to not have +to maintain implementations for several of them. While this would in theory +alleviate the problem it is unlikely to happen. On the one hand many political +issues are involved in such a decision. On the other hand deciding on one single +object and execution model would not be an equally good fit for all languages. + + From cfbolz at codespeak.net Sat May 26 02:07:52 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 26 May 2007 02:07:52 +0200 (CEST) Subject: [pypy-svn] r43659 - pypy/extradoc/talk/dyla2007 Message-ID: <20070526000752.2E320808D@code0.codespeak.net> Author: cfbolz Date: Sat May 26 02:07:51 2007 New Revision: 43659 Modified: pypy/extradoc/talk/dyla2007/section2.txt Log: uh? for some reasons some parts of the document were duplicated Modified: pypy/extradoc/talk/dyla2007/section2.txt ============================================================================== --- pypy/extradoc/talk/dyla2007/section2.txt (original) +++ pypy/extradoc/talk/dyla2007/section2.txt Sat May 26 02:07:51 2007 @@ -1,65 +1,3 @@ - -XXX maybe drop something like that we want to look at the "life cycle" of a -successful open source dynamic language - -XXX define OO VM somewhere: common effort, usually backed by a large group of -people or even company. lots of engineering effort, high tuning, typical -examples JVM, CLR (maybe parrot?) - -XXX describe open source and research general conditions somewhere - - low amount of workforce - - volunteers (in the case of open source) - - - due to complicated semantics: simple bytecode oder AST interpreter. no - threaded code (ref) or other advanced interpreter technology or even dynamic - compilation - - -Dynamic languages are traditionally implemented by writing a virtual -machine for them in a low-level language like C, or in a language that -can relatively easily be turned into C. The machine implements an -object model supporting the high level dynamic language's objects. It -typically provides features like automatic garbage collection. Recent -languages like Python, Ruby, Perl and JavaScript have complicated -semantics which are most easily mapped to a simple interpreter operating -on syntax trees or bytecode. Simpler languages like Lisp and Self -typically have more efficient implementations based on just-in-time code -generation. - -There are many good reasons for choosing C as an implementation language. It -gives a decent base performance, is highly portable across different hardware -architectures and operating system if some care is taken and it allows the reuse -of many existing libraries. - -On the other hand the use of C comes with a set of problems. Since C has no ways -to do automatic memory management and since the implemented languages are -garbage collected, the VM needs to contain a garbage collector. Implementing a -high-performance GC is not a simple task and so many implementations resort to -relatively simple algorithms like mark-and-sweep or reference counting. Also the -choice of garbage collector is fixed relatively early in the life of the project -and then manifests itself throughout the complete VM source code. Thus it -becomes extremely tedious to change it later. - -A similar problem is the threading model the VM uses. XXX expand a bit - - simpler threading models - - don't make single-threaded apps slower due to locks - - global locks - - green threading - - - C-level implementations of dyn language VMs have issues and trade-offs: - - reduced flexibility - - pressure to keep the implementation simple - - - wrapping with C libraries simple but tedious - -These listed limitations of a C-based implementation often lead to more -implementations of the language. For example in addition to the main C -implementation of Python ("CPython") there is a number of additional -implementations that want to fix some of the limitations of CPython. One of them -is "Stackless Python" which is a fork of CPython that adds micro-threading -capabilities to Python. One of the reasons for not incorporating back into -CPython was that it was felt that they would make the implementation too -complex. Another implementation of the Python language is the Psyco project which adds a JIT-compiler to CPython. XXX maybe drop something like that we want to look at the "life cycle" of a successful open source dynamic language @@ -125,7 +63,7 @@ CPython was that it was felt that they would make the implementation too complex. Another implementation of the Python language is the Psyco project which adds a JIT-compiler to CPython. As all other reimplementations, Psyco -contains the Pythonsemantics and needs to be kept synchronized with CPython +contains the Python semantics and needs to be kept synchronized with CPython manually. - free threading attempts for Python @@ -233,4 +171,7 @@ issues are involved in such a decision. On the other hand deciding on one single object and execution model would not be an equally good fit for all languages. - +In the next section we are exploring a different approach for implementing +dynamic languages that we hope is able to solve many of the problems of +implementing a language, in particular the problem of an explosion of the number +of implementations. From fijal at codespeak.net Sat May 26 02:20:24 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 26 May 2007 02:20:24 +0200 (CEST) Subject: [pypy-svn] r43660 - pypy/extradoc/talk/dyla2007 Message-ID: <20070526002024.6E0C7808E@code0.codespeak.net> Author: fijal Date: Sat May 26 02:20:24 2007 New Revision: 43660 Modified: pypy/extradoc/talk/dyla2007/section2.txt Log: s/Psyco/Jython/ Modified: pypy/extradoc/talk/dyla2007/section2.txt ============================================================================== --- pypy/extradoc/talk/dyla2007/section2.txt (original) +++ pypy/extradoc/talk/dyla2007/section2.txt Sat May 26 02:20:24 2007 @@ -116,7 +116,7 @@ At a closer look, some of these advantages are only partially true in practice. *Better performance:* So far it seems like performance of highly dynamic -languages is not actually significantly improved on OO VMs. Psyco is around 5 +languages is not actually significantly improved on OO VMs. Jython is around 5 times slower than CPython, for IronPython the figures vary but it is mostly within the same order of magnitude as CPython. The most important reason for this that the VM's JIT compilers are not prepared to deal with the highly From fijal at codespeak.net Sat May 26 12:00:37 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 26 May 2007 12:00:37 +0200 (CEST) Subject: [pypy-svn] r43661 - in pypy/branch/kill-ctypes/pypy/rpython/lltypesystem: . test Message-ID: <20070526100037.DBED28099@code0.codespeak.net> Author: fijal Date: Sat May 26 12:00:36 2007 New Revision: 43661 Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rffi.py Log: Another bit - a structure: test and a small helper Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py Sat May 26 12:00:36 2007 @@ -8,6 +8,19 @@ sources=tuple(sources), includes=tuple(includes)) +def cstruct(name, *fields, **kwds): + """ A small helper to create external C structure, not the + pypy one + """ + if 'hints' not in kwds: + kwds['hints'] = {} + kwds['hints']['external'] = 'C' + kwds['hints']['c_name'] = name + # XXX obscure hack, stolen for unknown reason from ctypes, + # probably because of _ attributes + c_fields = [('c_' + key, value) for key, value in fields] + return lltype.Ptr(lltype.Struct(name, *c_fields, **kwds)) + # char * CCHARP = lltype.Ptr(lltype.Array(lltype.Char, hints={'nolength': True})) Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rffi.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rffi.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rffi.py Sat May 26 12:00:36 2007 @@ -78,3 +78,41 @@ xf = compile(f, [], backendopt=False) assert xf() == 8 + +def test_struct(): + h_source = """ + struct xx { + int one; + char two; + int three; + }; + """ + h_file = udir.join("structxx.h") + h_file.write(h_source) + + c_source = """ + #include "structxx.h" + int f(struct xx* z) + { + return (z->one + z->three); + } + """ + TP = cstruct('xx', ('one', Signed), ('two', Char), ('three', Signed)) + + c_file = udir.join("structxx.c") + c_file.write(c_source) + z = llexternal('f', [TP], Signed, sources=[str(c_file)], + includes=[str(h_file)]) + + def f(): + struct = lltype.malloc(TP.TO, flavor='raw') + struct.c_one = 3 + struct.c_two = '\x33' + struct.c_three = 5 + result = z(struct) + lltype.free(struct, flavor='raw') + return result + + fn = compile(f, [], backendopt=False) + assert fn() == 8 + From fijal at codespeak.net Sat May 26 12:01:51 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 26 May 2007 12:01:51 +0200 (CEST) Subject: [pypy-svn] r43662 - in pypy/branch/kill-ctypes/pypy/rpython/lltypesystem: . test Message-ID: <20070526100151.5BA7B8099@code0.codespeak.net> Author: fijal Date: Sat May 26 12:01:50 2007 New Revision: 43662 Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rffi.py Log: Rename Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py Sat May 26 12:01:50 2007 @@ -8,7 +8,7 @@ sources=tuple(sources), includes=tuple(includes)) -def cstruct(name, *fields, **kwds): +def CStruct(name, *fields, **kwds): """ A small helper to create external C structure, not the pypy one """ Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rffi.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rffi.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rffi.py Sat May 26 12:01:50 2007 @@ -97,7 +97,7 @@ return (z->one + z->three); } """ - TP = cstruct('xx', ('one', Signed), ('two', Char), ('three', Signed)) + TP = CStruct('xx', ('one', Signed), ('two', Char), ('three', Signed)) c_file = udir.join("structxx.c") c_file.write(c_source) From fijal at codespeak.net Sat May 26 12:11:00 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 26 May 2007 12:11:00 +0200 (CEST) Subject: [pypy-svn] r43663 - pypy/dist/pypy/translator/jvm Message-ID: <20070526101100.490A28099@code0.codespeak.net> Author: fijal Date: Sat May 26 12:10:59 2007 New Revision: 43663 Modified: pypy/dist/pypy/translator/jvm/genjvm.py Log: probably shell=True is needed to find jasmin.bat (maaaybe...), but breaks all the tests on UNIX and spam wyvern automated tests. Modified: pypy/dist/pypy/translator/jvm/genjvm.py ============================================================================== --- pypy/dist/pypy/translator/jvm/genjvm.py (original) +++ pypy/dist/pypy/translator/jvm/genjvm.py Sat May 26 12:10:59 2007 @@ -91,9 +91,14 @@ self.jasmin_files = jfiles def _invoke(self, args, allow_stderr): + import sys + if sys.platform == 'nt': + shell = True + else: + shell = False subp = subprocess.Popen( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - shell=True, universal_newlines=True) + shell=shell, universal_newlines=True) stdout, stderr = subp.communicate() res = subp.wait() if res or (not allow_stderr and stderr): From fijal at codespeak.net Sat May 26 12:16:07 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 26 May 2007 12:16:07 +0200 (CEST) Subject: [pypy-svn] r43664 - pypy/dist/pypy/translator/cli/test Message-ID: <20070526101607.2F2398099@code0.codespeak.net> Author: fijal Date: Sat May 26 12:16:06 2007 New Revision: 43664 Modified: pypy/dist/pypy/translator/cli/test/test_op.py Log: Make -k happy Modified: pypy/dist/pypy/translator/cli/test/test_op.py ============================================================================== --- pypy/dist/pypy/translator/cli/test/test_op.py (original) +++ pypy/dist/pypy/translator/cli/test/test_op.py Sat May 26 12:16:06 2007 @@ -1,5 +1,7 @@ from pypy.translator.cli.test.runtest import CliTest from pypy.translator.oosupport.test_template.operations import BaseTestOperations +# ====> ../../oosupport/test_template/operations.py + class TestOperations(CliTest, BaseTestOperations): pass From fijal at codespeak.net Sat May 26 12:16:20 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 26 May 2007 12:16:20 +0200 (CEST) Subject: [pypy-svn] r43665 - pypy/dist/pypy/translator/oosupport/test_template Message-ID: <20070526101620.F37D78099@code0.codespeak.net> Author: fijal Date: Sat May 26 12:16:20 2007 New Revision: 43665 Modified: pypy/dist/pypy/translator/oosupport/test_template/operations.py Log: Fix the test Modified: pypy/dist/pypy/translator/oosupport/test_template/operations.py ============================================================================== --- pypy/dist/pypy/translator/oosupport/test_template/operations.py (original) +++ pypy/dist/pypy/translator/oosupport/test_template/operations.py Sat May 26 12:16:20 2007 @@ -192,9 +192,9 @@ self._check_all(fn) def test_is_early_constant(self): - from pypy.rlib import objectmodel + from pypy.rlib import jit def f(x): - if objectmodel._is_early_constant(x): + if jit._is_early_constant(x): return 42 return 0 assert self.interpret(f, [5]) == 0 From fijal at codespeak.net Sat May 26 12:17:43 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 26 May 2007 12:17:43 +0200 (CEST) Subject: [pypy-svn] r43666 - pypy/dist/pypy/translator/llvm/test Message-ID: <20070526101743.319678099@code0.codespeak.net> Author: fijal Date: Sat May 26 12:17:42 2007 New Revision: 43666 Modified: pypy/dist/pypy/translator/llvm/test/test_symbolic.py Log: Fix the tests (I hope), don't have llvm to test it here. Modified: pypy/dist/pypy/translator/llvm/test/test_symbolic.py ============================================================================== --- pypy/dist/pypy/translator/llvm/test/test_symbolic.py (original) +++ pypy/dist/pypy/translator/llvm/test/test_symbolic.py Sat May 26 12:17:42 2007 @@ -169,9 +169,9 @@ def test_is_early_constant(): - from pypy.rlib import objectmodel + from pypy.rlib import jit def f(x): - if objectmodel._is_early_constant(x): + if jit._is_early_constant(x): return 42 return 0 @@ -181,9 +181,9 @@ assert res == 0 def test_we_are_jitted(): - from pypy.rlib import objectmodel + from pypy.rlib import jit def f(): - return objectmodel.we_are_jitted() + return jit.we_are_jitted() fn = compile_function(f, []) res = fn() assert res == False From fijal at codespeak.net Sat May 26 12:37:01 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 26 May 2007 12:37:01 +0200 (CEST) Subject: [pypy-svn] r43667 - pypy/dist/pypy/translator/js/examples/console/test Message-ID: <20070526103701.E2CC48099@code0.codespeak.net> Author: fijal Date: Sat May 26 12:37:01 2007 New Revision: 43667 Modified: pypy/dist/pypy/translator/js/examples/console/test/test_session.py Log: Skip windows tests Modified: pypy/dist/pypy/translator/js/examples/console/test/test_session.py ============================================================================== --- pypy/dist/pypy/translator/js/examples/console/test/test_session.py (original) +++ pypy/dist/pypy/translator/js/examples/console/test/test_session.py Sat May 26 12:37:01 2007 @@ -8,6 +8,10 @@ from py.__.green.pipe.fd import FDInput from pypy.translator.js.examples.console.session import Interpreter, Killed +import sys +if sys.platform == 'nt': + py.test.skip("Those tests doesn't run on windows (yet)") + def test_greensock_reader_timeouter(): i = Interpreter("python", timeout=3) while not i.interact().endswith(">>> "): From fijal at codespeak.net Sat May 26 12:58:06 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 26 May 2007 12:58:06 +0200 (CEST) Subject: [pypy-svn] r43668 - in pypy/branch/kill-ctypes/pypy/rpython/lltypesystem: . test Message-ID: <20070526105806.74BDD809D@code0.codespeak.net> Author: fijal Date: Sat May 26 12:58:06 2007 New Revision: 43668 Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rffi.py Log: Slighlty hackish support for errno. Proper support for constants should come at some point, but let's leave with this... Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py Sat May 26 12:58:06 2007 @@ -1,6 +1,7 @@ from pypy.rpython.lltypesystem import lltype from pypy.rpython.lltypesystem.lloperation import llop +from pypy.rlib.objectmodel import CDefinedIntSymbolic def llexternal(name, args, result, sources=[], includes=[]): ext_type = lltype.FuncType(args, result) @@ -21,6 +22,9 @@ c_fields = [('c_' + key, value) for key, value in fields] return lltype.Ptr(lltype.Struct(name, *c_fields, **kwds)) +# XXX slightly hackish +c_errno = CDefinedIntSymbolic('errno', default=-1) + # char * CCHARP = lltype.Ptr(lltype.Array(lltype.Char, hints={'nolength': True})) Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rffi.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rffi.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rffi.py Sat May 26 12:58:06 2007 @@ -116,3 +116,22 @@ fn = compile(f, [], backendopt=False) assert fn() == 8 +def test_constant(): + import os + + def f(): + return c_errno + + def g(): + try: + os.write(12312312, "xxx") + except OSError: + pass + return c_errno + + fn = compile(f, []) + assert fn() == 0 + gn = compile(g, []) + import errno + assert gn() == errno.EBADF + From fijal at codespeak.net Sat May 26 13:40:59 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 26 May 2007 13:40:59 +0200 (CEST) Subject: [pypy-svn] r43669 - pypy/branch/kill-ctypes/pypy/translator/c Message-ID: <20070526114059.A451D80A4@code0.codespeak.net> Author: fijal Date: Sat May 26 13:40:59 2007 New Revision: 43669 Modified: pypy/branch/kill-ctypes/pypy/translator/c/funcgen.py Log: Argh. didn't meant to do this. Modified: pypy/branch/kill-ctypes/pypy/translator/c/funcgen.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/translator/c/funcgen.py (original) +++ pypy/branch/kill-ctypes/pypy/translator/c/funcgen.py Sat May 26 13:40:59 2007 @@ -564,6 +564,8 @@ result += '\nif(%s) %s->%s = %s;' % (eresult, eresult, lenfld, elength) result += '\n}' return result + + OP_ZERO_MALLOC_VARSIZE = OP_MALLOC_VARSIZE def OP_RAW_MALLOC(self, op): eresult = self.expr(op.result) From pedronis at codespeak.net Sat May 26 14:01:37 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Sat, 26 May 2007 14:01:37 +0200 (CEST) Subject: [pypy-svn] r43670 - in pypy/dist/pypy: jit/hintannotator lang/prolog/interpreter module/pypyjit Message-ID: <20070526120137.5715180AE@code0.codespeak.net> Author: pedronis Date: Sat May 26 14:01:35 2007 New Revision: 43670 Modified: pypy/dist/pypy/jit/hintannotator/policy.py pypy/dist/pypy/lang/prolog/interpreter/portal.py pypy/dist/pypy/module/pypyjit/portal.py Log: trying to fix translate.py --jit which should have been tested before merging :( still broken but goes further than when I started Modified: pypy/dist/pypy/jit/hintannotator/policy.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/policy.py (original) +++ pypy/dist/pypy/jit/hintannotator/policy.py Sat May 26 14:01:35 2007 @@ -51,7 +51,7 @@ self.timeshift_graphs = {} portal = getattr(self.PORTAL, 'im_func', self.PORTAL) portal_graph = graphof(t, portal) - self.fill_timeshift_graphs(t, portal_graph) + self.fill_timeshift_graphs(portal_graph) def look_inside_graph(self, graph): if graph in self.timeshift_graphs: @@ -75,7 +75,7 @@ def look_inside_graph_of_module(self, graph, func, mod): return True - def fill_timeshift_graphs(self, t, portal_graph): + def fill_timeshift_graphs(self, portal_graph): # subclasses should have their own pass @@ -87,7 +87,7 @@ def seefunc(self, fromfunc, *tofuncs): targetgraphs = {} for tofunc in tofuncs: - targetgraphs[_graph(tofunc)] = True + targetgraphs[self._graph(tofunc)] = True graphs = graphs_on_the_path_to(self.translator, self._graph(fromfunc), targetgraphs) for graph in graphs: Modified: pypy/dist/pypy/lang/prolog/interpreter/portal.py ============================================================================== --- pypy/dist/pypy/lang/prolog/interpreter/portal.py (original) +++ pypy/dist/pypy/lang/prolog/interpreter/portal.py Sat May 26 14:01:35 2007 @@ -25,7 +25,7 @@ return False return True - def fill_timeshift_graphs(self, t, portal_graph): + def fill_timeshift_graphs(self, portal_graph): import pypy for cls in [term.Var, term.Term, term.Number, term.Atom]: self.seegraph(cls.copy) Modified: pypy/dist/pypy/module/pypyjit/portal.py ============================================================================== --- pypy/dist/pypy/module/pypyjit/portal.py (original) +++ pypy/dist/pypy/module/pypyjit/portal.py Sat May 26 14:01:35 2007 @@ -1,3 +1,4 @@ +import pypy from pypy.module.pypyjit.interp_jit import PORTAL from pypy.module.pypyjit.newbool import NewBoolDesc from pypy.translator.translator import graphof @@ -5,6 +6,8 @@ from pypy.jit.hintannotator.policy import ManualGraphPolicy class PyPyHintAnnotatorPolicy(ManualGraphPolicy): + PORTAL = PORTAL + def look_inside_graph_of_module(self, graph, func, module): if mod.startswith('pypy.objspace'): return False @@ -125,6 +128,6 @@ portal = getattr(PORTAL, 'im_func', PORTAL) portal_graph = graphof(t, portal) - policy = PyPyHintAnnotatorPolicy(timeshift_graphs(t, portal_graph, - drv.log)) + policy = PyPyHintAnnotatorPolicy() + policy.seetranslator(t) return portal, policy From fijal at codespeak.net Sat May 26 14:15:49 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 26 May 2007 14:15:49 +0200 (CEST) Subject: [pypy-svn] r43671 - pypy/dist/pypy/translator/llvm Message-ID: <20070526121549.AF57680AE@code0.codespeak.net> Author: fijal Date: Sat May 26 14:15:48 2007 New Revision: 43671 Modified: pypy/dist/pypy/translator/llvm/database.py Log: An attempt to fix llvm build. Modified: pypy/dist/pypy/translator/llvm/database.py ============================================================================== --- pypy/dist/pypy/translator/llvm/database.py (original) +++ pypy/dist/pypy/translator/llvm/database.py Sat May 26 14:15:48 2007 @@ -17,6 +17,7 @@ from pypy.rlib.objectmodel import Symbolic, ComputedIntSymbolic from pypy.rlib.objectmodel import CDefinedIntSymbolic from pypy.rlib import objectmodel +from pypy.rlib import jit log = log.database @@ -508,7 +509,7 @@ elif isinstance(value, CDefinedIntSymbolic): if value is objectmodel.malloc_zero_filled: repr = '1' - elif value is objectmodel._we_are_jitted: + elif value is jit._we_are_jitted: repr = '0' else: raise NotImplementedError("CDefinedIntSymbolic: %r" % (value,)) From cfbolz at codespeak.net Sat May 26 14:20:11 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 26 May 2007 14:20:11 +0200 (CEST) Subject: [pypy-svn] r43672 - in pypy/dist/pypy: jit/hintannotator lang/prolog/interpreter module/pypyjit Message-ID: <20070526122011.0388480B1@code0.codespeak.net> Author: cfbolz Date: Sat May 26 14:20:11 2007 New Revision: 43672 Modified: pypy/dist/pypy/jit/hintannotator/policy.py pypy/dist/pypy/lang/prolog/interpreter/portal.py pypy/dist/pypy/module/pypyjit/portal.py Log: with these changes translate.py --jit gets to timeshifting again. Modified: pypy/dist/pypy/jit/hintannotator/policy.py ============================================================================== --- pypy/dist/pypy/jit/hintannotator/policy.py (original) +++ pypy/dist/pypy/jit/hintannotator/policy.py Sat May 26 14:20:11 2007 @@ -70,7 +70,7 @@ if getattr(func, '_pure_function_', False): return False mod = func.__module__ or '?' - return self.look_inside_graph_of_module(graph, mod) + return self.look_inside_graph_of_module(graph, func, mod) def look_inside_graph_of_module(self, graph, func, mod): return True Modified: pypy/dist/pypy/lang/prolog/interpreter/portal.py ============================================================================== --- pypy/dist/pypy/lang/prolog/interpreter/portal.py (original) +++ pypy/dist/pypy/lang/prolog/interpreter/portal.py Sat May 26 14:20:11 2007 @@ -16,7 +16,7 @@ class PyrologHintAnnotatorPolicy(ManualGraphPolicy): PORTAL = PORTAL - def look_inside_graph_of_module(self, graph, mod): + def look_inside_graph_of_module(self, graph, func, mod): if mod in forbidden_modules: return False if mod in good_modules: Modified: pypy/dist/pypy/module/pypyjit/portal.py ============================================================================== --- pypy/dist/pypy/module/pypyjit/portal.py (original) +++ pypy/dist/pypy/module/pypyjit/portal.py Sat May 26 14:20:11 2007 @@ -8,7 +8,7 @@ class PyPyHintAnnotatorPolicy(ManualGraphPolicy): PORTAL = PORTAL - def look_inside_graph_of_module(self, graph, func, module): + def look_inside_graph_of_module(self, graph, func, mod): if mod.startswith('pypy.objspace'): return False if '_geninterp_' in func.func_globals: # skip all geninterped stuff From cfbolz at codespeak.net Sat May 26 16:46:16 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 26 May 2007 16:46:16 +0200 (CEST) Subject: [pypy-svn] r43681 - pypy/extradoc/talk/dyla2007 Message-ID: <20070526144616.E30D780BD@code0.codespeak.net> Author: cfbolz Date: Sat May 26 16:46:15 2007 New Revision: 43681 Modified: pypy/extradoc/talk/dyla2007/outline.txt Log: a point that samuele thinks should be stressed more Modified: pypy/extradoc/talk/dyla2007/outline.txt ============================================================================== --- pypy/extradoc/talk/dyla2007/outline.txt (original) +++ pypy/extradoc/talk/dyla2007/outline.txt Sat May 26 16:46:15 2007 @@ -122,6 +122,8 @@ * might open the door to better solutions for interoperability (high-level bridges instead of low-level ones, cross-translation...) +in an appropriate place above: + - writing a translation toolchain is easier and simpler than writing an OO VM Conclusion ========== From cfbolz at codespeak.net Sat May 26 17:22:47 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 26 May 2007 17:22:47 +0200 (CEST) Subject: [pypy-svn] r43682 - pypy/extradoc/talk/dyla2007 Message-ID: <20070526152247.905D780C3@code0.codespeak.net> Author: cfbolz Date: Sat May 26 17:22:46 2007 New Revision: 43682 Modified: pypy/extradoc/talk/dyla2007/section2.txt Log: some improvements Modified: pypy/extradoc/talk/dyla2007/section2.txt ============================================================================== --- pypy/extradoc/talk/dyla2007/section2.txt (original) +++ pypy/extradoc/talk/dyla2007/section2.txt Sat May 26 17:22:46 2007 @@ -36,60 +36,60 @@ On the other hand the use of C comes with a set of problems. Since C has no ways to do automatic memory management and since the implemented languages are garbage collected, the VM needs to contain a garbage collector. Implementing a -high-performance GC is not a simple task and so many implementations resort to +high-performance GC is a tedious task and so many implementations resort to relatively simple algorithms like mark-and-sweep or reference counting. Also the choice of garbage collector is fixed relatively early in the life of the project and then manifests itself throughout the complete VM source code. Thus it becomes extremely tedious to change it later. -A similar problem is the threading model the VM uses. XXX expand a bit +A similar problem is the threading model the VM uses - simpler threading models - don't make single-threaded apps slower due to locks - global locks - green threading - - C-level implementations of dyn language VMs have issues and trade-offs: - - reduced flexibility - - pressure to keep the implementation simple - - wrapping with C libraries simple but tedious -These listed limitations of a C-based implementation often lead to more +These limitations of a C-based implementation often lead to more implementations of the language. For example in addition to the main C -implementation of Python ("CPython") there is a number of additional -implementations that want to fix some of the limitations of CPython. One of them -is "Stackless Python" which is a fork of CPython that adds micro-threading -capabilities to Python. One of the reasons for not incorporating back into +implementation of Python ("CPython") there are a number of additional +implementations that want to fix some of the limitations of CPython. One of +them is "Stackless Python" which is a fork of CPython that adds micro-threading +capabilities to Python. One of the reasons for not incorporating it back into CPython was that it was felt that they would make the implementation too complex. Another implementation of the Python language is the Psyco project -which adds a JIT-compiler to CPython. As all other reimplementations, Psyco +which adds a JIT-compiler to CPython. As all other re-implementations, Psyco contains the Python semantics and needs to be kept synchronized with CPython manually. - free threading attempts for Python As a language becomes more popular, yet another reason for different -implementations emerge: People want to use it together with Java and C# +implementations emerges: People want to use it together with Java and C# libraries and tools. Although it is in principle possible to do that by bridging the C-VM with the respective VMs of these languages (XXX examples) -this is an incomplete solution since it prevents many useful use-cases. +this is an incomplete solution since it prevents many use-cases. Therefore in the end implementations of the dynamic language on top of the object-oriented VMs are created to properly integrate with the complete -environment. +environment. Implementing a language on top of an OO VM is in many ways easier +than implementing it in C. Assessing the Advantages of Implementing a Language on Top of OO VMs -------------------------------------------------------------------- -Let's now take a look at the advantages that are usually cited for basing a -language implementation of a dynamic language on a standard object oriented virtual -machine, for example the JVM or the CLR: +Let's take a look at the advantages that are usually cited for basing a +language implementation of a dynamic language on a standard object oriented +virtual machine, for example the JVM or the CLR. The central theme of the +benefits of OO VMs is the ability to implement certain hard things only once +and share the benefits between all language implementations on top of the OO VM. *Better interoperability than the C level:* Since the VM offers a standard object model and all the languages implemented on top of it are using it, it -is easier to integrate the languages on top of the VM. This allows reuse of -libraries between all the implemented languages. This is typically the most -important reason for wanting an implementation on the VM in the first place. +is easier to integrate the languages that are running on top of the VM. This +allows reuse of libraries between all the implemented languages. This is +typically the most important reason for wanting an implementation on the VM in +the first place. *Cross-platform portability:* Only the underlying VM has to be ported to various hardware architectures and operating systems. The languages @@ -98,19 +98,27 @@ *Better tools:* XXX *Better implementation of low-level issues like garbage collection, threading:* -XXX - -*Better performance:* Object-oriented VMs usually come with a highly tuned -just-in-time compiler to make them perform well without requiring ahead-of-time -compilation to machine language. This in addition with the previous point leads -to much better performance of the languages running on top of the VM. +Since an OO VM is expected to be widely used and usually backed by a company, it +becomes worthwhile and possible to spend a lot of effort tuning its garbage +collector, threading model, exception support and other low-level implementation +details. + +*Better performance:* Similarly, object-oriented VMs usually come with a highly +tuned just-in-time compiler to make them perform well without requiring +ahead-of-time compilation to machine language. This in addition with the +previous point leads to much better performance of the languages running on top +of the VM. *Ease of implementation:* The implementation of a language on top of an OO VM is easier because it starts of at a higher level than C. Usually a high-level language like Java or C# is used for the language implementation, which both -offer the language implementer a much higher level of abstraction. XXX expand +offer the language implementer a much higher level of abstraction than when +implementing in C. -*A single unified implementation base:* XXX +*A single unified implementation base:* The higher level of abstraction +mentioned in the last point also helps to reduce the number of implementations +of the language. Since the implementation on top of an OO VM is more abstract it +is also easier to modify and to extend. XXX somewhat vague point At a closer look, some of these advantages are only partially true in practice. @@ -123,7 +131,8 @@ dynamic behaviour and the complex dispatch semantics of dynamic languages. (XXX expand) -*Better GCs:* XXX large memory overhead of OO VMs +*Better GCs:* While this is obvious in theory, OO VMs tend to have a much higher +memory overhead to start with (XXX ref) *Cross-platform portability:* While this is true to some extend, the situation with regard to portability is not significantly improved compared to e.g. @@ -133,8 +142,9 @@ architecture too or working at all, which leads to significantly less performance. -*Ease of implementation:* This point is disputable. While it is true that OO VMs -typically allow the language implementor to start at a higher level, they also +*Ease of implementation:* This point is disputable. On the one hand, OO VMs +typically allow the language implementor to start at a higher level. On the +other hand they also enforce a specific object and execution model. This means that the concepts of the implemented language need to be mapped to the execution model of the underlying VM, which may be easy or not, depending very much on the language in @@ -145,7 +155,8 @@ they are not particular efficient, especially when compared to good Prolog VMs in written in C. This is mostly because the Prolog execution model, which involves backtracking and deep recursion does not fit the JVM and .NET very -well. XXX some more +well. Therefore the Prolog implementations on top of OO VMs resort to models +that is quite unnatural both for the OO VM and for Prolog. On the other hand some of the benefits are real and very useful, the most From cfbolz at codespeak.net Sat May 26 18:25:29 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 26 May 2007 18:25:29 +0200 (CEST) Subject: [pypy-svn] r43684 - pypy/extradoc/talk/dyla2007 Message-ID: <20070526162529.64D0D80CD@code0.codespeak.net> Author: cfbolz Date: Sat May 26 18:25:27 2007 New Revision: 43684 Modified: pypy/extradoc/talk/dyla2007/dyla.txt Log: attempt at rewriting the abstract Modified: pypy/extradoc/talk/dyla2007/dyla.txt ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.txt (original) +++ pypy/extradoc/talk/dyla2007/dyla.txt Sat May 26 18:25:27 2007 @@ -8,12 +8,15 @@ Dynamic languages are traditionally implemented by writing a virtual machine for them, centered around an interpreter and/or a built-in -compiler and providing the object model and memory management. A -relatively recent alternative to this approach is to run the dynamic -language on top of an existing, well-tuned object-oriented virtual -machine like Java's. In this paper, we outline the reasons for this -trend and explore its limitations. We contrast it with a different, -complementary alternative: flexibly generating virtual machines instead +compiler and providing the object model and memory management. When +a language becomes more successful, the limitations of such an implementation +lead to the emergence of alternative implementations that try to solve some +of the problems. Another reason for new implementations is the desire to have +the language integrate well with existing, well-tuned object-oriented virtual +machine like Java's. In this paper, we describe the mechanisms that lead to an +abundance of implementations, and explore some of the limitations of standard +VMs. We propose a different complementary alternative: flexibly generating +virtual machines instead of writing them by hand. This approach has been validated by the PyPy project, with which we can automatically insert many features into the generated virtual machines - including good just-in-time compilers tuned From cfbolz at codespeak.net Sat May 26 19:01:51 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 26 May 2007 19:01:51 +0200 (CEST) Subject: [pypy-svn] r43685 - pypy/extradoc/talk/dyla2007 Message-ID: <20070526170151.D4D1580D6@code0.codespeak.net> Author: cfbolz Date: Sat May 26 19:01:51 2007 New Revision: 43685 Modified: pypy/extradoc/talk/dyla2007/dyla.txt Log: attempt to radically shorten the introduction Modified: pypy/extradoc/talk/dyla2007/dyla.txt ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.txt (original) +++ pypy/extradoc/talk/dyla2007/dyla.txt Sat May 26 19:01:51 2007 @@ -39,56 +39,28 @@ The efforts required to build a new virtual machine are relatively important. This is particularly true for languages which are complex -and in constant evolution. This is a major reason for keeping the -corresponding virtual machines simple. The C implementation of Python, -for example, is a simple bytecode interpreter and provides no more -advanced memory management techniques than reference counting coupled -with a reference cycle detector. This implementation is manageable for -an Open Source community with a limited amount of resources. The same -constraint can be found in many academic projects. - -Limitations of C implementations are XXX (limited GC, limited speed, -C-only platforms, lack of flexibility). - -To address some of these issues, a recent trend is to build an -implementation of the language on top of another virtual machine. Java -(and increasingly .NET) provides well-tuned object-oriented virtual -machines which offer many features that are of interest to all -programmers including language implementors (a higher level of -expression, automatic memory management, good just-in-time compilers, -richer libraries...). They also provide a higher level "base object -model", which enables better cross-language integration: a Jython -program, for example, can directly import Java classes and vice-versa. - -This trend enters in conflict with the goal of having to maintain -essentially a single, simple enough implementation for a given -programming language: as the language becomes popular, there will be a -demand for having it run on various platforms - high-level VMs as well -as C-level environments. - -In the case of Python and Ruby, this resulted in multiple -implementations of the language, with an oldest custom VM written in C -which is the "official" version in the sense that its evolution defines -the evolution of the language itself. The other implementations - -Jython and JRuby on the JVM, IronPython and IronRuby on .NET - -ultimately need to be tediously kept up-to-date. - - -What is the best way to implement dynamic languages? ----------------------------------------------------- - -The question is thus how the increased benefits of reusing -object-oriented virtual machines (OO VMs) can be balanced against the -fact that this introduces new implementations which may split the total -effort of the community in pieces that get increasingly -resource-consuming to keep in sync with each other. - -Standardizing on one OO VM would further increase interoperability and -avoid the division-of-efforts issue. In our opinion, though, a single -OO VM for everybody is not a practical idea; we consider the loss of -diversity to be more a problem than a solution, and any single OO VM -comes with limitations and trade-offs that may not be optimal for every -use case. +and in constant evolution. Language implementation communities from an +open-source or academic context have only limited resources. Therefore they +cannot afford to have a highly complex implementation and often chose simpler +techniques even if that entails lower execution speed. Similarly fragmentation +(for example because of other implementations of the same language) is a +problem because it divides available resources. All these points also apply to +the implementation of domain-specific-languages where it is important to keep +implementation effort small. + +For these reasons writing a virtual machine in C has many problems because it +forces the language implementer to deal with many low-level details. Limitations +of the C implementation lead to alternative implementations which draw +work-power from the reference implementation. An alternative to writing +implementations in C is to build them on top of one of the newer object oriented +virtual machines ("OO VM") such as the JVM or the CLR. This is often wanted by +the community anyway, since it leads to better re-usability of libraries of +these platforms. However, if a C implementation existed before the +implementation of such a VM is started, this enters in conflict with the goal of +having to maintain essentially a single, simple enough implementation for a +given programming language: as the language becomes popular, there will be a +demand for having it run on various platforms - high-level VMs as well as +C-level environments. The argument we will make in the present paper is that it is possible to benefit from and integrate with OO VMs while keeping the dynamic @@ -99,7 +71,7 @@ translation toolchain we can then generate whole virtual machines from this specification - either full custom VMs for C-level operating systems, or layers on top of various OO VMs. In other words, -metaprogramming techniques can be used to successfully replace a +meta-programming techniques can be used to successfully replace a foreseeable one-VM-fits-all standardization attempt. The argument boils down to: VMs for dynamic languages should not be @@ -108,3 +80,8 @@ feasible in practice. Just as importantly, it also brings new insights and concrete benefits in term of flexibility and performance that go beyond the state of the art. + +In section XXX we will explore the way VMs are typically implemented in C and +some of the problems of this approach. In section XXX we will describe our +proposed meta-programming approach. XXX + From santagada at codespeak.net Sat May 26 19:09:27 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Sat, 26 May 2007 19:09:27 +0200 (CEST) Subject: [pypy-svn] r43686 - in pypy/dist/pypy/lang/js: . test test/ecma Message-ID: <20070526170927.9EAAE80DF@code0.codespeak.net> Author: santagada Date: Sat May 26 19:09:26 2007 New Revision: 43686 Modified: pypy/dist/pypy/lang/js/astbuilder.py pypy/dist/pypy/lang/js/interpreter.py pypy/dist/pypy/lang/js/js_interactive.py pypy/dist/pypy/lang/js/jsgrammar.txt pypy/dist/pypy/lang/js/jsobj.py pypy/dist/pypy/lang/js/jsparser.py pypy/dist/pypy/lang/js/operations.py pypy/dist/pypy/lang/js/test/ecma/shell.js pypy/dist/pypy/lang/js/test/test_interp.py Log: Everything needed to start running the ecma tests. ForIn With and their variants suported. Creating objects with the new construct also fixed. Modified: pypy/dist/pypy/lang/js/astbuilder.py ============================================================================== --- pypy/dist/pypy/lang/js/astbuilder.py (original) +++ pypy/dist/pypy/lang/js/astbuilder.py Sat May 26 19:09:26 2007 @@ -163,6 +163,7 @@ pos = self.get_pos(node) nodes = [self.dispatch(child) for child in node.children] return operations.VariableDeclList(pos, nodes) + visit_variabledeclarationlistnoin = visit_variabledeclarationlist def visit_propertynameandvalue(self, node): pos = self.get_pos(node) @@ -194,16 +195,15 @@ def visit_sourceelements(self, node): pos = self.get_pos(node) - self.varlists.append(set()) + self.varlists.append({}) self.funclists.append({}) nodes=[] for child in node.children: node = self.dispatch(child) if node is not None: nodes.append(node) - var_decl = self.varlists.pop() + var_decl = self.varlists.pop().keys() func_decl = self.funclists.pop() - print var_decl, func_decl return operations.SourceElements(pos, var_decl, func_decl, nodes) def functioncommon(self, node, declaration=True): @@ -231,12 +231,13 @@ def visit_variabledeclaration(self, node): pos = self.get_pos(node) identifier = self.dispatch(node.children[0]) - self.varlists[-1].add(identifier.get_literal()) + self.varlists[-1][identifier.get_literal()] = None if len(node.children) > 1: expr = self.dispatch(node.children[1]) else: expr = None return operations.VariableDeclaration(pos, identifier, expr) + visit_variabledeclarationnoin = visit_variabledeclaration def visit_expressionstatement(self, node): return self.dispatch(node.children[0]) @@ -297,6 +298,15 @@ update, i = self.get_next_expr(node, i) body, i = self.get_next_expr(node, i) return operations.For(pos, setup, condition, update, body) + visit_regularvarfor = visit_regularfor + + def visit_infor(self, node): + pos = self.get_pos(node) + left = self.dispatch(node.children[1]) + right = self.dispatch(node.children[2]) + body= self.dispatch(node.children[3]) + return operations.ForIn(pos, left, right, body) + visit_invarfor = visit_infor def get_next_expr(self, node, i): if isinstance(node.children[i], Symbol) and \ @@ -346,4 +356,10 @@ def visit_primaryexpression(self, node): pos = self.get_pos(node) return operations.This(pos, 'this') - \ No newline at end of file + + def visit_withstatement(self, node): + pos = self.get_pos(node) + identifier = self.dispatch(node.children[0]) + body = self.dispatch(node.children[1]) + return operations.With(pos, identifier, body) + \ No newline at end of file Modified: pypy/dist/pypy/lang/js/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/js/interpreter.py (original) +++ pypy/dist/pypy/lang/js/interpreter.py Sat May 26 19:09:26 2007 @@ -4,7 +4,7 @@ from pypy.lang.js.astbuilder import ASTBuilder from pypy.lang.js.operations import * from pypy.rlib.objectmodel import we_are_translated - +from pypy.rlib.streamio import open_file_as_stream def writer(x): print x @@ -14,20 +14,16 @@ temp_tree = parse(script_source) return astb.dispatch(temp_tree) -import os.path - def load_file(filename): - # NOT RPYTHON - base, ext = os.path.splitext(filename) - f = open(filename) - t = load_source(f.read()) + f = open_file_as_stream(filename) + t = load_source(f.readall()) f.close() return t def evaljs(ctx, args, this): if len(args) >= 1: - if isinstance(args[0],W_String): + if isinstance(args[0], W_String): code = args[0] else: return args[0] @@ -109,6 +105,10 @@ return W_String(args[0].ToString()) return W_String('') +def arrayjs(ctx, args, this): + return W_Array() + + def numberjs(ctx, args, this): if len(args) > 0: return W_Number(args[0].ToNumber()) @@ -162,7 +162,7 @@ #Global Properties w_Global.Put('Object', w_Object) w_Global.Put('Function', w_Function) - w_Global.Put('Array', W_Array()) + w_Global.Put('Array', W_Builtin(arrayjs, Class='Array')) w_Global.Put('version', W_Builtin(versionjs)) #Number Modified: pypy/dist/pypy/lang/js/js_interactive.py ============================================================================== --- pypy/dist/pypy/lang/js/js_interactive.py (original) +++ pypy/dist/pypy/lang/js/js_interactive.py Sat May 26 19:09:26 2007 @@ -147,13 +147,13 @@ res = self.interp.run(load_source("\n".join(self.lines))) # XXX we should catch more stuff here, like not implemented # and such - except (jsparser.JsSyntaxError, ThrowException), e: + except (jsparser.ParseError, ThrowException), e: e_info = sys.exc_info() if self.debug: import pdb pdb.post_mortem(e_info[2]) else: - if isinstance(e, jsparser.JsSyntaxError): + if isinstance(e, jsparser.ParseError): print "\nSyntax error!" elif isinstance(e, ThrowException): print e.exception.ToString() Modified: pypy/dist/pypy/lang/js/jsgrammar.txt ============================================================================== --- pypy/dist/pypy/lang/js/jsgrammar.txt (original) +++ pypy/dist/pypy/lang/js/jsgrammar.txt Sat May 26 19:09:26 2007 @@ -376,10 +376,10 @@ ; #identifier -variabledeclarationnoin : identifier initialisernoin? +variabledeclarationnoin : identifier >initialisernoinvariabledeclarationlistnoin< Modified: pypy/dist/pypy/lang/js/jsobj.py ============================================================================== --- pypy/dist/pypy/lang/js/jsobj.py (original) +++ pypy/dist/pypy/lang/js/jsobj.py Sat May 26 19:09:26 2007 @@ -263,6 +263,9 @@ def Call(self, ctx, args=[], this = None): return self.callfuncbi(ctx, args, this) + + def Construct(self, ctx, args=[]): + return self.callfuncbi(ctx, args, None) def type(self): return 'builtin' @@ -288,7 +291,7 @@ def arraycallbi(ctx, args, this): return W_Array() -class W_Array(W_Builtin): +class W_Array(W_PrimitiveObject): def __init__(self, ctx=None, Prototype=None, Class='Array', Value=w_Undefined, callfunc=None): W_PrimitiveObject.__init__(self, ctx, Prototype, Class, Value, callfunc) @@ -296,7 +299,6 @@ self.Put('toString', toString, de=True) self.Put('length', W_Number(0)) self.length = r_uint(0) - self.set_builtin_call(arraycallbi) def Construct(self, ctx, args=[]): return self @@ -328,7 +330,8 @@ return if index < self.length: return - self.length = index+1 + + self.length = index+1 self.propdict['length'].value = W_Number(index+1) return Modified: pypy/dist/pypy/lang/js/jsparser.py ============================================================================== --- pypy/dist/pypy/lang/js/jsparser.py (original) +++ pypy/dist/pypy/lang/js/jsparser.py Sat May 26 19:09:26 2007 @@ -14,4 +14,10 @@ parsef = make_parse_function(regexs, rules, eof=True) def parse(code): - return parsef(code).visit(ToAST())[0] + try: + t = parsef(code) + except ParseError: + code += ';' + t = parsef(code) + + return ToAST().transform(t) Modified: pypy/dist/pypy/lang/js/operations.py ============================================================================== --- pypy/dist/pypy/lang/js/operations.py (original) +++ pypy/dist/pypy/lang/js/operations.py Sat May 26 19:09:26 2007 @@ -127,19 +127,19 @@ if op == "=": val = v3 elif op == "*=": - val = Mult().mathop(ctx, v1.GetValue(), v3) + val = mult(ctx, v1.GetValue(), v3) elif op == "+=": - val = Plus().mathop(ctx, v1.GetValue(), v3) + val = plus(ctx, v1.GetValue(), v3) elif op == "/=": - val = Div().mathop(ctx, v1.GetValue(), v3) + val = division(ctx, v1.GetValue(), v3) elif op == "%=": - val = Mod().mathop(ctx, v1.GetValue(), v3) + val = mod(ctx, v1.GetValue(), v3) elif op == "&=": - val = BitwiseAnd().decision(ctx, v1.GetValue().ToInt32(), v3.ToInt32()) + val = W_Number(v1.GetValue().ToInt32() & v3.ToInt32()) elif op == "|=": - val = BitwiseOR().decision(ctx, v1.GetValue().ToInt32(), v3.ToInt32()) + val = W_Number(v1.GetValue().ToInt32() | v3.ToInt32()) elif op == "^=": - val = BitwiseXOR().decision(ctx, v1.GetValue().ToInt32(), v3.ToInt32()) + val = W_Number(v1.GetValue().ToInt32() ^ v3.ToInt32()) else: print op raise NotImplementedError() @@ -699,7 +699,6 @@ x = self.expr.eval(ctx).GetValue() if not isinstance(x, W_PrimitiveObject): raise TypeError() - return x.Construct(ctx=ctx) @@ -742,9 +741,6 @@ else: singlequote = True - if stop < 0: - print stop - raise JsSyntaxError() internalstring = string[1:stop] for c in internalstring: @@ -780,9 +776,6 @@ self.nodes = nodes def execute(self, ctx): - print self.var_decl - print ctx - print repr(ctx.variable) for varname in self.var_decl: ctx.variable.Put(varname, w_Undefined) for funcname, funccode in self.func_decl.items(): @@ -793,14 +786,13 @@ last = w_Undefined for node in self.nodes: last = node.execute(ctx) - print repr(ctx.variable) return last except Exception, e: if isinstance(e, ExecutionReturned) and e.type == 'return': raise else: # TODO: proper exception handling - sys.stderr.write("exception in line: %s, on: %s%s"%(node.pos.lineno, node, os.linesep)) + print "exception in line: %s, on: %s%s"%(node.pos.lineno, node, os.linesep) raise @@ -850,7 +842,7 @@ e = excpt if self.catchblock is not None: obj = W_Object() - obj.Put(self.catchparam.name, e.exception) + obj.Put(self.catchparam.get_literal(), e.exception) ctx.push_object(obj) tryresult = self.catchblock.execute(ctx) ctx.pop_object() @@ -913,14 +905,13 @@ class With(Statement): - opcode = "WITH" - - def __init__(self, pos, t): - self.object = get_obj(t, 'object') - self.body = get_obj(t, 'body') + def __init__(self, pos, identifier, body): + self.pos = pos + self.identifier = identifier + self.body = body def execute(self, ctx): - obj = self.object.eval(ctx).GetValue().ToObject() + obj = self.identifier.eval(ctx).GetValue().ToObject() ctx.push_object(obj) try: @@ -969,10 +960,10 @@ class ForIn(Statement): - def __init__(self, pos, t): - self.object = get_obj(t, 'object') - self.body = get_obj(t, 'body') - self.iterator = get_obj(t, 'iterator') + def __init__(self, pos, iterator, lobject, body): + self.iterator = iterator + self.object = lobject + self.body = body def execute(self, ctx): obj = self.object.eval(ctx).GetValue().ToObject() Modified: pypy/dist/pypy/lang/js/test/ecma/shell.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/shell.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/shell.js Sat May 26 19:09:26 2007 @@ -50,7 +50,7 @@ var PASSED = " PASSED!"; var FAILED = " FAILED! expected: "; -var DEBUG = false; +var DEBUG = true; var DESCRIPTION; var EXPECTED; Modified: pypy/dist/pypy/lang/js/test/test_interp.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_interp.py (original) +++ pypy/dist/pypy/lang/js/test/test_interp.py Sat May 26 19:09:26 2007 @@ -449,7 +449,6 @@ """, 'undefined') def test_forin(): - py.test.skip("not ready yet") assertp(""" var x = {a:5}; for(y in x){ @@ -464,7 +463,6 @@ yield assertv, "2 !== 2;", False def test_with(): - py.test.skip("not ready yet") assertp(""" var mock = {x:2}; var x=4; @@ -488,7 +486,6 @@ yield assertv, "2 | 3;", 3 def test_for_vararg(): - py.test.skip("not ready yet") assertp(""" for (var arg = "", i = 0; i < 2; i++) { print(i);} """, ['0', '1']) @@ -520,3 +517,22 @@ case 1: 15; break; default: 30; };""", 15) + +def test_inplace_assign(): + yield assertv, "x=1; x+=1; x;", 2 + yield assertv, "x=2; x*=2; x;", 4 + yield assertv, "x=2; x/=2; x;", 1 + yield assertv, "x=4; x%=2; x;", 0 + yield assertv, "x=2; x&=2; x;", 2 + yield assertv, "x=0; x|=1; x;", 1 + yield assertv, "x=2; x^=2; x;", 0 + +def test_twoarray(): + assertp(""" + a1 = new Array(); + a2 = new Array(); + a1[0] = 1; + print(a1[0]); + a2[0] = 2; + print(a1[0]); + """, ['1', '1']) From fijal at codespeak.net Sat May 26 20:07:10 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 26 May 2007 20:07:10 +0200 (CEST) Subject: [pypy-svn] r43687 - pypy/branch/kill-ctypes/pypy/rpython/lltypesystem Message-ID: <20070526180710.965E580E6@code0.codespeak.net> Author: fijal Date: Sat May 26 20:07:08 2007 New Revision: 43687 Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py Log: Slightly less hackish errno implementation Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py Sat May 26 20:07:08 2007 @@ -1,7 +1,21 @@ from pypy.rpython.lltypesystem import lltype from pypy.rpython.lltypesystem.lloperation import llop -from pypy.rlib.objectmodel import CDefinedIntSymbolic +from pypy.annotation.model import lltype_to_annotation +from pypy.rlib.objectmodel import Symbolic, CDefinedIntSymbolic + +class CConstant(Symbolic): + """ A C-level constant, maybe #define, rendered directly. + """ + def __init__(self, c_name, TP): + self.c_name = c_name + self.TP = TP + + def annotation(self): + return lltype_to_annotation(self.TP) + + def lltype(self): + return self.TP def llexternal(name, args, result, sources=[], includes=[]): ext_type = lltype.FuncType(args, result) @@ -22,8 +36,7 @@ c_fields = [('c_' + key, value) for key, value in fields] return lltype.Ptr(lltype.Struct(name, *c_fields, **kwds)) -# XXX slightly hackish -c_errno = CDefinedIntSymbolic('errno', default=-1) +c_errno = CConstant('errno', lltype.Signed) # char * CCHARP = lltype.Ptr(lltype.Array(lltype.Char, hints={'nolength': True})) From fijal at codespeak.net Sat May 26 20:07:36 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 26 May 2007 20:07:36 +0200 (CEST) Subject: [pypy-svn] r43688 - pypy/branch/kill-ctypes/pypy/translator/c Message-ID: <20070526180736.0BCCC80EB@code0.codespeak.net> Author: fijal Date: Sat May 26 20:07:35 2007 New Revision: 43688 Modified: pypy/branch/kill-ctypes/pypy/translator/c/primitive.py Log: Support for CConstant Modified: pypy/branch/kill-ctypes/pypy/translator/c/primitive.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/translator/c/primitive.py (original) +++ pypy/branch/kill-ctypes/pypy/translator/c/primitive.py Sat May 26 20:07:35 2007 @@ -1,6 +1,7 @@ import sys from pypy.rlib.objectmodel import Symbolic, ComputedIntSymbolic from pypy.rlib.objectmodel import CDefinedIntSymbolic +from pypy.rpython.lltypesystem.rffi import CConstant from pypy.rpython.lltypesystem.lltype import * from pypy.rpython.lltypesystem.llmemory import Address, \ AddressOffset, ItemOffset, ArrayItemsOffset, FieldOffset, \ @@ -47,6 +48,8 @@ return str(value.expr) elif isinstance(value, ComputedIntSymbolic): value = value.compute_fn() + elif isinstance(value, CConstant): + return value.c_name else: raise Exception("unimplemented symbolic %r"%value) if value is None: From fijal at codespeak.net Sat May 26 20:14:02 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 26 May 2007 20:14:02 +0200 (CEST) Subject: [pypy-svn] r43689 - in pypy/dist/pypy: annotation rlib Message-ID: <20070526181402.6CBBD80D4@code0.codespeak.net> Author: fijal Date: Sat May 26 20:14:02 2007 New Revision: 43689 Modified: pypy/dist/pypy/annotation/bookkeeper.py pypy/dist/pypy/rlib/nonconst.py Log: SLAM! annotation_from_example goes away. Modified: pypy/dist/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/dist/pypy/annotation/bookkeeper.py (original) +++ pypy/dist/pypy/annotation/bookkeeper.py Sat May 26 20:14:02 2007 @@ -344,7 +344,7 @@ else: listdef = ListDef(self, s_ImpossibleValue) for e in x: - listdef.generalize(self.annotation_from_example(e)) + listdef.generalize(self.immutablevalue(e, False)) result = SomeList(listdef) elif tp is dict or tp is r_dict: if need_const: @@ -385,8 +385,8 @@ dictdef.dictkey.update_rdict_annotations(s_eqfn, s_hashfn) for ek, ev in x.iteritems(): - dictdef.generalize_key(self.annotation_from_example(ek)) - dictdef.generalize_value(self.annotation_from_example(ev)) + dictdef.generalize_key(self.immutablevalue(ek, False)) + dictdef.generalize_value(self.immutablevalue(ev, False)) result = SomeDict(dictdef) elif ishashable(x) and x in BUILTIN_ANALYZERS: _module = getattr(x,"__module__","unknown") @@ -449,10 +449,6 @@ result.const = x return result - def annotation_from_example(self, x): - # XXX to kill at some point - return self.immutablevalue(x, False) - def getdesc(self, pyobj): # get the XxxDesc wrapper for the given Python object, which must be # one of: Modified: pypy/dist/pypy/rlib/nonconst.py ============================================================================== --- pypy/dist/pypy/rlib/nonconst.py (original) +++ pypy/dist/pypy/rlib/nonconst.py Sat May 26 20:14:02 2007 @@ -22,7 +22,7 @@ def compute_result_annotation(self, arg): if hasattr(arg, 'const'): - return self.bookkeeper.annotation_from_example(arg.const) + return self.bookkeeper.immutablevalue(arg.const, False) else: return arg From cfbolz at codespeak.net Sat May 26 23:38:21 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sat, 26 May 2007 23:38:21 +0200 (CEST) Subject: [pypy-svn] r43701 - pypy/extradoc/talk/dyla2007 Message-ID: <20070526213821.6924180AA@code0.codespeak.net> Author: cfbolz Date: Sat May 26 23:38:20 2007 New Revision: 43701 Added: pypy/extradoc/talk/dyla2007/Makefile pypy/extradoc/talk/dyla2007/dyla.bib pypy/extradoc/talk/dyla2007/dyla.tex pypy/extradoc/talk/dyla2007/llncs.cls Removed: pypy/extradoc/talk/dyla2007/dyla.txt pypy/extradoc/talk/dyla2007/section2.txt pypy/extradoc/talk/dyla2007/section3.txt Log: latexify what is there of the paper Added: pypy/extradoc/talk/dyla2007/Makefile ============================================================================== --- (empty file) +++ pypy/extradoc/talk/dyla2007/Makefile Sat May 26 23:38:20 2007 @@ -0,0 +1,22 @@ +prolog-in-rpython.pdf: padl.tex padl.bib stack.pdf graph.png stackless.pdf memory2.pdf + pdflatex padl + bibtex padl + pdflatex padl + pdflatex padl + mv padl.pdf prolog-in-rpython.pdf + +dyla.pdf: dyla.tex dyla.bib + pdflatex dyla + bibtex dyla + pdflatex dyla + pdflatex dyla + +%.pdf: %.eps + epstopdf $< + +view: dyla.pdf + xpdf dyla.pdf & + +clean: + rm dyla.pdf + Added: pypy/extradoc/talk/dyla2007/dyla.bib ============================================================================== --- (empty file) +++ pypy/extradoc/talk/dyla2007/dyla.bib Sat May 26 23:38:20 2007 @@ -0,0 +1,245 @@ +% Psyco + + at inproceedings{ psyco, + author = "Armin Rigo", + title = "Representation-based just-in-time specialization and the psyco prototype for python", + booktitle = "PEPM '04: Proceedings of the 2004 ACM SIGPLAN symposium on Partial evaluation and semantics-based program manipulation", + year = "2004", + isbn = "1-58113-835-0", + pages = "15--26", + location = "Verona, Italy", + doi = "http://doi.acm.org/10.1145/1014007.1014010", + publisher = "ACM Press", + address = "New York, NY, USA" +} + +% P\# + + at inproceedings{ psharp, + author = "Jon Cook", + title = "P\#: A concurrent {P}rolog for the .NET Framework", + booktitle = "Software: Practice and Experience 34(9)", + year = "2004", + pages = "815--845", + publisher = "John Wiley \& Sons, Ltd" +} + +% Java Cafe + + at phdthesis{ prologcafe, + author = "Mutsunori Banbara", + title = "Design and Implementation of Linear Logic Programming Languages", + school = "The Graduate School of Science and Technology of Kobe University", + year = "2002", + month = "September" +} + +% PyPy doc and reports + at misc{ pypy, + title = {{PyPy. An Implementation of Python in Python}}, + note = "http://codespeak.net/pypy" +} + + at techreport{ dynamiclang, + title = "Compiling Dynamic Language Implementations", + author = "Armin Rigo and Michael Hudson and Samuele Pedroni", + year = "2005", + institution = "PyPy Consortium", + url = "http://codespeak.net/svn/pypy/extradoc/eu-report/D05.1_Publish_on_translating_a_very-high-level_description.pdf", + note = "http://codespeak.net/svn/pypy/extradoc/eu-report/" +} + + at techreport{ completepythonimpl, + title = "Complete Python Implementation running on top of CPython", + author = "{PyPy Team}", + year = "2005", + institution = "PyPy Consortium", + url = "http://codespeak.net/svn/pypy/extradoc/eu-report/D04.2_Complete_Python_Implementation_on_top_of_CPython.pdf", + note = "http://codespeak.net/pypy/dist/pypy/doc/index-report.html" +} + + at techreport{ translationaspects, + title = "Memory management and threading models as translation aspects -- solutions and challenges", + author = "Carl Friedrich Bolz and Armin Rigo", + year = "2005", + institution = "PyPy Consortium", + url = "http://codespeak.net/svn/pypy/extradoc/eu-report/D05.3_Publish_on_implementation_with_translation_aspects.pdf", + note = "http://codespeak.net/pypy/dist/pypy/doc/index-report.html" +} + +# XXX fix links and title + at techreport{ optimizations, + title = "DRAFT: Support for Massive Parallelism, Optimisation results, Practical Usages and Approaches for Translation Aspects", + author = "Carl Friedrich Bolz and Armin Rigo", + year = "2006", + institution = "PyPy Consortium", + url = "http://codespeak.net/svn/pypy/extradoc/eu-report/D07.1_Massive_Parallelism_and_Translation_Aspects-interim-2006-07-25.pdf", + note = "Interim version at: http://codespeak.net/pypy/dist/pypy/doc/index-report.html" +} + + at misc{ translationdoc, + title = "{PyPy - Translation}", + author = "{PyPy Team}", + url = "http://codespeak.net/pypy/dist/pypy/doc/translation.html", + note = "http://codespeak.net/pypy/dist/pypy/doc/trans-lation.html", + howpublished = "Web page, in-progress." +} + + at misc{ architecture, + title = "{PyPy - Architecture Overview}", + author = "{PyPy Team}", + url = "http://codespeak.net/pypy/dist/pypy/doc/architecture.html", + note = "http://codespeak.net/pypy/dist/pypy/doc/archi-tecture.html", + howpublished = "Web page" +} + + at misc{ cpy243, + author = "Guido van Rossum et al.", + title = "{CPython} 2.4.3", + note = "http://www.python.org/download/releases/2.4.3/", + url = "http://www.python.org/download/releases/2.4.3/", + year = 2006, + month = "March" +} + + at misc{ pylangref, + author = "Guido van Rossum et al.", + title = "Python language reference", + note = "http://docs.python.org/ref/ref.html", + url = "http://docs.python.org/ref/ref.html", + year = 2006 +} + + at InProceedings{ LLVM:CGO04, + author = "Chris Lattner and Vikram Adve", + title = "{LLVM: A Compilation Framework for Lifelong Program Analysis \& Transformation}", + booktitle = "{Proceedings of the 2004 International Symposium on Code Generation and Optimization (CGO'04)}", + address = "Palo Alto, California", + month = "Mar", + year = "2004" +} + + + at inproceedings{DBLP:conf/jelia/Calejo04, + author = {Miguel Calejo}, + title = {Inter{P}rolog: Towards a Declarative Embedding of Logic Programming + in {J}ava.}, + booktitle = {JELIA}, + year = {2004}, + pages = {714-717}, + ee = {http://springerlink.metapress.com/openurl.asp?genre=article{\&}issn=0302-9743{\&}volume=3229{\&}spage=714}, + crossref = {DBLP:conf/jelia/2004}, + bibsource = {DBLP, http://dblp.uni-trier.de} +} + +% Boehm + at article{Boehm, + author = {Hans-Juergen Boehm and Mark Weiser}, + title = {Garbage collection in an uncooperative environment}, + journal = {Softw. Pract. Exper.}, + volume = {18}, + number = {9}, + year = {1988}, + issn = {0038-0644}, + pages = {807--820}, + publisher = {John Wiley \& Sons, Inc.}, + address = {New York, NY, USA}, + } + + at misc{boehm-softw, +author = "Hans Boehm", +title = "A garbage collector for {C} and {C++}", +url = "http://www.hpl.hp.com/personal/Hans_Boehm/gc/", +note = "http://www.hpl.hp.com/personal/Hans\_Boehm/gc/" +} + + at misc{CLI, + author="ECMA", + title="Standard ECMA-335: Common Language Infrastructure", + publisher="ECMA International", + year = "2005", + month = "June", + note = "http://www.ecma-international.org/publications/standards/Ecma-335.htm", +} + + at inproceedings{pypyvmconstruction, + title = {Py{P}y's Approach to Virtual Machine Construction}, + author = {Armin Rigo and Samuele Pedroni}, + booktitle = {Proceedings of the 2006 conference on Dynamic languages + symposium}, + year = 2006, + note= "To appear, accepted for publication" +} + + at inproceedings{Squeak, + author = {Dan Ingalls and Ted Kaehler and John Maloney and Scott Wallace and + Alan Kay}, + title = {Back to the future: the story of {Squeak}, a practical {Smalltalk} written in itself}, + booktitle = {OOPSLA '97: Proceedings of the 12th ACM SIGPLAN conference on Object-oriented programming, systems, languages, and applications}, + year = {1997}, + isbn = {0-89791-908-4}, + pages = {318--326}, + location = {Atlanta, Georgia, United States}, + doi = {http://doi.acm.org/10.1145/263698.263754}, + publisher = {ACM Press}, + address = {New York, NY, USA}, + } + + at misc{kelsey-prescheme, + author = "R. Kelsey", + title = "Pre-Scheme: A Scheme Dialect for Systems Programming", + text = "Richard Kelsey. Pre-Scheme: A Scheme Dialect for Systems Programming. ?." } + + at misc{taivalsaari98implementing, + author = "A. Taivalsaari", + title = "Implementing a Java virtual machine in the java programming language", + text = "Antero Taivalsaari. Implementing a Java virtual machine in the java programming + language. Technical Report SMLI TR-98-64, Sun Microsystems, March 1998.", + year = "1998", + url = "citeseer.ist.psu.edu/taivalsaari98implementing.html" } + + at article{jalapeno, + author = {Bowen Alpern and + C. Richard Attanasio and + John J. Barton and + Michael G. Burke and + Perry Cheng and + Jong-Deok Choi and + Anthony Cocchi and + Stephen J. Fink and + David Grove and + Michael Hind and + Susan Flynn Hummel and + Derek Lieber and + Vassily Litvinov and + Mark F. Mergen and + Ton Ngo and + James R. Russell and + Vivek Sarkar and + Mauricio J. Serrano and + Janice C. Shepherd and + Stephen E. Smith and + Vugranam C. Sreedhar and + Harini Srinivasan and + John Whaley}, + title = {The {Jalapeno} virtual machine.}, + journal = {IBM Systems Journal}, + volume = {39}, + number = {1}, + year = {2000}, + pages = {211-}, + ee = {http://www.research.ibm.com/journal/sj/391/alpern.html}, + bibsource = {DBLP, http://dblp.uni-trier.de} +} + +% Jikes jit, more recent techreport + at techreport{Jikes-JIT, + title = {Architecture and Policy for Adaptive Optimization in Virtual Machines}, + author = {Matthew Arnold and Stephen Fink and David Grove and Michael Hind and Peter F. Sweeney}, + year = 2004, + institution = {IBM Research}, + number = 23429, + month = "November", +} + + Added: pypy/extradoc/talk/dyla2007/dyla.tex ============================================================================== --- (empty file) +++ pypy/extradoc/talk/dyla2007/dyla.tex Sat May 26 23:38:20 2007 @@ -0,0 +1,446 @@ +\documentclass{llncs} + +\usepackage{makeidx} +\usepackage{graphicx} + +\begin{document} + +\pagestyle{headings} + +\title{How to Not write Virtual Machines for Dynamic Languages} + +%\titlerunning{XXX} % abbreviated title (for running head) +% also used for the TOC unless +% \toctitle is used + +\author{Carl Friedrich Bolz and Armin Rigo} + +\authorrunning{Bolz and Rigo} % abbreviated author list (for running head) + +%%%% modified list of authors for the TOC (add the affiliations) +\tocauthor{Bolz and Rigo (D\"usseldorf)} + +\institute{ Lehrstuhl Softwaretechnik und Programmiersprachen\\ +Institut f\"{u}r Informatik, Universit\"at D\"usseldorf, Germany\\ + \email{ cfbolz at gmx.de, arigo at tunes.org}} + +\maketitle + +\begin{abstract} + +Dynamic languages are traditionally implemented by writing a virtual +machine for them, centered around an interpreter and/or a built-in +compiler and providing the object model and memory management. When +a language becomes more successful, the limitations of such an implementation +lead to the emergence of alternative implementations that try to solve some +of the problems. Another reason for new implementations is the desire to have +the language integrate well with existing, well-tuned object-oriented virtual +machine like Java's. In this paper, we describe the mechanisms that lead to an +abundance of implementations, and explore some of the limitations of standard +VMs. We propose a different complementary alternative: flexibly generating +virtual machines instead +of writing them by hand. This approach has been validated by the PyPy +project, with which we can automatically insert many features into the +generated virtual machines - including good just-in-time compilers tuned +to the dynamic language at hand. +\footnote{This research is partially supported by the EU funded %research + project: +IST 004779 PyPy (PyPy: Implementing Python in Python).} \\ + +\end{abstract} + +\section{Introduction} + +Dynamic languages are traditionally implemented by writing a virtual +machine for them in a low-level language like C, or in a language that +can relatively easily be turned into C. The machine implements an +object model supporting the high level dynamic language's objects. It +typically provides features like automatic garbage collection. Recent +languages like Python, Ruby, Perl and JavaScript have complicated +semantics which are most easily mapped to a simple interpreter operating +on syntax trees or bytecode; simpler languages like Lisp and Self +typically have more efficient implementations based on just-in-time code +generation. + +The efforts required to build a new virtual machine are relatively +important. This is particularly true for languages which are complex +and in constant evolution. Language implementation communities from an +open-source or academic context have only limited resources. Therefore they +cannot afford to have a highly complex implementation and often chose simpler +techniques even if that entails lower execution speed. Similarly fragmentation +(for example because of other implementations of the same language) is a +problem because it divides available resources. All these points also apply to +the implementation of domain-specific-languages where it is important to keep +implementation effort small. + +For these reasons writing a virtual machine in C has many problems because it +forces the language implementer to deal with many low-level details. Limitations +of the C implementation lead to alternative implementations which draw +work-power from the reference implementation. An alternative to writing +implementations in C is to build them on top of one of the newer object oriented +virtual machines ("OO VM") such as the JVM or the CLR. This is often wanted by +the community anyway, since it leads to better re-usability of libraries of +these platforms. However, if a C implementation existed before the +implementation of such a VM is started, this enters in conflict with the goal of +having to maintain essentially a single, simple enough implementation for a +given programming language: as the language becomes popular, there will be a +demand for having it run on various platforms - high-level VMs as well as +C-level environments. + +The argument we will make in the present paper is that it is possible to +benefit from and integrate with OO VMs while keeping the dynamic +language implemented by a single, simple source code base. The idea is +to write an interpreter for that language in another sufficiently +high-level but less dynamic language. This interpreter plays the role +of a specification for the dynamic language. With a good enough +translation toolchain we can then generate whole virtual machines from +this specification - either full custom VMs for C-level operating +systems, or layers on top of various OO VMs. In other words, +meta-programming techniques can be used to successfully replace a +foreseeable one-VM-fits-all standardization attempt. + +The argument boils down to: VMs for dynamic languages should not be +written by hand! The justification is based on the +PyPy project, which proves that the approach is +feasible in practice. Just as importantly, it also brings new insights +and concrete benefits in term of flexibility and performance that go +beyond the state of the art. + +In section XXX we will explore the way VMs are typically implemented +in C and some of the problems of this approach. In section XXX we will describe +our proposed meta-programming approach. XXX + + +\section{XXX XXX} + +Dynamic languages are traditionally implemented by writing a virtual +machine for them in a low-level language like C, or in a language that +can relatively easily be turned into C. The machine implements an +object model supporting the high level dynamic language's objects. It +typically provides features like automatic garbage collection. Recent +languages like Python, Ruby, Perl and JavaScript have complicated +semantics which are most easily mapped to a simple interpreter operating +on syntax trees or bytecode. Simpler languages like Lisp and Self +typically have more efficient implementations based on just-in-time code +generation. + +There are many good reasons for choosing C as an implementation language. It +gives a decent base performance, is highly portable across different hardware +architectures and operating system if some care is taken and it allows the reuse +of many existing libraries. + +On the other hand the use of C comes with a set of problems. Since C has no ways +to do automatic memory management and since the implemented languages are +garbage collected, the VM needs to contain a garbage collector. Implementing a +high-performance GC is a tedious task and so many implementations resort to +relatively simple algorithms like mark-and-sweep or reference counting. Also the +choice of garbage collector is fixed relatively early in the life of the project +and then manifests itself throughout the complete VM source code. Thus it +becomes extremely tedious to change it later. + +%A similar problem is the threading model the VM uses +% - simpler threading models +% - don't make single-threaded apps slower due to locks +% - global locks +% - green threading + +% - wrapping with C libraries simple but tedious + +These limitations of a C-based implementation often lead to more +implementations of the language. For example in addition to the main C +implementation of Python ("CPython") there are a number of additional +implementations that want to fix some of the limitations of CPython. One of +them is "Stackless Python" which is a fork of CPython that adds micro-threading +capabilities to Python. One of the reasons for not incorporating it back into +CPython was that it was felt that they would make the implementation too +complex. Another implementation of the Python language is the Psyco project +which adds a JIT-compiler to CPython. As all other re-implementations, Psyco +contains the Python semantics and needs to be kept synchronized with CPython +manually. + +% - free threading attempts for Python + +As a language becomes more popular, yet another reason for different +implementations emerges: People want to use it together with Java and C\# +libraries and tools. Although it is in principle possible to do that by +bridging the C-VM with the respective VMs of these languages (XXX examples) +this is an incomplete solution since it prevents many use-cases. +Therefore in the end implementations of the dynamic language on top of the +object-oriented VMs are created to properly integrate with the complete +environment. Implementing a language on top of an OO VM is in many ways easier +than implementing it in C. + + +\subsection{Assessing the Advantages of Implementing a Language on Top of OO +VMs} + +Let's take a look at the advantages that are usually cited for basing a +language implementation of a dynamic language on a standard object oriented +virtual machine, for example the JVM or the CLR. The central theme of the +benefits of OO VMs is the ability to implement certain hard things only once +and share the benefits between all language implementations on top of the OO VM. + +\emph{Better interoperability than the C level:} Since the VM offers a standard +object model and all the languages implemented on top of it are using it, it is +easier to integrate the languages that are running on top of the VM. This +allows reuse of libraries between all the implemented languages. This is +typically the most important reason for wanting an implementation on the VM in +the first place. + +\emph{Cross-platform portability:} Only the underlying VM has to be ported to +various hardware architectures and operating systems. The languages implemented +on top can then be run without change in various environments. + +\emph{Better tools:} XXX + +\emph{Better implementation of low-level issues like garbage collection, +threading:} Since an OO VM is expected to be widely used and usually backed by +a company, it becomes worthwhile and possible to spend a lot of effort tuning +its garbage collector, threading model, exception support and other low-level +implementation details. + +\emph{Better performance:} Similarly, object-oriented VMs usually come with a +highly tuned just-in-time compiler to make them perform well without requiring +ahead-of-time compilation to machine language. This in addition with the +previous point leads to much better performance of the languages running on top +of the VM. + +\emph{Ease of implementation:} The implementation of a language on top of an OO +VM is easier because it starts of at a higher level than C. Usually a +high-level language like Java or C\# is used for the language implementation, +which both offer the language implementer a much higher level of abstraction +than when implementing in C. + +\emph{A single unified implementation base:} The higher level of abstraction +mentioned in the last point also helps to reduce the number of implementations +of the language. Since the implementation on top of an OO VM is more abstract +it is also easier to modify and to extend. XXX somewhat vague point + + +At a closer look, some of these advantages are only partially true in practice. + +\emph{Better performance:} So far it seems like performance of highly dynamic +languages is not actually significantly improved on OO VMs. Jython is around 5 +times slower than CPython, for IronPython the figures vary but it is mostly +within the same order of magnitude as CPython. The most important reason for +this that the VM's JIT compilers are not prepared to deal with the highly +dynamic behaviour and the complex dispatch semantics of dynamic languages. (XXX +expand) + +\emph{Better GCs:} While this is obvious in theory, OO VMs tend to have a much +higher memory overhead to start with (XXX ref) + +\emph{Cross-platform portability:} While this is true to some extend, the +situation with regard to portability is not significantly improved compared to +e.g. C/Posix, which is relatively portable too. Also portability sometimes +comes at the price of performance, because even if the OO VM is running on a +particular hardware architecture it is not clear that the JIT is tuned for this +architecture too or working at all, which leads to significantly less +performance. + +\emph{Ease of implementation:} This point is disputable. On the one hand, OO +VMs typically allow the language implementor to start at a higher level. On the +other hand they also enforce a specific object and execution model. This means +that the concepts of the implemented language need to be mapped to the +execution model of the underlying VM, which may be easy or not, depending very +much on the language in question. XXX give example where it works well. + +An example where this mapping does not work too well is Prolog. While there +exist several implementations of Prolog on top of the JVM and also one on .NET, +they are not particular efficient, especially when compared to good Prolog VMs +in written in C. This is mostly because the Prolog execution model, which +involves backtracking and deep recursion does not fit the JVM and .NET very +well. Therefore the Prolog implementations on top of OO VMs resort to models +that is quite unnatural both for the OO VM and for Prolog. + + +On the other hand some of the benefits are real and very useful, the most +prominent being the easy interaction with the rest of the VM. Furthermore there +is better tool support and better GCs. XXX expand + + +\subsection{The Cost of Implementation-Proliferation} + +The described proliferation of language implementations is a big problem for +language communities. Although most individual implementations exist for good +reasons, the sum of all of them and the need to keep them synchronized with the +reference implementations lead to a lot of duplicated work and division of +efforts. This is especially true for open source languages which tend to evolve +quickly. At any one point in time some of the implementations will lag behind +which makes writing code which can work on all of the implementations harder. + +Implementing a language on top of a OO VM has many advantages, so some +people propose the solution of standardizing on one particular OO VM to not have +to maintain implementations for several of them. While this would in theory +alleviate the problem it is unlikely to happen. On the one hand many political +issues are involved in such a decision. On the other hand deciding on one single +object and execution model would not be an equally good fit for all languages. + +In the next section we are exploring a different approach for implementing +dynamic languages that we hope is able to solve many of the problems of +implementing a language, in particular the problem of an explosion of the number +of implementations. + +\section{Meta-Programming Is Good} + +The present paper proposes to approach the implementation of dynamic +languages from a meta-level: virtual machines for such languages should +not be written by hand, but generated automatically "around" a +description of the language in the form of an interpreter for it. We +argue that this approach gives many of the benefits usually expected by +an implementer when he decides to target an existing object-oriented +virtual machine. It also gives other benefits that we will describe - +mostly in term of flexibility. But most importantly, it lets a +community write a single source implementation of the language, avoiding +the time-consuming task of keeping multiple ones in sync. The single +source can be used to generate either custom VMs for C-like +environments, or interpreters running on top of OO VMs. It makes it +practical to experiment with large changes to the language and with +entirely new languages, like domain-specific languages, while at any +time being able to run the implemented language in a variety of +environments, from C/Posix to the JVM to .NET. + +\subsection{PyPy architecture} + +We implemented this idea in the PyPy project. The dynamic language for +which we wrote an interpreter is Python. It is a language which, +because of its size and rather intricate semantics, is a good target for +our approach, in the following sense: its previous reimplementation +(Jython for the JVM and IronPython for .NET) have each proved to be very +time-consuming to maintain. Our implementation is by construction +easier to maintain, and extremely portable (including to C/Posix, to the +JVM and to .NET). + +In meta-programming terms, the PyPy architecture is as follows: + +\begin{itemize} + +\item +we use a very expressive \emph{object language} (RPython - an analyzable +subset of Python) as the language in which the complete Python +interpreter is written, together with the implementation of its +built-in types. The language is still close to Python, e.g. it is +object-oriented, provides rich built-in types and has automatic memory +management. In other words, the source code of our complete Python +interpreter is mostly free of low-level details - no explicit memory +management, no pieces of C or C-level code. + +\item +we use a very expressive metalanguage (namely regular Python) to +perform the analysis of RPython code (control flow and data flow +construction, type inference, etc.) and its successive +transformations. + +\item +this meta-programming component of PyPy is called the \emph{translation +framework,} as it translates RPython source code (i.e. the full Python +interpreter) into lower-level code. Its purpose is to add aspects to +and specialize the interpreter to fit a selectable virtual or hardware +runtime environment. This either turns the interpreter into a +standalone virtual machine, or integrates it into an existing OO VM. +The necessary support code - e.g. the garbage collector when +targeting C - is itself written in RPython in much the same spirit +that the Jikes RVM's GCs are written in Java [JIKES]; as needed, it is +translated together with the interpreter to form the final custom VM. +\end{itemize} + +A detailed description of this translation process is beyond the scope of the +present paper; it can be found in \cite{pypyvmconstruction}. The actual Python +interpreter of PyPy and the results we achieved by translating it to C, LLVM +\ref{LLVM:CGO04} and .NET are described in [XXX]. These results show that the +approach is practical and gives results whose performance is within the same +order of magnitude (within a factor of 2 and improving) of the hand-written, +well-tuned CPython, the C reference implementation. + +\subsection{A single source} + +Our approach - a single "meta-written" implementation - naturally leads +to language implementations that have various advantages over the +"hand-written" implementations. First of all, it is a single-source +approach - we explicitly seek to solve the problem of proliferation of +implementations. In the sequel we will show more precise evidence that +this can be done in a practical way with no major drawback. By itself +this would already be a valid argument against the need for +standardization on a single OO VM. But there are also other advantages +in generating language implementations. In our opinion these are very +significant, to the extent that it hints that meta-programming, though +not widely used in general-purpose programming, is an essential tool in +a language implementer's toolbox. + +\subsection{Writing the interpreter is easier} + +A first point is that it makes interpreters easy to write, update and +generally experiment with. More expressiveness helps at all levels: our +Python interpreter is written in RPython as a relatively simple +interpreter, in some respects easier to understand than CPython. We are +using its high level and flexibility to quickly experiment with features +or implementation techniques in ways that would, in a traditional +approach, require pervasive changes to the source code. For example, +PyPy's Python interpreter can optionally provide lazily computed objects +- a 150-lines extension in PyPy that would require global changes in +CPython. Further examples can be found in our technical reports; we +should notably mention an extension adding a state-of-the-art security +model for Python based on data flow tacking [XXX], and general +performance improvements found by extensive experimentation [XXX], some +of which were back-ported to CPython. + +If we compare with hand-writing an implementation for a specific OO VM, +then the latter requires not only good knowledge of the OO VM in +question and its object model - it requires the language implementer to +fit the language into the imposed models. For example, it is natural to +map classes and instances of a dynamic object-oriented language to the +OO VM's notion of classes and instances, but this might not be a simple +task at all if the models are substantially different and/or if the OO +VM is essentially less dynamic than the language to implement. In our +approach, this efforts is done at two levels: in a first step, while +writing the interpreter, the implementer does not need to worry about +integration with an OO VM's object model. Of course, the integration +effort does not simply vanish - indeed, a simple translation of such an +interpreter to a given OO VM would give an interpreter for the dynamic +language which is unable to communicate with the host VM (which might +already be interesting in specific cases, but not in general). +Integration comes as a second step, and occurs at a different level, by +introducing mappings between the relevant classes of the interpreter and +the corresponding classes of the OO VM. As of yet we have no evidence +that this makes the total integration effort much lower or higher; the +point is that it has proven possible to take the single, OO +VM-independent source code of PyPy's Python interpreter, and produce +from it a version that runs in and integrates with the rest of the .NET +environment - by writing \emph{orthogonal} code. + +At the level of the translation framework, the ability to change or +insert new whole-program transformations makes some aspects of the +interpreter easier to deal with. By "aspect" we mean, in the original +AOP sense, a feature that is added to an object program by a +meta-program. The most obvious example in our context is the garbage +collector for the target environments that lack it. The hand-written C +source of CPython is littered with macro calls that increment and +decrement reference counters. Our translation framework can insert such +macro calls automatically - in fact, we have a choice of GCs, and +reference counting is only one of them (not a particularly efficient +one, either). Some GCs require different transformations of the code. +By contrast, supporting more than one GC in CPython is close to +impossible without forking the whole source code base. + +In fact, there is a well-known example of a CPython fork: Stackless +Python [XXX], which adds support for coroutines. This is very hard to +do in CPython because the interpreter is written in a highly recursive +style. Stackless Python required large-scale changes; it is not merged +back into CPython due to the pervasive increase in complexity that it +requires. In PyPy, though, an optional "stackless" transformation is +able to turn the Python interpreter - also written in a simple highly +recursive style - into an efficient variant of continuation-passing +style (CPS), enabling the usage of coroutines in the translated +interpreter. For more details and other examples of translation-level +transformations, see [XXX]. + +\subsection{Getting good GCs and tools is possible} + +XXX +% ---- Bibliography ---- +%\begin{small} +\bibliographystyle{abbrv} +\bibliography{dyla} +%\end{small} + +\end{document} Added: pypy/extradoc/talk/dyla2007/llncs.cls ============================================================================== --- (empty file) +++ pypy/extradoc/talk/dyla2007/llncs.cls Sat May 26 23:38:20 2007 @@ -0,0 +1,1190 @@ +% LLNCS DOCUMENT CLASS -- version 2.14 (17-Aug-2004) +% Springer Verlag LaTeX2e support for Lecture Notes in Computer Science +% +%% +%% \CharacterTable +%% {Upper-case \A\B\C\D\E\F\G\H\I\J\K\L\M\N\O\P\Q\R\S\T\U\V\W\X\Y\Z +%% Lower-case \a\b\c\d\e\f\g\h\i\j\k\l\m\n\o\p\q\r\s\t\u\v\w\x\y\z +%% Digits \0\1\2\3\4\5\6\7\8\9 +%% Exclamation \! Double quote \" Hash (number) \# +%% Dollar \$ Percent \% Ampersand \& +%% Acute accent \' Left paren \( Right paren \) +%% Asterisk \* Plus \+ Comma \, +%% Minus \- Point \. Solidus \/ +%% Colon \: Semicolon \; Less than \< +%% Equals \= Greater than \> Question mark \? +%% Commercial at \@ Left bracket \[ Backslash \\ +%% Right bracket \] Circumflex \^ Underscore \_ +%% Grave accent \` Left brace \{ Vertical bar \| +%% Right brace \} Tilde \~} +%% +\NeedsTeXFormat{LaTeX2e}[1995/12/01] +\ProvidesClass{llncs}[2004/08/17 v2.14 +^^J LaTeX document class for Lecture Notes in Computer Science] +% Options +\let\if at envcntreset\iffalse +\DeclareOption{envcountreset}{\let\if at envcntreset\iftrue} +\DeclareOption{citeauthoryear}{\let\citeauthoryear=Y} +\DeclareOption{oribibl}{\let\oribibl=Y} +\let\if at custvec\iftrue +\DeclareOption{orivec}{\let\if at custvec\iffalse} +\let\if at envcntsame\iffalse +\DeclareOption{envcountsame}{\let\if at envcntsame\iftrue} +\let\if at envcntsect\iffalse +\DeclareOption{envcountsect}{\let\if at envcntsect\iftrue} +\let\if at runhead\iffalse +\DeclareOption{runningheads}{\let\if at runhead\iftrue} + +\let\if at openbib\iffalse +\DeclareOption{openbib}{\let\if at openbib\iftrue} + +% languages +\let\switcht@@therlang\relax +\def\ds at deutsch{\def\switcht@@therlang{\switcht at deutsch}} +\def\ds at francais{\def\switcht@@therlang{\switcht at francais}} + +\DeclareOption*{\PassOptionsToClass{\CurrentOption}{article}} + +\ProcessOptions + +\LoadClass[twoside]{article} +\RequirePackage{multicol} % needed for the list of participants, index + +\setlength{\textwidth}{12.2cm} +\setlength{\textheight}{19.3cm} +\renewcommand\@pnumwidth{2em} +\renewcommand\@tocrmarg{3.5em} +% +\def\@dottedtocline#1#2#3#4#5{% + \ifnum #1>\c at tocdepth \else + \vskip \z@ \@plus.2\p@ + {\leftskip #2\relax \rightskip \@tocrmarg \advance\rightskip by 0pt plus 2cm + \parfillskip -\rightskip \pretolerance=10000 + \parindent #2\relax\@afterindenttrue + \interlinepenalty\@M + \leavevmode + \@tempdima #3\relax + \advance\leftskip \@tempdima \null\nobreak\hskip -\leftskip + {#4}\nobreak + \leaders\hbox{$\m at th + \mkern \@dotsep mu\hbox{.}\mkern \@dotsep + mu$}\hfill + \nobreak + \hb at xt@\@pnumwidth{\hfil\normalfont \normalcolor #5}% + \par}% + \fi} +% +\def\switcht at albion{% +\def\abstractname{Abstract.} +\def\ackname{Acknowledgement.} +\def\andname{and} +\def\lastandname{\unskip, and} +\def\appendixname{Appendix} +\def\chaptername{Chapter} +\def\claimname{Claim} +\def\conjecturename{Conjecture} +\def\contentsname{Table of Contents} +\def\corollaryname{Corollary} +\def\definitionname{Definition} +\def\examplename{Example} +\def\exercisename{Exercise} +\def\figurename{Fig.} +\def\keywordname{{\bf Key words:}} +\def\indexname{Index} +\def\lemmaname{Lemma} +\def\contriblistname{List of Contributors} +\def\listfigurename{List of Figures} +\def\listtablename{List of Tables} +\def\mailname{{\it Correspondence to\/}:} +\def\noteaddname{Note added in proof} +\def\notename{Note} +\def\partname{Part} +\def\problemname{Problem} +\def\proofname{Proof} +\def\propertyname{Property} +\def\propositionname{Proposition} +\def\questionname{Question} +\def\remarkname{Remark} +\def\seename{see} +\def\solutionname{Solution} +\def\subclassname{{\it Subject Classifications\/}:} +\def\tablename{Table} +\def\theoremname{Theorem}} +\switcht at albion +% Names of theorem like environments are already defined +% but must be translated if another language is chosen +% +% French section +\def\switcht at francais{%\typeout{On parle francais.}% + \def\abstractname{R\'esum\'e.}% + \def\ackname{Remerciements.}% + \def\andname{et}% + \def\lastandname{ et}% + \def\appendixname{Appendice} + \def\chaptername{Chapitre}% + \def\claimname{Pr\'etention}% + \def\conjecturename{Hypoth\`ese}% + \def\contentsname{Table des mati\`eres}% + \def\corollaryname{Corollaire}% + \def\definitionname{D\'efinition}% + \def\examplename{Exemple}% + \def\exercisename{Exercice}% + \def\figurename{Fig.}% + \def\keywordname{{\bf Mots-cl\'e:}} + \def\indexname{Index} + \def\lemmaname{Lemme}% + \def\contriblistname{Liste des contributeurs} + \def\listfigurename{Liste des figures}% + \def\listtablename{Liste des tables}% + \def\mailname{{\it Correspondence to\/}:} + \def\noteaddname{Note ajout\'ee \`a l'\'epreuve}% + \def\notename{Remarque}% + \def\partname{Partie}% + \def\problemname{Probl\`eme}% + \def\proofname{Preuve}% + \def\propertyname{Caract\'eristique}% +%\def\propositionname{Proposition}% + \def\questionname{Question}% + \def\remarkname{Remarque}% + \def\seename{voir} + \def\solutionname{Solution}% + \def\subclassname{{\it Subject Classifications\/}:} + \def\tablename{Tableau}% + \def\theoremname{Th\'eor\`eme}% +} +% +% German section +\def\switcht at deutsch{%\typeout{Man spricht deutsch.}% + \def\abstractname{Zusammenfassung.}% + \def\ackname{Danksagung.}% + \def\andname{und}% + \def\lastandname{ und}% + \def\appendixname{Anhang}% + \def\chaptername{Kapitel}% + \def\claimname{Behauptung}% + \def\conjecturename{Hypothese}% + \def\contentsname{Inhaltsverzeichnis}% + \def\corollaryname{Korollar}% +%\def\definitionname{Definition}% + \def\examplename{Beispiel}% + \def\exercisename{\"Ubung}% + \def\figurename{Abb.}% + \def\keywordname{{\bf Schl\"usselw\"orter:}} + \def\indexname{Index} +%\def\lemmaname{Lemma}% + \def\contriblistname{Mitarbeiter} + \def\listfigurename{Abbildungsverzeichnis}% + \def\listtablename{Tabellenverzeichnis}% + \def\mailname{{\it Correspondence to\/}:} + \def\noteaddname{Nachtrag}% + \def\notename{Anmerkung}% + \def\partname{Teil}% +%\def\problemname{Problem}% + \def\proofname{Beweis}% + \def\propertyname{Eigenschaft}% +%\def\propositionname{Proposition}% + \def\questionname{Frage}% + \def\remarkname{Anmerkung}% + \def\seename{siehe} + \def\solutionname{L\"osung}% + \def\subclassname{{\it Subject Classifications\/}:} + \def\tablename{Tabelle}% +%\def\theoremname{Theorem}% +} + +% Ragged bottom for the actual page +\def\thisbottomragged{\def\@textbottom{\vskip\z@ plus.0001fil +\global\let\@textbottom\relax}} + +\renewcommand\small{% + \@setfontsize\small\@ixpt{11}% + \abovedisplayskip 8.5\p@ \@plus3\p@ \@minus4\p@ + \abovedisplayshortskip \z@ \@plus2\p@ + \belowdisplayshortskip 4\p@ \@plus2\p@ \@minus2\p@ + \def\@listi{\leftmargin\leftmargini + \parsep 0\p@ \@plus1\p@ \@minus\p@ + \topsep 8\p@ \@plus2\p@ \@minus4\p@ + \itemsep0\p@}% + \belowdisplayskip \abovedisplayskip +} + +\frenchspacing +\widowpenalty=10000 +\clubpenalty=10000 + +\setlength\oddsidemargin {63\p@} +\setlength\evensidemargin {63\p@} +\setlength\marginparwidth {90\p@} + +\setlength\headsep {16\p@} + +\setlength\footnotesep{7.7\p@} +\setlength\textfloatsep{8mm\@plus 2\p@ \@minus 4\p@} +\setlength\intextsep {8mm\@plus 2\p@ \@minus 2\p@} + +\setcounter{secnumdepth}{2} + +\newcounter {chapter} +\renewcommand\thechapter {\@arabic\c at chapter} + +\newif\if at mainmatter \@mainmattertrue +\newcommand\frontmatter{\cleardoublepage + \@mainmatterfalse\pagenumbering{Roman}} +\newcommand\mainmatter{\cleardoublepage + \@mainmattertrue\pagenumbering{arabic}} +\newcommand\backmatter{\if at openright\cleardoublepage\else\clearpage\fi + \@mainmatterfalse} + +\renewcommand\part{\cleardoublepage + \thispagestyle{empty}% + \if at twocolumn + \onecolumn + \@tempswatrue + \else + \@tempswafalse + \fi + \null\vfil + \secdef\@part\@spart} + +\def\@part[#1]#2{% + \ifnum \c at secnumdepth >-2\relax + \refstepcounter{part}% + \addcontentsline{toc}{part}{\thepart\hspace{1em}#1}% + \else + \addcontentsline{toc}{part}{#1}% + \fi + \markboth{}{}% + {\centering + \interlinepenalty \@M + \normalfont + \ifnum \c at secnumdepth >-2\relax + \huge\bfseries \partname~\thepart + \par + \vskip 20\p@ + \fi + \Huge \bfseries #2\par}% + \@endpart} +\def\@spart#1{% + {\centering + \interlinepenalty \@M + \normalfont + \Huge \bfseries #1\par}% + \@endpart} +\def\@endpart{\vfil\newpage + \if at twoside + \null + \thispagestyle{empty}% + \newpage + \fi + \if at tempswa + \twocolumn + \fi} + +\newcommand\chapter{\clearpage + \thispagestyle{empty}% + \global\@topnum\z@ + \@afterindentfalse + \secdef\@chapter\@schapter} +\def\@chapter[#1]#2{\ifnum \c at secnumdepth >\m at ne + \if at mainmatter + \refstepcounter{chapter}% + \typeout{\@chapapp\space\thechapter.}% + \addcontentsline{toc}{chapter}% + {\protect\numberline{\thechapter}#1}% + \else + \addcontentsline{toc}{chapter}{#1}% + \fi + \else + \addcontentsline{toc}{chapter}{#1}% + \fi + \chaptermark{#1}% + \addtocontents{lof}{\protect\addvspace{10\p@}}% + \addtocontents{lot}{\protect\addvspace{10\p@}}% + \if at twocolumn + \@topnewpage[\@makechapterhead{#2}]% + \else + \@makechapterhead{#2}% + \@afterheading + \fi} +\def\@makechapterhead#1{% +% \vspace*{50\p@}% + {\centering + \ifnum \c at secnumdepth >\m at ne + \if at mainmatter + \large\bfseries \@chapapp{} \thechapter + \par\nobreak + \vskip 20\p@ + \fi + \fi + \interlinepenalty\@M + \Large \bfseries #1\par\nobreak + \vskip 40\p@ + }} +\def\@schapter#1{\if at twocolumn + \@topnewpage[\@makeschapterhead{#1}]% + \else + \@makeschapterhead{#1}% + \@afterheading + \fi} +\def\@makeschapterhead#1{% +% \vspace*{50\p@}% + {\centering + \normalfont + \interlinepenalty\@M + \Large \bfseries #1\par\nobreak + \vskip 40\p@ + }} + +\renewcommand\section{\@startsection{section}{1}{\z@}% + {-18\p@ \@plus -4\p@ \@minus -4\p@}% + {12\p@ \@plus 4\p@ \@minus 4\p@}% + {\normalfont\large\bfseries\boldmath + \rightskip=\z@ \@plus 8em\pretolerance=10000 }} +\renewcommand\subsection{\@startsection{subsection}{2}{\z@}% + {-18\p@ \@plus -4\p@ \@minus -4\p@}% + {8\p@ \@plus 4\p@ \@minus 4\p@}% + {\normalfont\normalsize\bfseries\boldmath + \rightskip=\z@ \@plus 8em\pretolerance=10000 }} +\renewcommand\subsubsection{\@startsection{subsubsection}{3}{\z@}% + {-18\p@ \@plus -4\p@ \@minus -4\p@}% + {-0.5em \@plus -0.22em \@minus -0.1em}% + {\normalfont\normalsize\bfseries\boldmath}} +\renewcommand\paragraph{\@startsection{paragraph}{4}{\z@}% + {-12\p@ \@plus -4\p@ \@minus -4\p@}% + {-0.5em \@plus -0.22em \@minus -0.1em}% + {\normalfont\normalsize\itshape}} +\renewcommand\subparagraph[1]{\typeout{LLNCS warning: You should not use + \string\subparagraph\space with this class}\vskip0.5cm +You should not use \verb|\subparagraph| with this class.\vskip0.5cm} + +\DeclareMathSymbol{\Gamma}{\mathalpha}{letters}{"00} +\DeclareMathSymbol{\Delta}{\mathalpha}{letters}{"01} +\DeclareMathSymbol{\Theta}{\mathalpha}{letters}{"02} +\DeclareMathSymbol{\Lambda}{\mathalpha}{letters}{"03} +\DeclareMathSymbol{\Xi}{\mathalpha}{letters}{"04} +\DeclareMathSymbol{\Pi}{\mathalpha}{letters}{"05} +\DeclareMathSymbol{\Sigma}{\mathalpha}{letters}{"06} +\DeclareMathSymbol{\Upsilon}{\mathalpha}{letters}{"07} +\DeclareMathSymbol{\Phi}{\mathalpha}{letters}{"08} +\DeclareMathSymbol{\Psi}{\mathalpha}{letters}{"09} +\DeclareMathSymbol{\Omega}{\mathalpha}{letters}{"0A} + +\let\footnotesize\small + +\if at custvec +\def\vec#1{\mathchoice{\mbox{\boldmath$\displaystyle#1$}} +{\mbox{\boldmath$\textstyle#1$}} +{\mbox{\boldmath$\scriptstyle#1$}} +{\mbox{\boldmath$\scriptscriptstyle#1$}}} +\fi + +\def\squareforqed{\hbox{\rlap{$\sqcap$}$\sqcup$}} +\def\qed{\ifmmode\squareforqed\else{\unskip\nobreak\hfil +\penalty50\hskip1em\null\nobreak\hfil\squareforqed +\parfillskip=0pt\finalhyphendemerits=0\endgraf}\fi} + +\def\getsto{\mathrel{\mathchoice {\vcenter{\offinterlineskip +\halign{\hfil +$\displaystyle##$\hfil\cr\gets\cr\to\cr}}} +{\vcenter{\offinterlineskip\halign{\hfil$\textstyle##$\hfil\cr\gets +\cr\to\cr}}} +{\vcenter{\offinterlineskip\halign{\hfil$\scriptstyle##$\hfil\cr\gets +\cr\to\cr}}} +{\vcenter{\offinterlineskip\halign{\hfil$\scriptscriptstyle##$\hfil\cr +\gets\cr\to\cr}}}}} +\def\lid{\mathrel{\mathchoice {\vcenter{\offinterlineskip\halign{\hfil +$\displaystyle##$\hfil\cr<\cr\noalign{\vskip1.2pt}=\cr}}} +{\vcenter{\offinterlineskip\halign{\hfil$\textstyle##$\hfil\cr<\cr +\noalign{\vskip1.2pt}=\cr}}} +{\vcenter{\offinterlineskip\halign{\hfil$\scriptstyle##$\hfil\cr<\cr +\noalign{\vskip1pt}=\cr}}} +{\vcenter{\offinterlineskip\halign{\hfil$\scriptscriptstyle##$\hfil\cr +<\cr +\noalign{\vskip0.9pt}=\cr}}}}} +\def\gid{\mathrel{\mathchoice {\vcenter{\offinterlineskip\halign{\hfil +$\displaystyle##$\hfil\cr>\cr\noalign{\vskip1.2pt}=\cr}}} +{\vcenter{\offinterlineskip\halign{\hfil$\textstyle##$\hfil\cr>\cr +\noalign{\vskip1.2pt}=\cr}}} +{\vcenter{\offinterlineskip\halign{\hfil$\scriptstyle##$\hfil\cr>\cr +\noalign{\vskip1pt}=\cr}}} +{\vcenter{\offinterlineskip\halign{\hfil$\scriptscriptstyle##$\hfil\cr +>\cr +\noalign{\vskip0.9pt}=\cr}}}}} +\def\grole{\mathrel{\mathchoice {\vcenter{\offinterlineskip +\halign{\hfil +$\displaystyle##$\hfil\cr>\cr\noalign{\vskip-1pt}<\cr}}} +{\vcenter{\offinterlineskip\halign{\hfil$\textstyle##$\hfil\cr +>\cr\noalign{\vskip-1pt}<\cr}}} +{\vcenter{\offinterlineskip\halign{\hfil$\scriptstyle##$\hfil\cr +>\cr\noalign{\vskip-0.8pt}<\cr}}} +{\vcenter{\offinterlineskip\halign{\hfil$\scriptscriptstyle##$\hfil\cr +>\cr\noalign{\vskip-0.3pt}<\cr}}}}} +\def\bbbr{{\rm I\!R}} %reelle Zahlen +\def\bbbm{{\rm I\!M}} +\def\bbbn{{\rm I\!N}} %natuerliche Zahlen +\def\bbbf{{\rm I\!F}} +\def\bbbh{{\rm I\!H}} +\def\bbbk{{\rm I\!K}} +\def\bbbp{{\rm I\!P}} +\def\bbbone{{\mathchoice {\rm 1\mskip-4mu l} {\rm 1\mskip-4mu l} +{\rm 1\mskip-4.5mu l} {\rm 1\mskip-5mu l}}} +\def\bbbc{{\mathchoice {\setbox0=\hbox{$\displaystyle\rm C$}\hbox{\hbox +to0pt{\kern0.4\wd0\vrule height0.9\ht0\hss}\box0}} +{\setbox0=\hbox{$\textstyle\rm C$}\hbox{\hbox +to0pt{\kern0.4\wd0\vrule height0.9\ht0\hss}\box0}} +{\setbox0=\hbox{$\scriptstyle\rm C$}\hbox{\hbox +to0pt{\kern0.4\wd0\vrule height0.9\ht0\hss}\box0}} +{\setbox0=\hbox{$\scriptscriptstyle\rm C$}\hbox{\hbox +to0pt{\kern0.4\wd0\vrule height0.9\ht0\hss}\box0}}}} +\def\bbbq{{\mathchoice {\setbox0=\hbox{$\displaystyle\rm +Q$}\hbox{\raise +0.15\ht0\hbox to0pt{\kern0.4\wd0\vrule height0.8\ht0\hss}\box0}} +{\setbox0=\hbox{$\textstyle\rm Q$}\hbox{\raise +0.15\ht0\hbox to0pt{\kern0.4\wd0\vrule height0.8\ht0\hss}\box0}} +{\setbox0=\hbox{$\scriptstyle\rm Q$}\hbox{\raise +0.15\ht0\hbox to0pt{\kern0.4\wd0\vrule height0.7\ht0\hss}\box0}} +{\setbox0=\hbox{$\scriptscriptstyle\rm Q$}\hbox{\raise +0.15\ht0\hbox to0pt{\kern0.4\wd0\vrule height0.7\ht0\hss}\box0}}}} +\def\bbbt{{\mathchoice {\setbox0=\hbox{$\displaystyle\rm +T$}\hbox{\hbox to0pt{\kern0.3\wd0\vrule height0.9\ht0\hss}\box0}} +{\setbox0=\hbox{$\textstyle\rm T$}\hbox{\hbox +to0pt{\kern0.3\wd0\vrule height0.9\ht0\hss}\box0}} +{\setbox0=\hbox{$\scriptstyle\rm T$}\hbox{\hbox +to0pt{\kern0.3\wd0\vrule height0.9\ht0\hss}\box0}} +{\setbox0=\hbox{$\scriptscriptstyle\rm T$}\hbox{\hbox +to0pt{\kern0.3\wd0\vrule height0.9\ht0\hss}\box0}}}} +\def\bbbs{{\mathchoice +{\setbox0=\hbox{$\displaystyle \rm S$}\hbox{\raise0.5\ht0\hbox +to0pt{\kern0.35\wd0\vrule height0.45\ht0\hss}\hbox +to0pt{\kern0.55\wd0\vrule height0.5\ht0\hss}\box0}} +{\setbox0=\hbox{$\textstyle \rm S$}\hbox{\raise0.5\ht0\hbox +to0pt{\kern0.35\wd0\vrule height0.45\ht0\hss}\hbox +to0pt{\kern0.55\wd0\vrule height0.5\ht0\hss}\box0}} +{\setbox0=\hbox{$\scriptstyle \rm S$}\hbox{\raise0.5\ht0\hbox +to0pt{\kern0.35\wd0\vrule height0.45\ht0\hss}\raise0.05\ht0\hbox +to0pt{\kern0.5\wd0\vrule height0.45\ht0\hss}\box0}} +{\setbox0=\hbox{$\scriptscriptstyle\rm S$}\hbox{\raise0.5\ht0\hbox +to0pt{\kern0.4\wd0\vrule height0.45\ht0\hss}\raise0.05\ht0\hbox +to0pt{\kern0.55\wd0\vrule height0.45\ht0\hss}\box0}}}} +\def\bbbz{{\mathchoice {\hbox{$\mathsf\textstyle Z\kern-0.4em Z$}} +{\hbox{$\mathsf\textstyle Z\kern-0.4em Z$}} +{\hbox{$\mathsf\scriptstyle Z\kern-0.3em Z$}} +{\hbox{$\mathsf\scriptscriptstyle Z\kern-0.2em Z$}}}} + +\let\ts\, + +\setlength\leftmargini {17\p@} +\setlength\leftmargin {\leftmargini} +\setlength\leftmarginii {\leftmargini} +\setlength\leftmarginiii {\leftmargini} +\setlength\leftmarginiv {\leftmargini} +\setlength \labelsep {.5em} +\setlength \labelwidth{\leftmargini} +\addtolength\labelwidth{-\labelsep} + +\def\@listI{\leftmargin\leftmargini + \parsep 0\p@ \@plus1\p@ \@minus\p@ + \topsep 8\p@ \@plus2\p@ \@minus4\p@ + \itemsep0\p@} +\let\@listi\@listI +\@listi +\def\@listii {\leftmargin\leftmarginii + \labelwidth\leftmarginii + \advance\labelwidth-\labelsep + \topsep 0\p@ \@plus2\p@ \@minus\p@} +\def\@listiii{\leftmargin\leftmarginiii + \labelwidth\leftmarginiii + \advance\labelwidth-\labelsep + \topsep 0\p@ \@plus\p@\@minus\p@ + \parsep \z@ + \partopsep \p@ \@plus\z@ \@minus\p@} + +\renewcommand\labelitemi{\normalfont\bfseries --} +\renewcommand\labelitemii{$\m at th\bullet$} + +\setlength\arraycolsep{1.4\p@} +\setlength\tabcolsep{1.4\p@} + +\def\tableofcontents{\chapter*{\contentsname\@mkboth{{\contentsname}}% + {{\contentsname}}} + \def\authcount##1{\setcounter{auco}{##1}\setcounter{@auth}{1}} + \def\lastand{\ifnum\value{auco}=2\relax + \unskip{} \andname\ + \else + \unskip \lastandname\ + \fi}% + \def\and{\stepcounter{@auth}\relax + \ifnum\value{@auth}=\value{auco}% + \lastand + \else + \unskip, + \fi}% + \@starttoc{toc}\if at restonecol\twocolumn\fi} + +\def\l at part#1#2{\addpenalty{\@secpenalty}% + \addvspace{2em plus\p@}% % space above part line + \begingroup + \parindent \z@ + \rightskip \z@ plus 5em + \hrule\vskip5pt + \large % same size as for a contribution heading + \bfseries\boldmath % set line in boldface + \leavevmode % TeX command to enter horizontal mode. + #1\par + \vskip5pt + \hrule + \vskip1pt + \nobreak % Never break after part entry + \endgroup} + +\def\@dotsep{2} + +\def\hyperhrefextend{\ifx\hyper at anchor\@undefined\else +{chapter.\thechapter}\fi} + +\def\addnumcontentsmark#1#2#3{% +\addtocontents{#1}{\protect\contentsline{#2}{\protect\numberline + {\thechapter}#3}{\thepage}\hyperhrefextend}} +\def\addcontentsmark#1#2#3{% +\addtocontents{#1}{\protect\contentsline{#2}{#3}{\thepage}\hyperhrefextend}} +\def\addcontentsmarkwop#1#2#3{% +\addtocontents{#1}{\protect\contentsline{#2}{#3}{0}\hyperhrefextend}} + +\def\@adcmk[#1]{\ifcase #1 \or +\def\@gtempa{\addnumcontentsmark}% + \or \def\@gtempa{\addcontentsmark}% + \or \def\@gtempa{\addcontentsmarkwop}% + \fi\@gtempa{toc}{chapter}} +\def\addtocmark{\@ifnextchar[{\@adcmk}{\@adcmk[3]}} + +\def\l at chapter#1#2{\addpenalty{-\@highpenalty} + \vskip 1.0em plus 1pt \@tempdima 1.5em \begingroup + \parindent \z@ \rightskip \@tocrmarg + \advance\rightskip by 0pt plus 2cm + \parfillskip -\rightskip \pretolerance=10000 + \leavevmode \advance\leftskip\@tempdima \hskip -\leftskip + {\large\bfseries\boldmath#1}\ifx0#2\hfil\null + \else + \nobreak + \leaders\hbox{$\m at th \mkern \@dotsep mu.\mkern + \@dotsep mu$}\hfill + \nobreak\hbox to\@pnumwidth{\hss #2}% + \fi\par + \penalty\@highpenalty \endgroup} + +\def\l at title#1#2{\addpenalty{-\@highpenalty} + \addvspace{8pt plus 1pt} + \@tempdima \z@ + \begingroup + \parindent \z@ \rightskip \@tocrmarg + \advance\rightskip by 0pt plus 2cm + \parfillskip -\rightskip \pretolerance=10000 + \leavevmode \advance\leftskip\@tempdima \hskip -\leftskip + #1\nobreak + \leaders\hbox{$\m at th \mkern \@dotsep mu.\mkern + \@dotsep mu$}\hfill + \nobreak\hbox to\@pnumwidth{\hss #2}\par + \penalty\@highpenalty \endgroup} + +\def\l at author#1#2{\addpenalty{\@highpenalty} + \@tempdima=15\p@ %\z@ + \begingroup + \parindent \z@ \rightskip \@tocrmarg + \advance\rightskip by 0pt plus 2cm + \pretolerance=10000 + \leavevmode \advance\leftskip\@tempdima %\hskip -\leftskip + \textit{#1}\par + \penalty\@highpenalty \endgroup} + +\setcounter{tocdepth}{0} +\newdimen\tocchpnum +\newdimen\tocsecnum +\newdimen\tocsectotal +\newdimen\tocsubsecnum +\newdimen\tocsubsectotal +\newdimen\tocsubsubsecnum +\newdimen\tocsubsubsectotal +\newdimen\tocparanum +\newdimen\tocparatotal +\newdimen\tocsubparanum +\tocchpnum=\z@ % no chapter numbers +\tocsecnum=15\p@ % section 88. plus 2.222pt +\tocsubsecnum=23\p@ % subsection 88.8 plus 2.222pt +\tocsubsubsecnum=27\p@ % subsubsection 88.8.8 plus 1.444pt +\tocparanum=35\p@ % paragraph 88.8.8.8 plus 1.666pt +\tocsubparanum=43\p@ % subparagraph 88.8.8.8.8 plus 1.888pt +\def\calctocindent{% +\tocsectotal=\tocchpnum +\advance\tocsectotal by\tocsecnum +\tocsubsectotal=\tocsectotal +\advance\tocsubsectotal by\tocsubsecnum +\tocsubsubsectotal=\tocsubsectotal +\advance\tocsubsubsectotal by\tocsubsubsecnum +\tocparatotal=\tocsubsubsectotal +\advance\tocparatotal by\tocparanum} +\calctocindent + +\def\l at section{\@dottedtocline{1}{\tocchpnum}{\tocsecnum}} +\def\l at subsection{\@dottedtocline{2}{\tocsectotal}{\tocsubsecnum}} +\def\l at subsubsection{\@dottedtocline{3}{\tocsubsectotal}{\tocsubsubsecnum}} +\def\l at paragraph{\@dottedtocline{4}{\tocsubsubsectotal}{\tocparanum}} +\def\l at subparagraph{\@dottedtocline{5}{\tocparatotal}{\tocsubparanum}} + +\def\listoffigures{\@restonecolfalse\if at twocolumn\@restonecoltrue\onecolumn + \fi\section*{\listfigurename\@mkboth{{\listfigurename}}{{\listfigurename}}} + \@starttoc{lof}\if at restonecol\twocolumn\fi} +\def\l at figure{\@dottedtocline{1}{0em}{1.5em}} + +\def\listoftables{\@restonecolfalse\if at twocolumn\@restonecoltrue\onecolumn + \fi\section*{\listtablename\@mkboth{{\listtablename}}{{\listtablename}}} + \@starttoc{lot}\if at restonecol\twocolumn\fi} +\let\l at table\l at figure + +\renewcommand\listoffigures{% + \section*{\listfigurename + \@mkboth{\listfigurename}{\listfigurename}}% + \@starttoc{lof}% + } + +\renewcommand\listoftables{% + \section*{\listtablename + \@mkboth{\listtablename}{\listtablename}}% + \@starttoc{lot}% + } + +\ifx\oribibl\undefined +\ifx\citeauthoryear\undefined +\renewenvironment{thebibliography}[1] + {\section*{\refname} + \def\@biblabel##1{##1.} + \small + \list{\@biblabel{\@arabic\c at enumiv}}% + {\settowidth\labelwidth{\@biblabel{#1}}% + \leftmargin\labelwidth + \advance\leftmargin\labelsep + \if at openbib + \advance\leftmargin\bibindent + \itemindent -\bibindent + \listparindent \itemindent + \parsep \z@ + \fi + \usecounter{enumiv}% + \let\p at enumiv\@empty + \renewcommand\theenumiv{\@arabic\c at enumiv}}% + \if at openbib + \renewcommand\newblock{\par}% + \else + \renewcommand\newblock{\hskip .11em \@plus.33em \@minus.07em}% + \fi + \sloppy\clubpenalty4000\widowpenalty4000% + \sfcode`\.=\@m} + {\def\@noitemerr + {\@latex at warning{Empty `thebibliography' environment}}% + \endlist} +\def\@lbibitem[#1]#2{\item[{[#1]}\hfill]\if at filesw + {\let\protect\noexpand\immediate + \write\@auxout{\string\bibcite{#2}{#1}}}\fi\ignorespaces} +\newcount\@tempcntc +\def\@citex[#1]#2{\if at filesw\immediate\write\@auxout{\string\citation{#2}}\fi + \@tempcnta\z@\@tempcntb\m at ne\def\@citea{}\@cite{\@for\@citeb:=#2\do + {\@ifundefined + {b@\@citeb}{\@citeo\@tempcntb\m at ne\@citea\def\@citea{,}{\bfseries + ?}\@warning + {Citation `\@citeb' on page \thepage \space undefined}}% + {\setbox\z@\hbox{\global\@tempcntc0\csname b@\@citeb\endcsname\relax}% + \ifnum\@tempcntc=\z@ \@citeo\@tempcntb\m at ne + \@citea\def\@citea{,}\hbox{\csname b@\@citeb\endcsname}% + \else + \advance\@tempcntb\@ne + \ifnum\@tempcntb=\@tempcntc + \else\advance\@tempcntb\m at ne\@citeo + \@tempcnta\@tempcntc\@tempcntb\@tempcntc\fi\fi}}\@citeo}{#1}} +\def\@citeo{\ifnum\@tempcnta>\@tempcntb\else + \@citea\def\@citea{,\,\hskip\z at skip}% + \ifnum\@tempcnta=\@tempcntb\the\@tempcnta\else + {\advance\@tempcnta\@ne\ifnum\@tempcnta=\@tempcntb \else + \def\@citea{--}\fi + \advance\@tempcnta\m at ne\the\@tempcnta\@citea\the\@tempcntb}\fi\fi} +\else +\renewenvironment{thebibliography}[1] + {\section*{\refname} + \small + \list{}% + {\settowidth\labelwidth{}% + \leftmargin\parindent + \itemindent=-\parindent + \labelsep=\z@ + \if at openbib + \advance\leftmargin\bibindent + \itemindent -\bibindent + \listparindent \itemindent + \parsep \z@ + \fi + \usecounter{enumiv}% + \let\p at enumiv\@empty + \renewcommand\theenumiv{}}% + \if at openbib + \renewcommand\newblock{\par}% + \else + \renewcommand\newblock{\hskip .11em \@plus.33em \@minus.07em}% + \fi + \sloppy\clubpenalty4000\widowpenalty4000% + \sfcode`\.=\@m} + {\def\@noitemerr + {\@latex at warning{Empty `thebibliography' environment}}% + \endlist} + \def\@cite#1{#1}% + \def\@lbibitem[#1]#2{\item[]\if at filesw + {\def\protect##1{\string ##1\space}\immediate + \write\@auxout{\string\bibcite{#2}{#1}}}\fi\ignorespaces} + \fi +\else +\@cons\@openbib at code{\noexpand\small} +\fi + +\def\idxquad{\hskip 10\p@}% space that divides entry from number + +\def\@idxitem{\par\hangindent 10\p@} + +\def\subitem{\par\setbox0=\hbox{--\enspace}% second order + \noindent\hangindent\wd0\box0}% index entry + +\def\subsubitem{\par\setbox0=\hbox{--\,--\enspace}% third + \noindent\hangindent\wd0\box0}% order index entry + +\def\indexspace{\par \vskip 10\p@ plus5\p@ minus3\p@\relax} + +\renewenvironment{theindex} + {\@mkboth{\indexname}{\indexname}% + \thispagestyle{empty}\parindent\z@ + \parskip\z@ \@plus .3\p@\relax + \let\item\par + \def\,{\relax\ifmmode\mskip\thinmuskip + \else\hskip0.2em\ignorespaces\fi}% + \normalfont\small + \begin{multicols}{2}[\@makeschapterhead{\indexname}]% + } + {\end{multicols}} + +\renewcommand\footnoterule{% + \kern-3\p@ + \hrule\@width 2truecm + \kern2.6\p@} + \newdimen\fnindent + \fnindent1em +\long\def\@makefntext#1{% + \parindent \fnindent% + \leftskip \fnindent% + \noindent + \llap{\hb at xt@1em{\hss\@makefnmark\ }}\ignorespaces#1} + +\long\def\@makecaption#1#2{% + \vskip\abovecaptionskip + \sbox\@tempboxa{{\bfseries #1.} #2}% + \ifdim \wd\@tempboxa >\hsize + {\bfseries #1.} #2\par + \else + \global \@minipagefalse + \hb at xt@\hsize{\hfil\box\@tempboxa\hfil}% + \fi + \vskip\belowcaptionskip} + +\def\fps at figure{htbp} +\def\fnum at figure{\figurename\thinspace\thefigure} +\def \@floatboxreset {% + \reset at font + \small + \@setnobreak + \@setminipage +} +\def\fps at table{htbp} +\def\fnum at table{\tablename~\thetable} +\renewenvironment{table} + {\setlength\abovecaptionskip{0\p@}% + \setlength\belowcaptionskip{10\p@}% + \@float{table}} + {\end at float} +\renewenvironment{table*} + {\setlength\abovecaptionskip{0\p@}% + \setlength\belowcaptionskip{10\p@}% + \@dblfloat{table}} + {\end at dblfloat} + +\long\def\@caption#1[#2]#3{\par\addcontentsline{\csname + ext@#1\endcsname}{#1}{\protect\numberline{\csname + the#1\endcsname}{\ignorespaces #2}}\begingroup + \@parboxrestore + \@makecaption{\csname fnum@#1\endcsname}{\ignorespaces #3}\par + \endgroup} + +% LaTeX does not provide a command to enter the authors institute +% addresses. The \institute command is defined here. + +\newcounter{@inst} +\newcounter{@auth} +\newcounter{auco} +\newdimen\instindent +\newbox\authrun +\newtoks\authorrunning +\newtoks\tocauthor +\newbox\titrun +\newtoks\titlerunning +\newtoks\toctitle + +\def\clearheadinfo{\gdef\@author{No Author Given}% + \gdef\@title{No Title Given}% + \gdef\@subtitle{}% + \gdef\@institute{No Institute Given}% + \gdef\@thanks{}% + \global\titlerunning={}\global\authorrunning={}% + \global\toctitle={}\global\tocauthor={}} + +\def\institute#1{\gdef\@institute{#1}} + +\def\institutename{\par + \begingroup + \parskip=\z@ + \parindent=\z@ + \setcounter{@inst}{1}% + \def\and{\par\stepcounter{@inst}% + \noindent$^{\the at inst}$\enspace\ignorespaces}% + \setbox0=\vbox{\def\thanks##1{}\@institute}% + \ifnum\c@@inst=1\relax + \gdef\fnnstart{0}% + \else + \xdef\fnnstart{\c@@inst}% + \setcounter{@inst}{1}% + \noindent$^{\the at inst}$\enspace + \fi + \ignorespaces + \@institute\par + \endgroup} + +\def\@fnsymbol#1{\ensuremath{\ifcase#1\or\star\or{\star\star}\or + {\star\star\star}\or \dagger\or \ddagger\or + \mathchar "278\or \mathchar "27B\or \|\or **\or \dagger\dagger + \or \ddagger\ddagger \else\@ctrerr\fi}} + +\def\inst#1{\unskip$^{#1}$} +\def\fnmsep{\unskip$^,$} +\def\email#1{{\tt#1}} +\AtBeginDocument{\@ifundefined{url}{\def\url#1{#1}}{}% +\@ifpackageloaded{babel}{% +\@ifundefined{extrasenglish}{}{\addto\extrasenglish{\switcht at albion}}% +\@ifundefined{extrasfrenchb}{}{\addto\extrasfrenchb{\switcht at francais}}% +\@ifundefined{extrasgerman}{}{\addto\extrasgerman{\switcht at deutsch}}% +}{\switcht@@therlang}% +} +\def\homedir{\~{ }} + +\def\subtitle#1{\gdef\@subtitle{#1}} +\clearheadinfo +% +\renewcommand\maketitle{\newpage + \refstepcounter{chapter}% + \stepcounter{section}% + \setcounter{section}{0}% + \setcounter{subsection}{0}% + \setcounter{figure}{0} + \setcounter{table}{0} + \setcounter{equation}{0} + \setcounter{footnote}{0}% + \begingroup + \parindent=\z@ + \renewcommand\thefootnote{\@fnsymbol\c at footnote}% + \if at twocolumn + \ifnum \col at number=\@ne + \@maketitle + \else + \twocolumn[\@maketitle]% + \fi + \else + \newpage + \global\@topnum\z@ % Prevents figures from going at top of page. + \@maketitle + \fi + \thispagestyle{empty}\@thanks +% + \def\\{\unskip\ \ignorespaces}\def\inst##1{\unskip{}}% + \def\thanks##1{\unskip{}}\def\fnmsep{\unskip}% + \instindent=\hsize + \advance\instindent by-\headlineindent + \if!\the\toctitle!\addcontentsline{toc}{title}{\@title}\else + \addcontentsline{toc}{title}{\the\toctitle}\fi + \if at runhead + \if!\the\titlerunning!\else + \edef\@title{\the\titlerunning}% + \fi + \global\setbox\titrun=\hbox{\small\rm\unboldmath\ignorespaces\@title}% + \ifdim\wd\titrun>\instindent + \typeout{Title too long for running head. Please supply}% + \typeout{a shorter form with \string\titlerunning\space prior to + \string\maketitle}% + \global\setbox\titrun=\hbox{\small\rm + Title Suppressed Due to Excessive Length}% + \fi + \xdef\@title{\copy\titrun}% + \fi +% + \if!\the\tocauthor!\relax + {\def\and{\noexpand\protect\noexpand\and}% + \protected at xdef\toc at uthor{\@author}}% + \else + \def\\{\noexpand\protect\noexpand\newline}% + \protected at xdef\scratch{\the\tocauthor}% + \protected at xdef\toc at uthor{\scratch}% + \fi + \addtocontents{toc}{\noexpand\protect\noexpand\authcount{\the\c at auco}}% + \addcontentsline{toc}{author}{\toc at uthor}% + \if at runhead + \if!\the\authorrunning! + \value{@inst}=\value{@auth}% + \setcounter{@auth}{1}% + \else + \edef\@author{\the\authorrunning}% + \fi + \global\setbox\authrun=\hbox{\small\unboldmath\@author\unskip}% + \ifdim\wd\authrun>\instindent + \typeout{Names of authors too long for running head. Please supply}% + \typeout{a shorter form with \string\authorrunning\space prior to + \string\maketitle}% + \global\setbox\authrun=\hbox{\small\rm + Authors Suppressed Due to Excessive Length}% + \fi + \xdef\@author{\copy\authrun}% + \markboth{\@author}{\@title}% + \fi + \endgroup + \setcounter{footnote}{\fnnstart}% + \clearheadinfo} +% +\def\@maketitle{\newpage + \markboth{}{}% + \def\lastand{\ifnum\value{@inst}=2\relax + \unskip{} \andname\ + \else + \unskip \lastandname\ + \fi}% + \def\and{\stepcounter{@auth}\relax + \ifnum\value{@auth}=\value{@inst}% + \lastand + \else + \unskip, + \fi}% + \begin{center}% + \let\newline\\ + {\Large \bfseries\boldmath + \pretolerance=10000 + \@title \par}\vskip .8cm +\if!\@subtitle!\else {\large \bfseries\boldmath + \vskip -.65cm + \pretolerance=10000 + \@subtitle \par}\vskip .8cm\fi + \setbox0=\vbox{\setcounter{@auth}{1}\def\and{\stepcounter{@auth}}% + \def\thanks##1{}\@author}% + \global\value{@inst}=\value{@auth}% + \global\value{auco}=\value{@auth}% + \setcounter{@auth}{1}% +{\lineskip .5em +\noindent\ignorespaces +\@author\vskip.35cm} + {\small\institutename} + \end{center}% + } + +% definition of the "\spnewtheorem" command. +% +% Usage: +% +% \spnewtheorem{env_nam}{caption}[within]{cap_font}{body_font} +% or \spnewtheorem{env_nam}[numbered_like]{caption}{cap_font}{body_font} +% or \spnewtheorem*{env_nam}{caption}{cap_font}{body_font} +% +% New is "cap_font" and "body_font". It stands for +% fontdefinition of the caption and the text itself. +% +% "\spnewtheorem*" gives a theorem without number. +% +% A defined spnewthoerem environment is used as described +% by Lamport. +% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +\def\@thmcountersep{} +\def\@thmcounterend{.} + +\def\spnewtheorem{\@ifstar{\@sthm}{\@Sthm}} + +% definition of \spnewtheorem with number + +\def\@spnthm#1#2{% + \@ifnextchar[{\@spxnthm{#1}{#2}}{\@spynthm{#1}{#2}}} +\def\@Sthm#1{\@ifnextchar[{\@spothm{#1}}{\@spnthm{#1}}} + +\def\@spxnthm#1#2[#3]#4#5{\expandafter\@ifdefinable\csname #1\endcsname + {\@definecounter{#1}\@addtoreset{#1}{#3}% + \expandafter\xdef\csname the#1\endcsname{\expandafter\noexpand + \csname the#3\endcsname \noexpand\@thmcountersep \@thmcounter{#1}}% + \expandafter\xdef\csname #1name\endcsname{#2}% + \global\@namedef{#1}{\@spthm{#1}{\csname #1name\endcsname}{#4}{#5}}% + \global\@namedef{end#1}{\@endtheorem}}} + +\def\@spynthm#1#2#3#4{\expandafter\@ifdefinable\csname #1\endcsname + {\@definecounter{#1}% + \expandafter\xdef\csname the#1\endcsname{\@thmcounter{#1}}% + \expandafter\xdef\csname #1name\endcsname{#2}% + \global\@namedef{#1}{\@spthm{#1}{\csname #1name\endcsname}{#3}{#4}}% + \global\@namedef{end#1}{\@endtheorem}}} + +\def\@spothm#1[#2]#3#4#5{% + \@ifundefined{c@#2}{\@latexerr{No theorem environment `#2' defined}\@eha}% + {\expandafter\@ifdefinable\csname #1\endcsname + {\global\@namedef{the#1}{\@nameuse{the#2}}% + \expandafter\xdef\csname #1name\endcsname{#3}% + \global\@namedef{#1}{\@spthm{#2}{\csname #1name\endcsname}{#4}{#5}}% + \global\@namedef{end#1}{\@endtheorem}}}} + +\def\@spthm#1#2#3#4{\topsep 7\p@ \@plus2\p@ \@minus4\p@ +\refstepcounter{#1}% +\@ifnextchar[{\@spythm{#1}{#2}{#3}{#4}}{\@spxthm{#1}{#2}{#3}{#4}}} + +\def\@spxthm#1#2#3#4{\@spbegintheorem{#2}{\csname the#1\endcsname}{#3}{#4}% + \ignorespaces} + +\def\@spythm#1#2#3#4[#5]{\@spopargbegintheorem{#2}{\csname + the#1\endcsname}{#5}{#3}{#4}\ignorespaces} + +\def\@spbegintheorem#1#2#3#4{\trivlist + \item[\hskip\labelsep{#3#1\ #2\@thmcounterend}]#4} + +\def\@spopargbegintheorem#1#2#3#4#5{\trivlist + \item[\hskip\labelsep{#4#1\ #2}]{#4(#3)\@thmcounterend\ }#5} + +% definition of \spnewtheorem* without number + +\def\@sthm#1#2{\@Ynthm{#1}{#2}} + +\def\@Ynthm#1#2#3#4{\expandafter\@ifdefinable\csname #1\endcsname + {\global\@namedef{#1}{\@Thm{\csname #1name\endcsname}{#3}{#4}}% + \expandafter\xdef\csname #1name\endcsname{#2}% + \global\@namedef{end#1}{\@endtheorem}}} + +\def\@Thm#1#2#3{\topsep 7\p@ \@plus2\p@ \@minus4\p@ +\@ifnextchar[{\@Ythm{#1}{#2}{#3}}{\@Xthm{#1}{#2}{#3}}} + +\def\@Xthm#1#2#3{\@Begintheorem{#1}{#2}{#3}\ignorespaces} + +\def\@Ythm#1#2#3[#4]{\@Opargbegintheorem{#1} + {#4}{#2}{#3}\ignorespaces} + +\def\@Begintheorem#1#2#3{#3\trivlist + \item[\hskip\labelsep{#2#1\@thmcounterend}]} + +\def\@Opargbegintheorem#1#2#3#4{#4\trivlist + \item[\hskip\labelsep{#3#1}]{#3(#2)\@thmcounterend\ }} + +\if at envcntsect + \def\@thmcountersep{.} + \spnewtheorem{theorem}{Theorem}[section]{\bfseries}{\itshape} +\else + \spnewtheorem{theorem}{Theorem}{\bfseries}{\itshape} + \if at envcntreset + \@addtoreset{theorem}{section} + \else + \@addtoreset{theorem}{chapter} + \fi +\fi + +%definition of divers theorem environments +\spnewtheorem*{claim}{Claim}{\itshape}{\rmfamily} +\spnewtheorem*{proof}{Proof}{\itshape}{\rmfamily} +\if at envcntsame % alle Umgebungen wie Theorem. + \def\spn at wtheorem#1#2#3#4{\@spothm{#1}[theorem]{#2}{#3}{#4}} +\else % alle Umgebungen mit eigenem Zaehler + \if at envcntsect % mit section numeriert + \def\spn at wtheorem#1#2#3#4{\@spxnthm{#1}{#2}[section]{#3}{#4}} + \else % nicht mit section numeriert + \if at envcntreset + \def\spn at wtheorem#1#2#3#4{\@spynthm{#1}{#2}{#3}{#4} + \@addtoreset{#1}{section}} + \else + \def\spn at wtheorem#1#2#3#4{\@spynthm{#1}{#2}{#3}{#4} + \@addtoreset{#1}{chapter}}% + \fi + \fi +\fi +\spn at wtheorem{case}{Case}{\itshape}{\rmfamily} +\spn at wtheorem{conjecture}{Conjecture}{\itshape}{\rmfamily} +\spn at wtheorem{corollary}{Corollary}{\bfseries}{\itshape} +\spn at wtheorem{definition}{Definition}{\bfseries}{\itshape} +\spn at wtheorem{example}{Example}{\itshape}{\rmfamily} +\spn at wtheorem{exercise}{Exercise}{\itshape}{\rmfamily} +\spn at wtheorem{lemma}{Lemma}{\bfseries}{\itshape} +\spn at wtheorem{note}{Note}{\itshape}{\rmfamily} +\spn at wtheorem{problem}{Problem}{\itshape}{\rmfamily} +\spn at wtheorem{property}{Property}{\itshape}{\rmfamily} +\spn at wtheorem{proposition}{Proposition}{\bfseries}{\itshape} +\spn at wtheorem{question}{Question}{\itshape}{\rmfamily} +\spn at wtheorem{solution}{Solution}{\itshape}{\rmfamily} +\spn at wtheorem{remark}{Remark}{\itshape}{\rmfamily} + +\def\@takefromreset#1#2{% + \def\@tempa{#1}% + \let\@tempd\@elt + \def\@elt##1{% + \def\@tempb{##1}% + \ifx\@tempa\@tempb\else + \@addtoreset{##1}{#2}% + \fi}% + \expandafter\expandafter\let\expandafter\@tempc\csname cl@#2\endcsname + \expandafter\def\csname cl@#2\endcsname{}% + \@tempc + \let\@elt\@tempd} + +\def\theopargself{\def\@spopargbegintheorem##1##2##3##4##5{\trivlist + \item[\hskip\labelsep{##4##1\ ##2}]{##4##3\@thmcounterend\ }##5} + \def\@Opargbegintheorem##1##2##3##4{##4\trivlist + \item[\hskip\labelsep{##3##1}]{##3##2\@thmcounterend\ }} + } + +\renewenvironment{abstract}{% + \list{}{\advance\topsep by0.35cm\relax\small + \leftmargin=1cm + \labelwidth=\z@ + \listparindent=\z@ + \itemindent\listparindent + \rightmargin\leftmargin}\item[\hskip\labelsep + \bfseries\abstractname]} + {\endlist} + +\newdimen\headlineindent % dimension for space between +\headlineindent=1.166cm % number and text of headings. + +\def\ps at headings{\let\@mkboth\@gobbletwo + \let\@oddfoot\@empty\let\@evenfoot\@empty + \def\@evenhead{\normalfont\small\rlap{\thepage}\hspace{\headlineindent}% + \leftmark\hfil} + \def\@oddhead{\normalfont\small\hfil\rightmark\hspace{\headlineindent}% + \llap{\thepage}} + \def\chaptermark##1{}% + \def\sectionmark##1{}% + \def\subsectionmark##1{}} + +\def\ps at titlepage{\let\@mkboth\@gobbletwo + \let\@oddfoot\@empty\let\@evenfoot\@empty + \def\@evenhead{\normalfont\small\rlap{\thepage}\hspace{\headlineindent}% + \hfil} + \def\@oddhead{\normalfont\small\hfil\hspace{\headlineindent}% + \llap{\thepage}} + \def\chaptermark##1{}% + \def\sectionmark##1{}% + \def\subsectionmark##1{}} + +\if at runhead\ps at headings\else +\ps at empty\fi + +\setlength\arraycolsep{1.4\p@} +\setlength\tabcolsep{1.4\p@} + +\endinput +%end of file llncs.cls From cfbolz at codespeak.net Sun May 27 00:20:03 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 27 May 2007 00:20:03 +0200 (CEST) Subject: [pypy-svn] r43705 - pypy/extradoc/talk/dyla2007 Message-ID: <20070526222003.14C968082@code0.codespeak.net> Author: cfbolz Date: Sun May 27 00:20:03 2007 New Revision: 43705 Modified: pypy/extradoc/talk/dyla2007/dyla.bib pypy/extradoc/talk/dyla2007/dyla.tex Log: fix some of the references Modified: pypy/extradoc/talk/dyla2007/dyla.bib ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.bib (original) +++ pypy/extradoc/talk/dyla2007/dyla.bib Sun May 27 00:20:03 2007 @@ -40,42 +40,6 @@ note = "http://codespeak.net/pypy" } - at techreport{ dynamiclang, - title = "Compiling Dynamic Language Implementations", - author = "Armin Rigo and Michael Hudson and Samuele Pedroni", - year = "2005", - institution = "PyPy Consortium", - url = "http://codespeak.net/svn/pypy/extradoc/eu-report/D05.1_Publish_on_translating_a_very-high-level_description.pdf", - note = "http://codespeak.net/svn/pypy/extradoc/eu-report/" -} - - at techreport{ completepythonimpl, - title = "Complete Python Implementation running on top of CPython", - author = "{PyPy Team}", - year = "2005", - institution = "PyPy Consortium", - url = "http://codespeak.net/svn/pypy/extradoc/eu-report/D04.2_Complete_Python_Implementation_on_top_of_CPython.pdf", - note = "http://codespeak.net/pypy/dist/pypy/doc/index-report.html" -} - - at techreport{ translationaspects, - title = "Memory management and threading models as translation aspects -- solutions and challenges", - author = "Carl Friedrich Bolz and Armin Rigo", - year = "2005", - institution = "PyPy Consortium", - url = "http://codespeak.net/svn/pypy/extradoc/eu-report/D05.3_Publish_on_implementation_with_translation_aspects.pdf", - note = "http://codespeak.net/pypy/dist/pypy/doc/index-report.html" -} - -# XXX fix links and title - at techreport{ optimizations, - title = "DRAFT: Support for Massive Parallelism, Optimisation results, Practical Usages and Approaches for Translation Aspects", - author = "Carl Friedrich Bolz and Armin Rigo", - year = "2006", - institution = "PyPy Consortium", - url = "http://codespeak.net/svn/pypy/extradoc/eu-report/D07.1_Massive_Parallelism_and_Translation_Aspects-interim-2006-07-25.pdf", - note = "Interim version at: http://codespeak.net/pypy/dist/pypy/doc/index-report.html" -} @misc{ translationdoc, title = "{PyPy - Translation}", @@ -110,7 +74,7 @@ year = 2006 } - at InProceedings{ LLVM:CGO04, + at InProceedings{LLVM, author = "Chris Lattner and Vikram Adve", title = "{LLVM: A Compilation Framework for Lifelong Program Analysis \& Transformation}", booktitle = "{Proceedings of the 2004 International Symposium on Code Generation and Optimization (CGO'04)}", @@ -119,7 +83,13 @@ year = "2004" } - + at InProceedings{JikesGC, + author = "Stephen M Blackburn and Perry Cheng and Kathryn S McKinley", + title = "Oil and Water? High Performance Garbage Collection in Java with MMTk", + booktitle = "ICSE 2004, 26th International Conference on Software Engineering", + year = 2004 +} +\newblock In {\em ICSE}, pages 137--146, 2004. @inproceedings{DBLP:conf/jelia/Calejo04, author = {Miguel Calejo}, title = {Inter{P}rolog: Towards a Declarative Embedding of Logic Programming @@ -153,6 +123,14 @@ note = "http://www.hpl.hp.com/personal/Hans\_Boehm/gc/" } + at misc{stackless, + author = "Christian Tismer", + title = "Stackless Python", + url = "http://www.stackless.com/", + note = "http://www.stackless.com/" +} + + @misc{CLI, author="ECMA", title="Standard ECMA-335: Common Language Infrastructure", @@ -243,3 +221,76 @@ } + at techreport{security, + author = "Michael Franz", + title = "Moving Trust Out of Application Programs: A Software Architecture Based on Multi-Level Security Virtual Machines", + year = 2006, + month = August, + number = "06-10", + institution = "Donald Bren School of Information and Computer Science, University of California, Irvine" +} + + + +% PyPy eu reports + + + at techreport{ D05.1, + title = "Compiling Dynamic Language Implementations", + author = "Armin Rigo and Michael Hudson and Samuele Pedroni", + year = "2005", + institution = "PyPy Consortium", + number = "D05.1", + url = "http://codespeak.net/svn/pypy/extradoc/eu-report/D05.1_Publish_on_translating_a_very-high-level_description.pdf", + note = "http://codespeak.net/svn/pypy/extradoc/eu-report/" +} + + at techreport{ D04.2, + title = "Complete Python Implementation running on top of CPython", + author = "{PyPy Team}", + year = "2005", + institution = "PyPy Consortium", + number = "D04.2", + url = "http://codespeak.net/svn/pypy/extradoc/eu-report/D04.2_Complete_Python_Implementation_on_top_of_CPython.pdf", + note = "http://codespeak.net/pypy/dist/pypy/doc/index-report.html" +} + + at techreport{ D05.3, + title = "Memory management and threading models as translation aspects -- solutions and challenges", + author = "Carl Friedrich Bolz and Armin Rigo", + year = "2005", + institution = "PyPy Consortium", + number = "D05.3", + url = "http://codespeak.net/svn/pypy/extradoc/eu-report/D05.3_Publish_on_implementation_with_translation_aspects.pdf", + note = "http://codespeak.net/pypy/dist/pypy/doc/index-report.html" +} + + at techreport{ D07.1, + title = "Support for Massive Parallelism, Optimisation results, Practical Usages and Approaches for Translation Aspects", + author = "Carl Friedrich Bolz and Armin Rigo", + year = "2007", + institution = "PyPy Consortium", + number = "D07.1", + note = "http://codespeak.net/pypy/dist/pypy/doc/index-report.html" +} + + at techreport{ D06.1, + title = "Core Object Optimization Results", + author = "Michael Hudson and Armin Rigo and Carl Friedrich Bolz", + year = "2007", + institution = "PyPy Consortium", + number = "D06.1", + note = "http://codespeak.net/pypy/dist/pypy/doc/index-report.html" +} + + at techreport{ D12.1, + title = "High-Level Backends and Interpreter Feature Prototypes", + author = "Antonio Cuni and Samuele Pedroni and Anders Chrigstr?m and +Holger Krekel and Guido Wesdorp and Carl Friedrich Bolz", + year = "2007", + institution = "PyPy Consortium", + number = "D12.1", + note = "http://codespeak.net/pypy/dist/pypy/doc/index-report.html" +} + + Modified: pypy/extradoc/talk/dyla2007/dyla.tex ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.tex (original) +++ pypy/extradoc/talk/dyla2007/dyla.tex Sun May 27 00:20:03 2007 @@ -333,21 +333,21 @@ \item this meta-programming component of PyPy is called the \emph{translation -framework,} as it translates RPython source code (i.e. the full Python +framework}, as it translates RPython source code (i.e. the full Python interpreter) into lower-level code. Its purpose is to add aspects to and specialize the interpreter to fit a selectable virtual or hardware runtime environment. This either turns the interpreter into a standalone virtual machine, or integrates it into an existing OO VM. The necessary support code - e.g. the garbage collector when targeting C - is itself written in RPython in much the same spirit -that the Jikes RVM's GCs are written in Java [JIKES]; as needed, it is +that the Jikes RVM's GCs are written in Java \cite{JikesGC}; as needed, it is translated together with the interpreter to form the final custom VM. \end{itemize} A detailed description of this translation process is beyond the scope of the present paper; it can be found in \cite{pypyvmconstruction}. The actual Python interpreter of PyPy and the results we achieved by translating it to C, LLVM -\ref{LLVM:CGO04} and .NET are described in [XXX]. These results show that the +\cite{LLVM} and .NET are described in [XXX]. These results show that the approach is practical and gives results whose performance is within the same order of magnitude (within a factor of 2 and improving) of the hand-written, well-tuned CPython, the C reference implementation. @@ -380,8 +380,8 @@ - a 150-lines extension in PyPy that would require global changes in CPython. Further examples can be found in our technical reports; we should notably mention an extension adding a state-of-the-art security -model for Python based on data flow tacking [XXX], and general -performance improvements found by extensive experimentation [XXX], some +model for Python based on data flow tacking \cite{D12.1}, and general +performance improvements found by extensive experimentation \cite{D06.1}, some of which were back-ported to CPython. If we compare with hand-writing an implementation for a specific OO VM, @@ -423,7 +423,7 @@ impossible without forking the whole source code base. In fact, there is a well-known example of a CPython fork: Stackless -Python [XXX], which adds support for coroutines. This is very hard to +Python \cite{stackless}, which adds support for coroutines. This is very hard to do in CPython because the interpreter is written in a highly recursive style. Stackless Python required large-scale changes; it is not merged back into CPython due to the pervasive increase in complexity that it @@ -432,7 +432,7 @@ recursive style - into an efficient variant of continuation-passing style (CPS), enabling the usage of coroutines in the translated interpreter. For more details and other examples of translation-level -transformations, see [XXX]. +transformations, see \cite{D07.1}. \subsection{Getting good GCs and tools is possible} From cfbolz at codespeak.net Sun May 27 00:28:44 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 27 May 2007 00:28:44 +0200 (CEST) Subject: [pypy-svn] r43706 - pypy/extradoc/talk/dyla2007 Message-ID: <20070526222844.590468082@code0.codespeak.net> Author: cfbolz Date: Sun May 27 00:28:43 2007 New Revision: 43706 Modified: pypy/extradoc/talk/dyla2007/dyla.bib pypy/extradoc/talk/dyla2007/dyla.tex Log: use proper quotes and long dashes Modified: pypy/extradoc/talk/dyla2007/dyla.bib ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.bib (original) +++ pypy/extradoc/talk/dyla2007/dyla.bib Sun May 27 00:28:43 2007 @@ -285,8 +285,8 @@ @techreport{ D12.1, title = "High-Level Backends and Interpreter Feature Prototypes", - author = "Antonio Cuni and Samuele Pedroni and Anders Chrigstr?m and -Holger Krekel and Guido Wesdorp and Carl Friedrich Bolz", + author = {Antonio Cuni and Samuele Pedroni and Anders Chrigstr\"om and +Holger Krekel and Guido Wesdorp and Carl Friedrich Bolz}, year = "2007", institution = "PyPy Consortium", number = "D12.1", Modified: pypy/extradoc/talk/dyla2007/dyla.tex ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.tex (original) +++ pypy/extradoc/talk/dyla2007/dyla.tex Sun May 27 00:28:43 2007 @@ -41,7 +41,7 @@ virtual machines instead of writing them by hand. This approach has been validated by the PyPy project, with which we can automatically insert many features into the -generated virtual machines - including good just-in-time compilers tuned +generated virtual machines ? including good just-in-time compilers tuned to the dynamic language at hand. \footnote{This research is partially supported by the EU funded %research project: @@ -78,13 +78,13 @@ of the C implementation lead to alternative implementations which draw work-power from the reference implementation. An alternative to writing implementations in C is to build them on top of one of the newer object oriented -virtual machines ("OO VM") such as the JVM or the CLR. This is often wanted by +virtual machines (``OO VM'') such as the JVM or the CLR. This is often wanted by the community anyway, since it leads to better re-usability of libraries of these platforms. However, if a C implementation existed before the implementation of such a VM is started, this enters in conflict with the goal of having to maintain essentially a single, simple enough implementation for a given programming language: as the language becomes popular, there will be a -demand for having it run on various platforms - high-level VMs as well as +demand for having it run on various platforms ? high-level VMs as well as C-level environments. The argument we will make in the present paper is that it is possible to @@ -94,7 +94,7 @@ high-level but less dynamic language. This interpreter plays the role of a specification for the dynamic language. With a good enough translation toolchain we can then generate whole virtual machines from -this specification - either full custom VMs for C-level operating +this specification ? either full custom VMs for C-level operating systems, or layers on top of various OO VMs. In other words, meta-programming techniques can be used to successfully replace a foreseeable one-VM-fits-all standardization attempt. @@ -148,9 +148,9 @@ These limitations of a C-based implementation often lead to more implementations of the language. For example in addition to the main C -implementation of Python ("CPython") there are a number of additional +implementation of Python (``CPython'') there are a number of additional implementations that want to fix some of the limitations of CPython. One of -them is "Stackless Python" which is a fork of CPython that adds micro-threading +them is ``Stackless Python'' which is a fork of CPython that adds micro-threading capabilities to Python. One of the reasons for not incorporating it back into CPython was that it was felt that they would make the implementation too complex. Another implementation of the Python language is the Psyco project @@ -285,11 +285,11 @@ The present paper proposes to approach the implementation of dynamic languages from a meta-level: virtual machines for such languages should -not be written by hand, but generated automatically "around" a +not be written by hand, but generated automatically ``around'' a description of the language in the form of an interpreter for it. We argue that this approach gives many of the benefits usually expected by an implementer when he decides to target an existing object-oriented -virtual machine. It also gives other benefits that we will describe - +virtual machine. It also gives other benefits that we will describe ? mostly in term of flexibility. But most importantly, it lets a community write a single source implementation of the language, avoiding the time-consuming task of keeping multiple ones in sync. The single @@ -316,13 +316,13 @@ \begin{itemize} \item -we use a very expressive \emph{object language} (RPython - an analyzable +we use a very expressive \emph{object language} (RPython ? an analyzable subset of Python) as the language in which the complete Python interpreter is written, together with the implementation of its built-in types. The language is still close to Python, e.g. it is object-oriented, provides rich built-in types and has automatic memory management. In other words, the source code of our complete Python -interpreter is mostly free of low-level details - no explicit memory +interpreter is mostly free of low-level details ? no explicit memory management, no pieces of C or C-level code. \item @@ -338,8 +338,8 @@ and specialize the interpreter to fit a selectable virtual or hardware runtime environment. This either turns the interpreter into a standalone virtual machine, or integrates it into an existing OO VM. -The necessary support code - e.g. the garbage collector when -targeting C - is itself written in RPython in much the same spirit +The necessary support code ? e.g. the garbage collector when +targeting C ? is itself written in RPython in much the same spirit that the Jikes RVM's GCs are written in Java \cite{JikesGC}; as needed, it is translated together with the interpreter to form the final custom VM. \end{itemize} @@ -354,10 +354,10 @@ \subsection{A single source} -Our approach - a single "meta-written" implementation - naturally leads +Our approach ? a single ``meta-written'' implementation ? naturally leads to language implementations that have various advantages over the -"hand-written" implementations. First of all, it is a single-source -approach - we explicitly seek to solve the problem of proliferation of +``hand-written'' implementations. First of all, it is a single-source +approach ? we explicitly seek to solve the problem of proliferation of implementations. In the sequel we will show more precise evidence that this can be done in a practical way with no major drawback. By itself this would already be a valid argument against the need for @@ -377,7 +377,7 @@ or implementation techniques in ways that would, in a traditional approach, require pervasive changes to the source code. For example, PyPy's Python interpreter can optionally provide lazily computed objects -- a 150-lines extension in PyPy that would require global changes in +? a 150-lines extension in PyPy that would require global changes in CPython. Further examples can be found in our technical reports; we should notably mention an extension adding a state-of-the-art security model for Python based on data flow tacking \cite{D12.1}, and general @@ -386,7 +386,7 @@ If we compare with hand-writing an implementation for a specific OO VM, then the latter requires not only good knowledge of the OO VM in -question and its object model - it requires the language implementer to +question and its object model ? it requires the language implementer to fit the language into the imposed models. For example, it is natural to map classes and instances of a dynamic object-oriented language to the OO VM's notion of classes and instances, but this might not be a simple @@ -395,7 +395,7 @@ approach, this efforts is done at two levels: in a first step, while writing the interpreter, the implementer does not need to worry about integration with an OO VM's object model. Of course, the integration -effort does not simply vanish - indeed, a simple translation of such an +effort does not simply vanish ? indeed, a simple translation of such an interpreter to a given OO VM would give an interpreter for the dynamic language which is unable to communicate with the host VM (which might already be interesting in specific cases, but not in general). @@ -406,17 +406,17 @@ point is that it has proven possible to take the single, OO VM-independent source code of PyPy's Python interpreter, and produce from it a version that runs in and integrates with the rest of the .NET -environment - by writing \emph{orthogonal} code. +environment ? by writing \emph{orthogonal} code. At the level of the translation framework, the ability to change or insert new whole-program transformations makes some aspects of the -interpreter easier to deal with. By "aspect" we mean, in the original +interpreter easier to deal with. By ``aspect'' we mean, in the original AOP sense, a feature that is added to an object program by a meta-program. The most obvious example in our context is the garbage collector for the target environments that lack it. The hand-written C source of CPython is littered with macro calls that increment and decrement reference counters. Our translation framework can insert such -macro calls automatically - in fact, we have a choice of GCs, and +macro calls automatically ? in fact, we have a choice of GCs, and reference counting is only one of them (not a particularly efficient one, either). Some GCs require different transformations of the code. By contrast, supporting more than one GC in CPython is close to @@ -427,15 +427,20 @@ do in CPython because the interpreter is written in a highly recursive style. Stackless Python required large-scale changes; it is not merged back into CPython due to the pervasive increase in complexity that it -requires. In PyPy, though, an optional "stackless" transformation is -able to turn the Python interpreter - also written in a simple highly -recursive style - into an efficient variant of continuation-passing +requires. In PyPy, though, an optional ``stackless'' transformation is +able to turn the Python interpreter ? also written in a simple highly +recursive style ? into an efficient variant of continuation-passing style (CPS), enabling the usage of coroutines in the translated interpreter. For more details and other examples of translation-level transformations, see \cite{D07.1}. \subsection{Getting good GCs and tools is possible} +XXX + +\section{Related Work} +XXX +\section{Conclusion} XXX % ---- Bibliography ---- %\begin{small} From arigo at codespeak.net Sun May 27 12:14:21 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 27 May 2007 12:14:21 +0200 (CEST) Subject: [pypy-svn] r43708 - pypy/extradoc/talk/dyla2007 Message-ID: <20070527101421.7F45080B1@code0.codespeak.net> Author: arigo Date: Sun May 27 12:14:20 2007 New Revision: 43708 Modified: pypy/extradoc/talk/dyla2007/dyla.tex Log: Details in section 2. Started reworking and completing section 3, intermediate check-in. Modified: pypy/extradoc/talk/dyla2007/dyla.tex ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.tex (original) +++ pypy/extradoc/talk/dyla2007/dyla.tex Sun May 27 12:14:20 2007 @@ -206,7 +206,7 @@ of the VM. \emph{Ease of implementation:} The implementation of a language on top of an OO -VM is easier because it starts of at a higher level than C. Usually a +VM is easier because it starts at a higher level than C. Usually a high-level language like Java or C\# is used for the language implementation, which both offer the language implementer a much higher level of abstraction than when implementing in C. @@ -219,11 +219,14 @@ At a closer look, some of these advantages are only partially true in practice. -\emph{Better performance:} So far it seems like performance of highly dynamic -languages is not actually significantly improved on OO VMs. Jython is around 5 -times slower than CPython, for IronPython the figures vary but it is mostly +\emph{Better performance:} So far it seems that performance of highly dynamic +languages is not actually significantly improved on OO VMs. Jython +\footnote{Python on the Java VM} is around 5 +times slower than CPython, for IronPython\footnote{Python on .NET, which +gives up on some features to improve performance} +the figures vary but it is mostly within the same order of magnitude as CPython. The most important reason for -this that the VM's JIT compilers are not prepared to deal with the highly +this is that the VM's JIT compilers are not prepared to deal with the highly dynamic behaviour and the complex dispatch semantics of dynamic languages. (XXX expand) @@ -289,7 +292,7 @@ description of the language in the form of an interpreter for it. We argue that this approach gives many of the benefits usually expected by an implementer when he decides to target an existing object-oriented -virtual machine. It also gives other benefits that we will describe ? +virtual machine. It also gives other benefits that we will describe -- mostly in term of flexibility. But most importantly, it lets a community write a single source implementation of the language, avoiding the time-consuming task of keeping multiple ones in sync. The single @@ -316,13 +319,13 @@ \begin{itemize} \item -we use a very expressive \emph{object language} (RPython ? an analyzable +we use a very expressive \emph{object language} (RPython -- an analyzable subset of Python) as the language in which the complete Python interpreter is written, together with the implementation of its built-in types. The language is still close to Python, e.g. it is object-oriented, provides rich built-in types and has automatic memory management. In other words, the source code of our complete Python -interpreter is mostly free of low-level details ? no explicit memory +interpreter is mostly free of low-level details -- no explicit memory management, no pieces of C or C-level code. \item @@ -338,8 +341,8 @@ and specialize the interpreter to fit a selectable virtual or hardware runtime environment. This either turns the interpreter into a standalone virtual machine, or integrates it into an existing OO VM. -The necessary support code ? e.g. the garbage collector when -targeting C ? is itself written in RPython in much the same spirit +The necessary support code -- e.g. the garbage collector when +targeting C -- is itself written in RPython in much the same spirit that the Jikes RVM's GCs are written in Java \cite{JikesGC}; as needed, it is translated together with the interpreter to form the final custom VM. \end{itemize} @@ -354,18 +357,17 @@ \subsection{A single source} -Our approach ? a single ``meta-written'' implementation ? naturally leads +Our approach -- a single ``meta-written'' implementation -- naturally leads to language implementations that have various advantages over the ``hand-written'' implementations. First of all, it is a single-source -approach ? we explicitly seek to solve the problem of proliferation of -implementations. In the sequel we will show more precise evidence that -this can be done in a practical way with no major drawback. By itself -this would already be a valid argument against the need for -standardization on a single OO VM. But there are also other advantages -in generating language implementations. In our opinion these are very -significant, to the extent that it hints that meta-programming, though -not widely used in general-purpose programming, is an essential tool in -a language implementer's toolbox. +approach -- we explicitly seek to solve the problem of proliferation of +implementations. + +Separating the implementation of a language in a high-level +``description'' and a custom translation framework has also many +advantages -- in our opinion significant enough to hint that +meta-programming, though not widely used in general-purpose programming, +is an essential tool in a language implementer's toolbox. \subsection{Writing the interpreter is easier} @@ -377,36 +379,27 @@ or implementation techniques in ways that would, in a traditional approach, require pervasive changes to the source code. For example, PyPy's Python interpreter can optionally provide lazily computed objects -? a 150-lines extension in PyPy that would require global changes in +-- a 150-lines extension in PyPy that would require global changes in CPython. Further examples can be found in our technical reports; we should notably mention an extension adding a state-of-the-art security model for Python based on data flow tacking \cite{D12.1}, and general performance improvements found by extensive experimentation \cite{D06.1}, some of which were back-ported to CPython. -If we compare with hand-writing an implementation for a specific OO VM, -then the latter requires not only good knowledge of the OO VM in -question and its object model ? it requires the language implementer to -fit the language into the imposed models. For example, it is natural to -map classes and instances of a dynamic object-oriented language to the -OO VM's notion of classes and instances, but this might not be a simple -task at all if the models are substantially different and/or if the OO -VM is essentially less dynamic than the language to implement. In our -approach, this efforts is done at two levels: in a first step, while -writing the interpreter, the implementer does not need to worry about -integration with an OO VM's object model. Of course, the integration -effort does not simply vanish ? indeed, a simple translation of such an -interpreter to a given OO VM would give an interpreter for the dynamic -language which is unable to communicate with the host VM (which might -already be interesting in specific cases, but not in general). -Integration comes as a second step, and occurs at a different level, by -introducing mappings between the relevant classes of the interpreter and -the corresponding classes of the OO VM. As of yet we have no evidence -that this makes the total integration effort much lower or higher; the -point is that it has proven possible to take the single, OO -VM-independent source code of PyPy's Python interpreter, and produce -from it a version that runs in and integrates with the rest of the .NET -environment ? by writing \emph{orthogonal} code. +\subsection{The effort of writing a translation toolchain} + +Of course, the price to pay is the need for a translation toolchain +capable of analyzing and transforming the high-level source code and +generating lower-level output in various languages. Of course, the +translation toolchain, once written, can be reused to implement other +languages as well. Even so, we found that the required effort that must +be put into the translation toolchain in the first place is still much +lower than that of writing a complete, commercial-quality OO VM. A +reason appears to be that we could design our translation toolchain +specifically for our needs, i.e. a language implementer's needs, instead +of for general-purpose usage. + +... At the level of the translation framework, the ability to change or insert new whole-program transformations makes some aspects of the @@ -416,7 +409,7 @@ collector for the target environments that lack it. The hand-written C source of CPython is littered with macro calls that increment and decrement reference counters. Our translation framework can insert such -macro calls automatically ? in fact, we have a choice of GCs, and +macro calls automatically -- in fact, we have a choice of GCs, and reference counting is only one of them (not a particularly efficient one, either). Some GCs require different transformations of the code. By contrast, supporting more than one GC in CPython is close to @@ -428,12 +421,41 @@ style. Stackless Python required large-scale changes; it is not merged back into CPython due to the pervasive increase in complexity that it requires. In PyPy, though, an optional ``stackless'' transformation is -able to turn the Python interpreter ? also written in a simple highly -recursive style ? into an efficient variant of continuation-passing +able to turn the Python interpreter -- also written in a simple highly +recursive style -- into an efficient variant of continuation-passing style (CPS), enabling the usage of coroutines in the translated interpreter. For more details and other examples of translation-level transformations, see \cite{D07.1}. + +\subsection{Integration with a host OO VM} + +A good example of this is to compare it against the task of hand-writing +an implementation for a specific, choosen OO VM. The latter requires +not only good knowledge of the OO VM in question and its object model -- +it requires the language implementer to fit the language into the +imposed models. For example, it is natural to map classes and instances +of a dynamic object-oriented language to the OO VM's notion of classes +and instances, but this might not be a simple task at all if the models +are substantially different and/or if the OO VM is essentially less +dynamic than the language to implement. In our approach, this task +is done at two levels: in a first step, while writing the interpreter, +the implementer does not need to worry about integration with an OO VM's +object model. Of course, the integration effort does not simply vanish +-- indeed, a simple translation of such an interpreter to a given OO VM +would give an interpreter for the dynamic language which is unable to +communicate with the host VM (which might already be interesting in +specific cases, but not in general). Integration comes as a second +step, and occurs at a different level, by introducing mappings between +the relevant classes of the interpreter and the corresponding classes of +the OO VM. As of yet we have no evidence that this makes the total +integration effort much lower or higher; the point is that it has proven +possible to take the single, OO VM-independent source code of PyPy's +Python interpreter, and produce from it a version that runs in and +integrates with the rest of the .NET environment -- by writing +\emph{orthogonal} code. + + \subsection{Getting good GCs and tools is possible} XXX From cfbolz at codespeak.net Sun May 27 13:46:06 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 27 May 2007 13:46:06 +0200 (CEST) Subject: [pypy-svn] r43709 - pypy/extradoc/talk/dyla2007 Message-ID: <20070527114606.9576B80AA@code0.codespeak.net> Author: cfbolz Date: Sun May 27 13:46:05 2007 New Revision: 43709 Modified: pypy/extradoc/talk/dyla2007/dyla.tex Log: removing some XXXs Modified: pypy/extradoc/talk/dyla2007/dyla.tex ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.tex (original) +++ pypy/extradoc/talk/dyla2007/dyla.tex Sun May 27 13:46:05 2007 @@ -106,12 +106,15 @@ and concrete benefits in term of flexibility and performance that go beyond the state of the art. -In section XXX we will explore the way VMs are typically implemented -in C and some of the problems of this approach. In section XXX we will describe -our proposed meta-programming approach. XXX +In section \ref{sect:approaches} we will explore the way VMs are typically +implemented in C and on top of OO VMs and some of the problems of these +approaches. In \ref{section sect:metaprogramming} we will describe our proposed +meta-programming approach. -\section{XXX XXX} +\section{Approaches to Dynamic Language Implementation} +\label{sect:appraches} + Dynamic languages are traditionally implemented by writing a virtual machine for them in a low-level language like C, or in a language that @@ -167,14 +170,17 @@ this is an incomplete solution since it prevents many use-cases. Therefore in the end implementations of the dynamic language on top of the object-oriented VMs are created to properly integrate with the complete -environment. Implementing a language on top of an OO VM is in many ways easier -than implementing it in C. +environment. \subsection{Assessing the Advantages of Implementing a Language on Top of OO VMs} -Let's take a look at the advantages that are usually cited for basing a +XXX mention that OO languages don't have good metaprogramming support + +Implementing a language on top of an OO VM is in many ways easier than +implementing it in C. Let's take a look at the advantages that are usually +cited for basing a language implementation of a dynamic language on a standard object oriented virtual machine, for example the JVM or the CLR. The central theme of the benefits of OO VMs is the ability to implement certain hard things only once @@ -191,7 +197,7 @@ various hardware architectures and operating systems. The languages implemented on top can then be run without change in various environments. -\emph{Better tools:} XXX +\emph{Better tools:} Better IDEs, debuggers and profilers. \emph{Better implementation of low-level issues like garbage collection, threading:} Since an OO VM is expected to be widely used and usually backed by @@ -226,9 +232,10 @@ gives up on some features to improve performance} the figures vary but it is mostly within the same order of magnitude as CPython. The most important reason for -this is that the VM's JIT compilers are not prepared to deal with the highly -dynamic behaviour and the complex dispatch semantics of dynamic languages. (XXX -expand) +this is that the VM's JIT compilers are optimized for specific usage patterns +that are common in the main language of the OO VM. To get good speeds the +language implementers would have to carefully produce code that matches these +usage patterns, which is not a simple task. \emph{Better GCs:} While this is obvious in theory, OO VMs tend to have a much higher memory overhead to start with (XXX ref) @@ -285,6 +292,7 @@ of implementations. \section{Meta-Programming Is Good} +\label{sect:metaprogramming} The present paper proposes to approach the implementation of dynamic languages from a meta-level: virtual machines for such languages should From arigo at codespeak.net Sun May 27 13:54:22 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 27 May 2007 13:54:22 +0200 (CEST) Subject: [pypy-svn] r43710 - pypy/extradoc/talk/dyla2007 Message-ID: <20070527115422.614C780B4@code0.codespeak.net> Author: arigo Date: Sun May 27 13:54:20 2007 New Revision: 43710 Modified: pypy/extradoc/talk/dyla2007/dyla.tex Log: More reorganization, and finished another subsection. Modified: pypy/extradoc/talk/dyla2007/dyla.tex ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.tex (original) +++ pypy/extradoc/talk/dyla2007/dyla.tex Sun May 27 13:54:20 2007 @@ -296,8 +296,8 @@ The present paper proposes to approach the implementation of dynamic languages from a meta-level: virtual machines for such languages should -not be written by hand, but generated automatically ``around'' a -description of the language in the form of an interpreter for it. We +not be written by hand, but generated automatically ``around'' an +interpreter playing the role of a high-level description of the language. We argue that this approach gives many of the benefits usually expected by an implementer when he decides to target an existing object-oriented virtual machine. It also gives other benefits that we will describe -- @@ -365,17 +365,16 @@ \subsection{A single source} -Our approach -- a single ``meta-written'' implementation -- naturally leads -to language implementations that have various advantages over the +Our approach -- a single ``meta-written'' implementation -- naturally +leads to language implementations that have various advantages over the ``hand-written'' implementations. First of all, it is a single-source approach -- we explicitly seek to solve the problem of proliferation of -implementations. - -Separating the implementation of a language in a high-level -``description'' and a custom translation framework has also many -advantages -- in our opinion significant enough to hint that -meta-programming, though not widely used in general-purpose programming, -is an essential tool in a language implementer's toolbox. +implementations. In the sequel, we will show that this goal can be +achieved without giving up on the advantages of hand-written +implementations for OO VMs. Moreover, there are additional advantages +-- in our opinion significant enough to hint that meta-programming, +though not widely used in general-purpose programming, is an essential +tool in a language implementer's toolbox. \subsection{Writing the interpreter is easier} @@ -394,78 +393,77 @@ performance improvements found by extensive experimentation \cite{D06.1}, some of which were back-ported to CPython. -\subsection{The effort of writing a translation toolchain} - -Of course, the price to pay is the need for a translation toolchain -capable of analyzing and transforming the high-level source code and -generating lower-level output in various languages. Of course, the -translation toolchain, once written, can be reused to implement other -languages as well. Even so, we found that the required effort that must -be put into the translation toolchain in the first place is still much -lower than that of writing a complete, commercial-quality OO VM. A -reason appears to be that we could design our translation toolchain -specifically for our needs, i.e. a language implementer's needs, instead -of for general-purpose usage. - -... +\subsection{Separation of concerns} At the level of the translation framework, the ability to change or insert new whole-program transformations makes some aspects of the interpreter easier to deal with. By ``aspect'' we mean, in the original AOP sense, a feature that is added to an object program by a -meta-program. The most obvious example in our context is the garbage -collector for the target environments that lack it. The hand-written C -source of CPython is littered with macro calls that increment and -decrement reference counters. Our translation framework can insert such -macro calls automatically -- in fact, we have a choice of GCs, and -reference counting is only one of them (not a particularly efficient -one, either). Some GCs require different transformations of the code. -By contrast, supporting more than one GC in CPython is close to -impossible without forking the whole source code base. - -In fact, there is a well-known example of a CPython fork: Stackless -Python \cite{stackless}, which adds support for coroutines. This is very hard to -do in CPython because the interpreter is written in a highly recursive -style. Stackless Python required large-scale changes; it is not merged -back into CPython due to the pervasive increase in complexity that it -requires. In PyPy, though, an optional ``stackless'' transformation is -able to turn the Python interpreter -- also written in a simple highly -recursive style -- into an efficient variant of continuation-passing -style (CPS), enabling the usage of coroutines in the translated -interpreter. For more details and other examples of translation-level -transformations, see \cite{D07.1}. +meta-program. The most obvious example in our context is the insertion +of a garbage collector (chosen among several available ones) for the +target environments that lack it. Another example is the translation of +the interpreter into a form of continuation-passing style (CPS), which +allows the translated interpreter to provide coroutines even though its +source is written in a simple highly recursive style. For more details +and other examples of translation-level transformations, see +\cite{D07.1}. + +A more subtle example of separation of concerns is the way our generated +implementations can be integrated with a host OO VM. As mentioned +above, an implementer deciding to directly target a specific OO VM needs +not only good knowledge of the OO VM in question and its object model -- +he must fit the language into the imposed models. Instead, in our +approach this task is done at two levels: in a first step, a stand-alone +interpreter is written -- which, if translated to a given OO VM, would +simply give an interpreter for the dynamic language which is unable to +communicate with the host VM. Integration comes as a second step, and +occurs at a different level, by introducing mappings between the +relevant classes of the interpreter and the corresponding classes of the +OO VM. +\subsection{The effort of writing a translation toolchain} -\subsection{Integration with a host OO VM} +What are the efforts required to develop a translation toolchain capable +of analyzing and transforming the high-level source code and generating +lower-level output in various languages? + +Although it is able to generate, among other things, a complete custom +VM for C-like environments, we found that the required effort that must +be put into the translation toolchain was still much lower than that of +writing a good-quality OO VM. A reason seems to be that we could design +our translation toolchain specifically for our needs, i.e. a language +implementer's needs, instead of for general-purpose usage. Of course, +the translation toolchain, once written, can also be reused to implement +other languages, and possibly tailored on a case-by-case basis to fit +each implementer's need. The process is incremental: we can add more +features as needed instead of starting from a maximal up-front design, +and gradually improve the quality of the tools, the garbage collectors, +the various optimizations, etc. + +Let us expand on the topic of the garbage collector, which for C-like +envrionments is inserted into the generated VM by a transformation step. +We started by ignoring the issue and just using the conservative Boehm +\cite{Boehm} collector for C. Later, we experimented with a range of +simple custom collectors - reference counting, mark-and-sweep, etc. +Ultimately, though, more advanced GCs will be needed to get the best +performance. It seems that RPython, enhanced with support for direct +address manipulations, is a good language for writing GCs, so it would +be possible for a GC expert to write one for our translation framework. +However, this is not the only way to obtain good GCs: we will soon +investigate a more practical course of action, which is to reuse +existing GCs. A good candidate is the GCs written in the Jikes RVM +\cite{JikesGC}. As they are in Java, it should be relatively +straightforward to add a translation step that turns one of them into +RPython (or directly our RPython-level intermediate representation) and +integrate it with the rest of the program being translated. + +In summary, developing a meta-programming translation toolchain requires +some work, but it can be done incrementally, it can reuse existing code, +and it gives a toolchain that is itself highly reusable and flexible in +nature. -A good example of this is to compare it against the task of hand-writing -an implementation for a specific, choosen OO VM. The latter requires -not only good knowledge of the OO VM in question and its object model -- -it requires the language implementer to fit the language into the -imposed models. For example, it is natural to map classes and instances -of a dynamic object-oriented language to the OO VM's notion of classes -and instances, but this might not be a simple task at all if the models -are substantially different and/or if the OO VM is essentially less -dynamic than the language to implement. In our approach, this task -is done at two levels: in a first step, while writing the interpreter, -the implementer does not need to worry about integration with an OO VM's -object model. Of course, the integration effort does not simply vanish --- indeed, a simple translation of such an interpreter to a given OO VM -would give an interpreter for the dynamic language which is unable to -communicate with the host VM (which might already be interesting in -specific cases, but not in general). Integration comes as a second -step, and occurs at a different level, by introducing mappings between -the relevant classes of the interpreter and the corresponding classes of -the OO VM. As of yet we have no evidence that this makes the total -integration effort much lower or higher; the point is that it has proven -possible to take the single, OO VM-independent source code of PyPy's -Python interpreter, and produce from it a version that runs in and -integrates with the rest of the .NET environment -- by writing -\emph{orthogonal} code. -\subsection{Getting good GCs and tools is possible} -XXX \section{Related Work} XXX From cfbolz at codespeak.net Sun May 27 14:35:08 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 27 May 2007 14:35:08 +0200 (CEST) Subject: [pypy-svn] r43715 - pypy/extradoc/talk/dyla2007 Message-ID: <20070527123508.527A280B2@code0.codespeak.net> Author: cfbolz Date: Sun May 27 14:34:59 2007 New Revision: 43715 Modified: pypy/extradoc/talk/dyla2007/dyla.bib pypy/extradoc/talk/dyla2007/dyla.tex Log: some more references, an XXX, a reformulation Modified: pypy/extradoc/talk/dyla2007/dyla.bib ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.bib (original) +++ pypy/extradoc/talk/dyla2007/dyla.bib Sun May 27 14:34:59 2007 @@ -57,13 +57,13 @@ howpublished = "Web page" } - at misc{ cpy243, + at misc{ cpy251, author = "Guido van Rossum et al.", - title = "{CPython} 2.4.3", - note = "http://www.python.org/download/releases/2.4.3/", - url = "http://www.python.org/download/releases/2.4.3/", - year = 2006, - month = "March" + title = "{CPython} 2.5.1", + note = "http://www.python.org/download/releases/2.5.1/", + url = "http://www.python.org/download/releases/2.5.1/", + year = 2007, + month = "April" } @misc{ pylangref, @@ -130,6 +130,13 @@ note = "http://www.stackless.com/" } + at misc{pythonnet, + author = "Brian Lloyd", + title = "Python for .NET", + url = "http://pythonnet.sourceforge.net/", + note = "http://pythonnet.sourceforge.net/" +} + @misc{CLI, author="ECMA", Modified: pypy/extradoc/talk/dyla2007/dyla.tex ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.tex (original) +++ pypy/extradoc/talk/dyla2007/dyla.tex Sun May 27 14:34:59 2007 @@ -108,13 +108,15 @@ In section \ref{sect:approaches} we will explore the way VMs are typically implemented in C and on top of OO VMs and some of the problems of these -approaches. In \ref{section sect:metaprogramming} we will describe our proposed +approaches, using various Python implementations as the main example. In +\ref{section sect:metaprogramming} we will describe our proposed meta-programming approach. \section{Approaches to Dynamic Language Implementation} \label{sect:appraches} +XXX the next paragraph is also the start of the introduction right now! Dynamic languages are traditionally implemented by writing a virtual machine for them in a low-level language like C, or in a language that @@ -149,24 +151,25 @@ % - wrapping with C libraries simple but tedious -These limitations of a C-based implementation often lead to more -implementations of the language. For example in addition to the main C -implementation of Python (``CPython'') there are a number of additional -implementations that want to fix some of the limitations of CPython. One of -them is ``Stackless Python'' which is a fork of CPython that adds micro-threading +These limitations of a C-based implementation often lead to more implementations +of the language. For example in addition to the main C implementation of Python +(``CPython'' \cite{cpy251}) there are a number of additional implementations +that want to fix some of the limitations of CPython. One of them is ``Stackless +Python'' \cite{stackless} which is a fork of CPython that adds micro-threading capabilities to Python. One of the reasons for not incorporating it back into CPython was that it was felt that they would make the implementation too -complex. Another implementation of the Python language is the Psyco project -which adds a JIT-compiler to CPython. As all other re-implementations, Psyco -contains the Python semantics and needs to be kept synchronized with CPython -manually. +complex. Another implementation of the Python language is the Psyco \cite{Psyco} +project which adds a JIT-compiler to CPython. As all other re-implementations, +Psyco contains the Python semantics and needs to be kept synchronized with +CPython manually. % - free threading attempts for Python As a language becomes more popular, yet another reason for different implementations emerges: People want to use it together with Java and C\# libraries and tools. Although it is in principle possible to do that by -bridging the C-VM with the respective VMs of these languages (XXX examples) +bridging the C-VM with the respective VMs of these languages (for example the +``PythonNet'' project \cite{pythonnet} that brigdes CPython with a .NET VM) this is an incomplete solution since it prevents many use-cases. Therefore in the end implementations of the dynamic language on top of the object-oriented VMs are created to properly integrate with the complete @@ -435,7 +438,7 @@ implementer's needs, instead of for general-purpose usage. Of course, the translation toolchain, once written, can also be reused to implement other languages, and possibly tailored on a case-by-case basis to fit -each implementer's need. The process is incremental: we can add more +the specific need of a language. The process is incremental: we can add more features as needed instead of starting from a maximal up-front design, and gradually improve the quality of the tools, the garbage collectors, the various optimizations, etc. From arigo at codespeak.net Sun May 27 15:03:19 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 27 May 2007 15:03:19 +0200 (CEST) Subject: [pypy-svn] r43720 - pypy/extradoc/talk/dyla2007 Message-ID: <20070527130319.E97C280B4@code0.codespeak.net> Author: arigo Date: Sun May 27 15:03:19 2007 New Revision: 43720 Modified: pypy/extradoc/talk/dyla2007/dyla.tex Log: A short dynamic compilers section. Modified: pypy/extradoc/talk/dyla2007/dyla.tex ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.tex (original) +++ pypy/extradoc/talk/dyla2007/dyla.tex Sun May 27 15:03:19 2007 @@ -364,7 +364,9 @@ \cite{LLVM} and .NET are described in [XXX]. These results show that the approach is practical and gives results whose performance is within the same order of magnitude (within a factor of 2 and improving) of the hand-written, -well-tuned CPython, the C reference implementation. +well-tuned CPython, the C reference implementation. These figures do not +include the spectacular speed-ups obtained in some cases by the JIT compiler +described in section \ref{subsect:dynamic_compilers}. \subsection{A single source} @@ -411,18 +413,18 @@ and other examples of translation-level transformations, see \cite{D07.1}. -A more subtle example of separation of concerns is the way our generated -implementations can be integrated with a host OO VM. As mentioned -above, an implementer deciding to directly target a specific OO VM needs -not only good knowledge of the OO VM in question and its object model -- -he must fit the language into the imposed models. Instead, in our -approach this task is done at two levels: in a first step, a stand-alone -interpreter is written -- which, if translated to a given OO VM, would -simply give an interpreter for the dynamic language which is unable to -communicate with the host VM. Integration comes as a second step, and -occurs at a different level, by introducing mappings between the -relevant classes of the interpreter and the corresponding classes of the -OO VM. +%A more subtle example of separation of concerns is the way our generated +%implementations can be integrated with a host OO VM. As mentioned +%above, an implementer deciding to directly target a specific OO VM needs +%not only good knowledge of the OO VM in question and its object model -- +%he must fit the language into the imposed models. Instead, in our +%approach this task is done at two levels: in a first step, a stand-alone +%interpreter is written -- which, if translated to a given OO VM, would +%simply give an interpreter for the dynamic language which is unable to +%communicate with the host VM. Integration comes as a second step, and +%occurs at a different level, by introducing mappings between the +%relevant classes of the interpreter and the corresponding classes of the +%OO VM. \subsection{The effort of writing a translation toolchain} @@ -433,43 +435,87 @@ Although it is able to generate, among other things, a complete custom VM for C-like environments, we found that the required effort that must be put into the translation toolchain was still much lower than that of -writing a good-quality OO VM. A reason seems to be that we could design -our translation toolchain specifically for our needs, i.e. a language -implementer's needs, instead of for general-purpose usage. Of course, -the translation toolchain, once written, can also be reused to implement -other languages, and possibly tailored on a case-by-case basis to fit -the specific need of a language. The process is incremental: we can add more -features as needed instead of starting from a maximal up-front design, -and gradually improve the quality of the tools, the garbage collectors, -the various optimizations, etc. - -Let us expand on the topic of the garbage collector, which for C-like -envrionments is inserted into the generated VM by a transformation step. -We started by ignoring the issue and just using the conservative Boehm -\cite{Boehm} collector for C. Later, we experimented with a range of -simple custom collectors - reference counting, mark-and-sweep, etc. -Ultimately, though, more advanced GCs will be needed to get the best -performance. It seems that RPython, enhanced with support for direct -address manipulations, is a good language for writing GCs, so it would -be possible for a GC expert to write one for our translation framework. -However, this is not the only way to obtain good GCs: we will soon -investigate a more practical course of action, which is to reuse -existing GCs. A good candidate is the GCs written in the Jikes RVM -\cite{JikesGC}. As they are in Java, it should be relatively -straightforward to add a translation step that turns one of them into -RPython (or directly our RPython-level intermediate representation) and -integrate it with the rest of the program being translated. +writing a good-quality OO VM. A reason is that a translation toolchain +operates in a more static way, which allows it to leverage good C +compilers. It is self-supporting: pieces of the implementation can be +written in RPython as well and translated along with the rest of the +RPython source, and they can all be compiled and optimized by the C +compiler. In order to write an OO VM in this style you need to start by +assuming an efficient dynamic compiler. + +Of course, the translation toolchain, once written, can also be reused +to implement other languages, and possibly tailored on a case-by-case +basis to fit the specific needs of a language. The process is +incremental: we can add more features as needed instead of starting from +a maximal up-front design, and gradually improve the quality of the +tools, the garbage collectors, the various optimizations, etc. + +Writing a good garbage collector remains hard, though. At least, it is +easy to experiment with various kind of GCs, so we started by just using +the conservative Boehm \cite{Boehm} collector for C and moved up to a +range of simple custom collectors -- reference counting, mark-and-sweep, +etc. Ultimately, though, more advanced GCs will be needed to get the +best performance. It seems that RPython, enhanced with support for +direct address manipulations, is a good language for writing GCs, so it +would be possible for a GC expert to write one for our translation +framework. However, this is not the only way to obtain good GCs: +existing GCs can also be reused. Good candidates are the GCs written in +the Jikes RVM \cite{JikesGC}. As they are in Java, it should be +relatively straightforward to add a translation step that turns one of +them into RPython (or directly into our RPython-level intermediate +representation) and integrate it with the rest of the program being +translated. In summary, developing a meta-programming translation toolchain requires -some work, but it can be done incrementally, it can reuse existing code, -and it gives a toolchain that is itself highly reusable and flexible in +work, but it can be done incrementally, it can reuse existing code, and +it gives a toolchain that is itself highly reusable and flexible in nature. +\subsection{Dynamic compilers} +\label{subsect:dynamic_compilers} + +As mentioned above, the performance of the VMs generated by our +translation framework are quite acceptable -- e.g. the Python VM +generated via C code is much faster than Jython running on the best +JVMs. Of course, the JIT compilers in these JVMs are essential to +achieve even this performance, which further proves the point that +writing good OO VMs -- especially ones meant to support dynamic +languages -- is a lot of work. + +The deeper problem with the otherwise highly-tuned JIT compilers of the +OO VMs is that they are not a very good match for running dynamic +languages. It might be possible to tune a general-purpose JIT compiler +enough, and write the dynamic language implementation accordingly, so +that most of the bookkeeping work involved in running the dynamic +language can be removed -- dispatching, boxing, unboxing... However +this has not been demonstrated yet. + +By far the fastest Python implementation, Psyco \cite{Psyco} contains a +hand-written language-specific dynamic compiler. PyPy's translation +tool-chain is able to extend the generated VMs with an automatically +generated dynamic compiler similar to Psyco, derived from the +interpreter. This is achieved by a pragmatic application of partial +evaluation techniques guided by a few hints added to the source of the +interpreter. In other words, it is possible to produce a reasonably +good language-specific JIT compiler and insert it into a VM, alongside +with the necessary support code and the rest of the regular interpreter. + +This result was one of the major goals and motivations for the whole +approach. By construction, any code written in the dynamic language +runs correctly under the JIT. Some very simple Python examples run more +than 100 times faster. At the time of this writing this is still rather +experimental, and the techniques involved are well beyond the scope of +the present paper. The reader is referred to \cite{D08.2} for more +information. + + + + \section{Related Work} -XXX +XXX \section{Conclusion} XXX From cfbolz at codespeak.net Sun May 27 15:09:43 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 27 May 2007 15:09:43 +0200 (CEST) Subject: [pypy-svn] r43721 - pypy/extradoc/talk/dyla2007 Message-ID: <20070527130943.ED20E80B5@code0.codespeak.net> Author: cfbolz Date: Sun May 27 15:09:43 2007 New Revision: 43721 Modified: pypy/extradoc/talk/dyla2007/dyla.bib pypy/extradoc/talk/dyla2007/dyla.tex Log: attempts to make the OO VM (dis)advantages more readable by itemizing them. more references Modified: pypy/extradoc/talk/dyla2007/dyla.bib ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.bib (original) +++ pypy/extradoc/talk/dyla2007/dyla.bib Sun May 27 15:09:43 2007 @@ -24,7 +24,7 @@ publisher = "John Wiley \& Sons, Ltd" } -% Java Cafe +% Prolog Cafe @phdthesis{ prologcafe, author = "Mutsunori Banbara", Modified: pypy/extradoc/talk/dyla2007/dyla.tex ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.tex (original) +++ pypy/extradoc/talk/dyla2007/dyla.tex Sun May 27 15:09:43 2007 @@ -189,6 +189,8 @@ benefits of OO VMs is the ability to implement certain hard things only once and share the benefits between all language implementations on top of the OO VM. +\begin{itemize} +\item \emph{Better interoperability than the C level:} Since the VM offers a standard object model and all the languages implemented on top of it are using it, it is easier to integrate the languages that are running on top of the VM. This @@ -196,41 +198,50 @@ typically the most important reason for wanting an implementation on the VM in the first place. +\item \emph{Cross-platform portability:} Only the underlying VM has to be ported to various hardware architectures and operating systems. The languages implemented on top can then be run without change in various environments. +\item \emph{Better tools:} Better IDEs, debuggers and profilers. +\item \emph{Better implementation of low-level issues like garbage collection, threading:} Since an OO VM is expected to be widely used and usually backed by a company, it becomes worthwhile and possible to spend a lot of effort tuning its garbage collector, threading model, exception support and other low-level implementation details. +\item \emph{Better performance:} Similarly, object-oriented VMs usually come with a highly tuned just-in-time compiler to make them perform well without requiring ahead-of-time compilation to machine language. This in addition with the previous point leads to much better performance of the languages running on top of the VM. +\item \emph{Ease of implementation:} The implementation of a language on top of an OO VM is easier because it starts at a higher level than C. Usually a high-level language like Java or C\# is used for the language implementation, which both offer the language implementer a much higher level of abstraction than when implementing in C. +\item \emph{A single unified implementation base:} The higher level of abstraction mentioned in the last point also helps to reduce the number of implementations of the language. Since the implementation on top of an OO VM is more abstract it is also easier to modify and to extend. XXX somewhat vague point +\end{itemize} -At a closer look, some of these advantages are only partially true in practice. +At a closer look, some of these advantages are only partially true in practice: +\begin{itemize} +\item \emph{Better performance:} So far it seems that performance of highly dynamic -languages is not actually significantly improved on OO VMs. Jython -\footnote{Python on the Java VM} is around 5 +languages is not actually significantly improved on OO VMs. +Jython\footnote{Python on the Java VM} is around 5 times slower than CPython, for IronPython\footnote{Python on .NET, which gives up on some features to improve performance} the figures vary but it is mostly @@ -240,17 +251,20 @@ language implementers would have to carefully produce code that matches these usage patterns, which is not a simple task. +\item \emph{Better GCs:} While this is obvious in theory, OO VMs tend to have a much higher memory overhead to start with (XXX ref) +\item \emph{Cross-platform portability:} While this is true to some extend, the situation with regard to portability is not significantly improved compared to e.g. C/Posix, which is relatively portable too. Also portability sometimes comes at the price of performance, because even if the OO VM is running on a particular hardware architecture it is not clear that the JIT is tuned for this architecture too or working at all, which leads to significantly less -performance. +speed. +\item \emph{Ease of implementation:} This point is disputable. On the one hand, OO VMs typically allow the language implementor to start at a higher level. On the other hand they also enforce a specific object and execution model. This means @@ -259,17 +273,19 @@ much on the language in question. XXX give example where it works well. An example where this mapping does not work too well is Prolog. While there -exist several implementations of Prolog on top of the JVM and also one on .NET, +exist several implementations of Prolog on top of the JVM \cite{prologcafe} +\cite{DBLP:conf/jelia/Calejo04} and also one on .NET \cite{psharp}, they are not particular efficient, especially when compared to good Prolog VMs in written in C. This is mostly because the Prolog execution model, which involves backtracking and deep recursion does not fit the JVM and .NET very well. Therefore the Prolog implementations on top of OO VMs resort to models that is quite unnatural both for the OO VM and for Prolog. - +\end{itemize} On the other hand some of the benefits are real and very useful, the most prominent being the easy interaction with the rest of the VM. Furthermore there -is better tool support and better GCs. XXX expand +is better tool support and better GCs. Also for languages where the execution +model fits the OO VM well, many of the disadvantages disappear. \subsection{The Cost of Implementation-Proliferation} @@ -316,10 +332,10 @@ \subsection{PyPy architecture} -We implemented this idea in the PyPy project. The dynamic language for -which we wrote an interpreter is Python. It is a language which, +We implemented this idea in the PyPy project \cite{pypy}. The dynamic language +for which we wrote an interpreter is Python. It is a language which, because of its size and rather intricate semantics, is a good target for -our approach, in the following sense: its previous reimplementation +our approach, in the following sense: its previous reimplementations (Jython for the JVM and IronPython for .NET) have each proved to be very time-consuming to maintain. Our implementation is by construction easier to maintain, and extremely portable (including to C/Posix, to the @@ -361,7 +377,8 @@ A detailed description of this translation process is beyond the scope of the present paper; it can be found in \cite{pypyvmconstruction}. The actual Python interpreter of PyPy and the results we achieved by translating it to C, LLVM -\cite{LLVM} and .NET are described in [XXX]. These results show that the +\cite{LLVM} and .NET are described in \cite{architecture} \cite{translationdoc} +These results show that the approach is practical and gives results whose performance is within the same order of magnitude (within a factor of 2 and improving) of the hand-written, well-tuned CPython, the C reference implementation. These figures do not From cfbolz at codespeak.net Sun May 27 15:20:08 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 27 May 2007 15:20:08 +0200 (CEST) Subject: [pypy-svn] r43724 - pypy/extradoc/talk/dyla2007 Message-ID: <20070527132008.D2C7680B5@code0.codespeak.net> Author: cfbolz Date: Sun May 27 15:20:08 2007 New Revision: 43724 Modified: pypy/extradoc/talk/dyla2007/dyla.bib Log: add 8.2 reference Modified: pypy/extradoc/talk/dyla2007/dyla.bib ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.bib (original) +++ pypy/extradoc/talk/dyla2007/dyla.bib Sun May 27 15:20:08 2007 @@ -290,6 +290,15 @@ note = "http://codespeak.net/pypy/dist/pypy/doc/index-report.html" } + at techreport{ D08.2, + title = "JIT Compiler Architecture", + author = {Armin Rigo and Samuele Pedroni}, + year = "2007", + institution = "PyPy Consortium", + number = "D08.2", + note = "http://codespeak.net/pypy/dist/pypy/doc/index-report.html" +} + @techreport{ D12.1, title = "High-Level Backends and Interpreter Feature Prototypes", author = {Antonio Cuni and Samuele Pedroni and Anders Chrigstr\"om and From cfbolz at codespeak.net Sun May 27 15:31:39 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 27 May 2007 15:31:39 +0200 (CEST) Subject: [pypy-svn] r43725 - pypy/extradoc/talk/dyla2007 Message-ID: <20070527133139.DAFB680B5@code0.codespeak.net> Author: cfbolz Date: Sun May 27 15:31:39 2007 New Revision: 43725 Modified: pypy/extradoc/talk/dyla2007/Makefile Log: remove cruft from the makefile Modified: pypy/extradoc/talk/dyla2007/Makefile ============================================================================== --- pypy/extradoc/talk/dyla2007/Makefile (original) +++ pypy/extradoc/talk/dyla2007/Makefile Sun May 27 15:31:39 2007 @@ -1,10 +1,3 @@ -prolog-in-rpython.pdf: padl.tex padl.bib stack.pdf graph.png stackless.pdf memory2.pdf - pdflatex padl - bibtex padl - pdflatex padl - pdflatex padl - mv padl.pdf prolog-in-rpython.pdf - dyla.pdf: dyla.tex dyla.bib pdflatex dyla bibtex dyla From arigo at codespeak.net Sun May 27 15:34:14 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 27 May 2007 15:34:14 +0200 (CEST) Subject: [pypy-svn] r43726 - pypy/extradoc/talk/dyla2007 Message-ID: <20070527133414.13BCC80B4@code0.codespeak.net> Author: arigo Date: Sun May 27 15:34:13 2007 New Revision: 43726 Modified: pypy/extradoc/talk/dyla2007/dyla.tex Log: Detail. Modified: pypy/extradoc/talk/dyla2007/dyla.tex ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.tex (original) +++ pypy/extradoc/talk/dyla2007/dyla.tex Sun May 27 15:34:13 2007 @@ -518,9 +518,10 @@ with the necessary support code and the rest of the regular interpreter. This result was one of the major goals and motivations for the whole -approach. By construction, any code written in the dynamic language -runs correctly under the JIT. Some very simple Python examples run more -than 100 times faster. At the time of this writing this is still rather +approach. By construction, the JIT does not get out of sync when the +language evolves, and any code written in the dynamic language runs +correctly under the JIT. Some very simple Python examples run more than +100 times faster. At the time of this writing this is still rather experimental, and the techniques involved are well beyond the scope of the present paper. The reader is referred to \cite{D08.2} for more information. From cfbolz at codespeak.net Sun May 27 15:34:47 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 27 May 2007 15:34:47 +0200 (CEST) Subject: [pypy-svn] r43727 - pypy/extradoc/talk/dyla2007 Message-ID: <20070527133447.C638A80B4@code0.codespeak.net> Author: cfbolz Date: Sun May 27 15:34:47 2007 New Revision: 43727 Modified: pypy/extradoc/talk/dyla2007/dyla.bib pypy/extradoc/talk/dyla2007/dyla.tex Log: add a citation of the psyco home page too, differentiate better between citations of the psyco paper and psyco-the-program Modified: pypy/extradoc/talk/dyla2007/dyla.bib ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.bib (original) +++ pypy/extradoc/talk/dyla2007/dyla.bib Sun May 27 15:34:47 2007 @@ -1,6 +1,6 @@ % Psyco - at inproceedings{ psyco, + at inproceedings{ psyco-paper, author = "Armin Rigo", title = "Representation-based just-in-time specialization and the psyco prototype for python", booktitle = "PEPM '04: Proceedings of the 2004 ACM SIGPLAN symposium on Partial evaluation and semantics-based program manipulation", @@ -13,6 +13,13 @@ address = "New York, NY, USA" } + at misc{ psyco-software, + author = "Armin Rigo", + title = "Psyco", + note = "http://psyco.sourceforge.net/", + url = "http://psyco.sourceforge.net/" +} + % P\# @inproceedings{ psharp, @@ -89,8 +96,8 @@ booktitle = "ICSE 2004, 26th International Conference on Software Engineering", year = 2004 } -\newblock In {\em ICSE}, pages 137--146, 2004. - at inproceedings{DBLP:conf/jelia/Calejo04, + + at inproceedings{InterProlog, author = {Miguel Calejo}, title = {Inter{P}rolog: Towards a Declarative Embedding of Logic Programming in {J}ava.}, @@ -98,7 +105,6 @@ year = {2004}, pages = {714-717}, ee = {http://springerlink.metapress.com/openurl.asp?genre=article{\&}issn=0302-9743{\&}volume=3229{\&}spage=714}, - crossref = {DBLP:conf/jelia/2004}, bibsource = {DBLP, http://dblp.uni-trier.de} } Modified: pypy/extradoc/talk/dyla2007/dyla.tex ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.tex (original) +++ pypy/extradoc/talk/dyla2007/dyla.tex Sun May 27 15:34:47 2007 @@ -158,7 +158,8 @@ Python'' \cite{stackless} which is a fork of CPython that adds micro-threading capabilities to Python. One of the reasons for not incorporating it back into CPython was that it was felt that they would make the implementation too -complex. Another implementation of the Python language is the Psyco \cite{Psyco} +complex. Another implementation of the Python language is the Psyco +\cite{psyco-software} project which adds a JIT-compiler to CPython. As all other re-implementations, Psyco contains the Python semantics and needs to be kept synchronized with CPython manually. @@ -274,7 +275,7 @@ An example where this mapping does not work too well is Prolog. While there exist several implementations of Prolog on top of the JVM \cite{prologcafe} -\cite{DBLP:conf/jelia/Calejo04} and also one on .NET \cite{psharp}, +\cite{InterProlog} and also one on .NET \cite{psharp}, they are not particular efficient, especially when compared to good Prolog VMs in written in C. This is mostly because the Prolog execution model, which involves backtracking and deep recursion does not fit the JVM and .NET very @@ -507,10 +508,11 @@ language can be removed -- dispatching, boxing, unboxing... However this has not been demonstrated yet. -By far the fastest Python implementation, Psyco \cite{Psyco} contains a +By far the fastest Python implementation, Psyco \cite{psyco-software} contains a hand-written language-specific dynamic compiler. PyPy's translation tool-chain is able to extend the generated VMs with an automatically -generated dynamic compiler similar to Psyco, derived from the +generated dynamic compiler that uses techniques similar to those of Psyco +\cite{Psyco-paper}, derived from the interpreter. This is achieved by a pragmatic application of partial evaluation techniques guided by a few hints added to the source of the interpreter. In other words, it is possible to produce a reasonably @@ -528,13 +530,6 @@ - - - - -\section{Related Work} -XXX - \section{Conclusion} XXX % ---- Bibliography ---- From arigo at codespeak.net Sun May 27 15:43:17 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 27 May 2007 15:43:17 +0200 (CEST) Subject: [pypy-svn] r43729 - pypy/extradoc/talk/dyla2007 Message-ID: <20070527134317.AC26180B8@code0.codespeak.net> Author: arigo Date: Sun May 27 15:43:17 2007 New Revision: 43729 Modified: pypy/extradoc/talk/dyla2007/outline.txt Log: Dilute "No need for VM standardization" into the conclusion. Modified: pypy/extradoc/talk/dyla2007/outline.txt ============================================================================== --- pypy/extradoc/talk/dyla2007/outline.txt (original) +++ pypy/extradoc/talk/dyla2007/outline.txt Sun May 27 15:43:17 2007 @@ -113,15 +113,6 @@ * JIT generation framework * expand... -- No need for VM standardization - * we reduce the need for alternative implementations by making - one source code enough for many environments - * even the translation framework need not be standardized, as - we could cross-translate high-level source code written for one - with another [e.g. steal the Jikes GC] - * might open the door to better solutions for interoperability - (high-level bridges instead of low-level ones, cross-translation...) - in an appropriate place above: - writing a translation toolchain is easier and simpler than writing an OO VM @@ -139,8 +130,11 @@ - Let's write more metaprogramming translation toolchains * diversity is good + * no need for VM standardization * very large, mostly unexplored design space (some of PyPy's choices will probably turn out to not be optimal) + * might open the door to better solutions for interoperability + (high-level bridges instead of low-level ones, cross-translation...) * ultimately a better investment of efforts than writing general-purpose VMs From arigo at codespeak.net Sun May 27 17:39:00 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 27 May 2007 17:39:00 +0200 (CEST) Subject: [pypy-svn] r43730 - pypy/extradoc/talk/dyla2007 Message-ID: <20070527153900.4CFAB80BB@code0.codespeak.net> Author: arigo Date: Sun May 27 17:38:59 2007 New Revision: 43730 Modified: pypy/extradoc/talk/dyla2007/dyla.tex Log: Conclusion. Modified: pypy/extradoc/talk/dyla2007/dyla.tex ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.tex (original) +++ pypy/extradoc/talk/dyla2007/dyla.tex Sun May 27 17:38:59 2007 @@ -386,6 +386,11 @@ include the spectacular speed-ups obtained in some cases by the JIT compiler described in section \ref{subsect:dynamic_compilers}. +In the sequel, we will focus on the relative advantages and +inconvenients of the PyPy approach compared to the approach of +hand-writing a language implementation on top of an OO VM. + + \subsection{A single source} Our approach -- a single ``meta-written'' implementation -- naturally @@ -529,9 +534,57 @@ information. - \section{Conclusion} -XXX + +Here are the two central points that we have asserted in the present +paper: + +\begin{itemize} +\item \emph{Do not write dynamic language implementations ``by hand''.} +Writing them more abstractly, at a higher level, has mostly only +advantages, among them avoiding a proliferation of implementations +growing out of sync. Writing interpreters both flexibly and efficiently +is difficult, and meta-programming is a good way to achieve it. +Moreover, this is not incompatible with targetting and benefiting from +existing good object-oriented virtual machines like the Java and .NET +ones. + +\item \emph{Do not write VMs ``by hand''.} +Writing language-specific virtual machines is a time-consuming task for +medium to large languages. Unless large amounts of resources can be +invested, the resulting VMs are bound to have limitations which lead to +the emergence of many implementations, a fact that is taxing precisely +for a community with limited resources. (This is of course even more +true for general-purpose VMs.) +\end{itemize} + +As a better alternative, we advocate a more general usage of +meta-programming: + +\begin{itemize} +\item \emph{Let's write more meta-programming translation toolchains.} +Aside from the advantages described in section +\ref{sect:metaprogramming}, a translation toolchain need not be +standardized for inter-operability, but can be tailored to the needs of +each project. Diversity is good; there is no need to attempt to +standardize on a single OO VM. +\end{itemize} + +The approach we outlined is actually just one in a very large, mostly +unexplored design space; it is likely that some of the choices made in +PyPy will turn out to be suboptimal. We are hoping that other +toolchains will emerge over time, exploring other aspects and proposing +other solutions. By their ``meta'' nature, these multiple approachs +should be easier to bridge together than, say, multiple OO VMs with +different object and runtime models. We believe that further research +in this area might open the door to better solutions for +interoperability in general -- e.g. high-level bridges instead of +(virtual-)machine-level ones, enabled by cross-translation. + +We believe this to be ultimately a better investment of efforts than the +development of more advanced general-purpose OO VMs. + + % ---- Bibliography ---- %\begin{small} \bibliographystyle{abbrv} From arigo at codespeak.net Sun May 27 17:57:06 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 27 May 2007 17:57:06 +0200 (CEST) Subject: [pypy-svn] r43731 - pypy/extradoc/talk/dyla2007 Message-ID: <20070527155706.D425980BC@code0.codespeak.net> Author: arigo Date: Sun May 27 17:57:06 2007 New Revision: 43731 Modified: pypy/extradoc/talk/dyla2007/dyla.tex Log: Fix an XXX. Modified: pypy/extradoc/talk/dyla2007/dyla.tex ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.tex (original) +++ pypy/extradoc/talk/dyla2007/dyla.tex Sun May 27 17:57:06 2007 @@ -229,10 +229,10 @@ than when implementing in C. \item -\emph{A single unified implementation base:} The higher level of abstraction -mentioned in the last point also helps to reduce the number of implementations -of the language. Since the implementation on top of an OO VM is more abstract -it is also easier to modify and to extend. XXX somewhat vague point +\emph{A single unified implementation base:} The .NET and Java VMs are trying +to position themselves as all-encompassing platforms; if one succeeds, then +implementations of the dynamic language for other platforms might no longer +be required. \end{itemize} From antocuni at codespeak.net Sun May 27 18:00:57 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sun, 27 May 2007 18:00:57 +0200 (CEST) Subject: [pypy-svn] r43732 - in pypy/dist/pypy/rpython: ootypesystem test Message-ID: <20070527160057.B82D580BC@code0.codespeak.net> Author: antocuni Date: Sun May 27 18:00:57 2007 New Revision: 43732 Modified: pypy/dist/pypy/rpython/ootypesystem/rclass.py pypy/dist/pypy/rpython/test/test_rtyper.py Log: test for a bug that occured when trying to translate pyrolog to cli, and relative bugfix Modified: pypy/dist/pypy/rpython/ootypesystem/rclass.py ============================================================================== --- pypy/dist/pypy/rpython/ootypesystem/rclass.py (original) +++ pypy/dist/pypy/rpython/ootypesystem/rclass.py Sun May 27 18:00:57 2007 @@ -169,6 +169,7 @@ self.lowleveltype = ootype.Instance(classdef.name, b, {}, {}, _hints = hints) self.prebuiltinstances = {} # { id(x): (x, _ptr) } self.object_type = self.lowleveltype + self.gcflavor = gcflavor def _setup_repr(self): if self.classdef is None: Modified: pypy/dist/pypy/rpython/test/test_rtyper.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rtyper.py (original) +++ pypy/dist/pypy/rpython/test/test_rtyper.py Sun May 27 18:00:57 2007 @@ -147,3 +147,21 @@ cdef = DummyClsDescDef(A) cdef._cpy_exported_type_ = type(Ellipsis) assert rmodel.getgcflavor(cdef) == 'cpy' + +def test_missing_gvflavor_bug(): + class MyClass: + def set_x(self): + self.x = create_tuple() + def create_tuple(): + return MyClass(), 42 + def fn(): + obj = MyClass() + obj.set_x() + create_tuple() + t = TranslationContext() + t.buildannotator().build_types(fn, []) + t.buildrtyper(type_system='ootype').specialize() + #t.view() + t.checkgraphs() + graph = graphof(t, fn) + assert graph.getreturnvar().concretetype == Void From antocuni at codespeak.net Sun May 27 18:17:02 2007 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Sun, 27 May 2007 18:17:02 +0200 (CEST) Subject: [pypy-svn] r43733 - in pypy/dist/pypy: rpython/lltypesystem rpython/test translator/cli Message-ID: <20070527161702.E882180BC@code0.codespeak.net> Author: antocuni Date: Sun May 27 18:17:00 2007 New Revision: 43733 Modified: pypy/dist/pypy/rpython/lltypesystem/opimpl.py pypy/dist/pypy/rpython/test/test_rfloat.py pypy/dist/pypy/translator/cli/opcodes.py Log: test for float**float, missing import in opimpl.py and support for it in gencli Modified: pypy/dist/pypy/rpython/lltypesystem/opimpl.py ============================================================================== --- pypy/dist/pypy/rpython/lltypesystem/opimpl.py (original) +++ pypy/dist/pypy/rpython/lltypesystem/opimpl.py Sun May 27 18:17:00 2007 @@ -1,4 +1,5 @@ import sys +import math from pypy.tool.sourcetools import func_with_new_name from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.lltypesystem.lloperation import opimpls Modified: pypy/dist/pypy/rpython/test/test_rfloat.py ============================================================================== --- pypy/dist/pypy/rpython/test/test_rfloat.py (original) +++ pypy/dist/pypy/rpython/test/test_rfloat.py Sun May 27 18:17:00 2007 @@ -102,6 +102,12 @@ res = self.interpret(fn, []) assert res == 42.0 + def test_pow(self): + def fn(x, y): + return x**y + res = self.interpret(fn, [2.0, 3.0]) + assert res == 8.0 + class TestLLtype(BaseTestRfloat, LLRtypeMixin): def test_hash(self): Modified: pypy/dist/pypy/translator/cli/opcodes.py ============================================================================== --- pypy/dist/pypy/translator/cli/opcodes.py (original) +++ pypy/dist/pypy/translator/cli/opcodes.py Sun May 27 18:17:00 2007 @@ -164,7 +164,8 @@ 'float_ne': _not('ceq'), 'float_gt': 'cgt', 'float_ge': _not('clt'), - + 'float_pow': [PushAllArgs, 'call float64 [mscorlib]System.Math::Pow(float64, float64)'], + 'llong_is_true': [PushAllArgs, 'ldc.i8 0', 'cgt.un'], 'llong_neg': 'neg', 'llong_neg_ovf': _check_ovf(['ldc.i8 0', PushAllArgs, 'sub.ovf', StoreResult]), From cfbolz at codespeak.net Sun May 27 18:19:16 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 27 May 2007 18:19:16 +0200 (CEST) Subject: [pypy-svn] r43734 - pypy/dist/pypy/lang/prolog/interpreter Message-ID: <20070527161916.B724E80BC@code0.codespeak.net> Author: cfbolz Date: Sun May 27 18:19:16 2007 New Revision: 43734 Modified: pypy/dist/pypy/lang/prolog/interpreter/parsing.py Log: calling float directly is supported nowadays Modified: pypy/dist/pypy/lang/prolog/interpreter/parsing.py ============================================================================== --- pypy/dist/pypy/lang/prolog/interpreter/parsing.py (original) +++ pypy/dist/pypy/lang/prolog/interpreter/parsing.py Sun May 27 18:19:16 2007 @@ -371,12 +371,7 @@ try: return Number(int(s)) except ValueError: - # XXX float is not supported by the rtyper yet :-( - # return Float(float(s)) - from pypy.objspace.std.strutil import break_up_float - from pypy.rlib.rarithmetic import parts_to_float - a, b, c, d = break_up_float(s) - return Float(parts_to_float(a, b, c, d)) + return Float(float(s)) def visit_complexterm(self, node): from pypy.lang.prolog.interpreter.term import Term From cfbolz at codespeak.net Sun May 27 18:29:46 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 27 May 2007 18:29:46 +0200 (CEST) Subject: [pypy-svn] r43735 - pypy/extradoc/talk/dyla2007 Message-ID: <20070527162946.ACFCC80BB@code0.codespeak.net> Author: cfbolz Date: Sun May 27 18:29:46 2007 New Revision: 43735 Modified: pypy/extradoc/talk/dyla2007/dyla.tex Log: Remove one XXX, add a (less purely aesthetic) one. Progress! Modified: pypy/extradoc/talk/dyla2007/dyla.tex ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.tex (original) +++ pypy/extradoc/talk/dyla2007/dyla.tex Sun May 27 18:29:46 2007 @@ -180,8 +180,6 @@ \subsection{Assessing the Advantages of Implementing a Language on Top of OO VMs} -XXX mention that OO languages don't have good metaprogramming support - Implementing a language on top of an OO VM is in many ways easier than implementing it in C. Let's take a look at the advantages that are usually cited for basing a @@ -281,6 +279,10 @@ involves backtracking and deep recursion does not fit the JVM and .NET very well. Therefore the Prolog implementations on top of OO VMs resort to models that is quite unnatural both for the OO VM and for Prolog. + +Another important point that make implementations of languages on top of OO VMs +harder is that typically they don't support meta-programming very well, or only +at the bytecode level. \end{itemize} On the other hand some of the benefits are real and very useful, the most @@ -539,6 +541,7 @@ Here are the two central points that we have asserted in the present paper: +%XXX doesn't look entirely nice \begin{itemize} \item \emph{Do not write dynamic language implementations ``by hand''.} Writing them more abstractly, at a higher level, has mostly only From cfbolz at codespeak.net Sun May 27 18:36:18 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 27 May 2007 18:36:18 +0200 (CEST) Subject: [pypy-svn] r43736 - pypy/extradoc/talk/dyla2007 Message-ID: <20070527163618.E8F6480BC@code0.codespeak.net> Author: cfbolz Date: Sun May 27 18:36:18 2007 New Revision: 43736 Modified: pypy/extradoc/talk/dyla2007/dyla.tex Log: just remove one XXX. we have space problems anyway. Modified: pypy/extradoc/talk/dyla2007/dyla.tex ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.tex (original) +++ pypy/extradoc/talk/dyla2007/dyla.tex Sun May 27 18:36:18 2007 @@ -269,7 +269,7 @@ other hand they also enforce a specific object and execution model. This means that the concepts of the implemented language need to be mapped to the execution model of the underlying VM, which may be easy or not, depending very -much on the language in question. XXX give example where it works well. +much on the language in question. An example where this mapping does not work too well is Prolog. While there exist several implementations of Prolog on top of the JVM \cite{prologcafe} From arigo at codespeak.net Sun May 27 19:05:45 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 27 May 2007 19:05:45 +0200 (CEST) Subject: [pypy-svn] r43737 - pypy/extradoc/talk/dyla2007 Message-ID: <20070527170545.2A60780B7@code0.codespeak.net> Author: arigo Date: Sun May 27 19:05:44 2007 New Revision: 43737 Modified: pypy/extradoc/talk/dyla2007/dyla.tex Log: Fix refs. Modified: pypy/extradoc/talk/dyla2007/dyla.tex ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.tex (original) +++ pypy/extradoc/talk/dyla2007/dyla.tex Sun May 27 19:05:44 2007 @@ -109,12 +109,12 @@ In section \ref{sect:approaches} we will explore the way VMs are typically implemented in C and on top of OO VMs and some of the problems of these approaches, using various Python implementations as the main example. In -\ref{section sect:metaprogramming} we will describe our proposed +section \ref{sect:metaprogramming} we will describe our proposed meta-programming approach. \section{Approaches to Dynamic Language Implementation} -\label{sect:appraches} +\label{sect:approaches} XXX the next paragraph is also the start of the introduction right now! From cfbolz at codespeak.net Sun May 27 19:08:09 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 27 May 2007 19:08:09 +0200 (CEST) Subject: [pypy-svn] r43738 - pypy/extradoc/talk/dyla2007 Message-ID: <20070527170809.66EE580B7@code0.codespeak.net> Author: cfbolz Date: Sun May 27 19:08:08 2007 New Revision: 43738 Added: pypy/extradoc/talk/dyla2007/dyla.pdf (contents, props changed) Log: check pdf in for holger who has no latex installed. Added: pypy/extradoc/talk/dyla2007/dyla.pdf ============================================================================== Binary file. No diff available. From cfbolz at codespeak.net Sun May 27 19:37:48 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 27 May 2007 19:37:48 +0200 (CEST) Subject: [pypy-svn] r43739 - pypy/extradoc/talk/dyla2007 Message-ID: <20070527173748.2479880BC@code0.codespeak.net> Author: cfbolz Date: Sun May 27 19:37:46 2007 New Revision: 43739 Modified: pypy/extradoc/talk/dyla2007/dyla.pdf pypy/extradoc/talk/dyla2007/dyla.tex Log: replacing long utf-8 dashes by -- since they don't seem to work well. Modified: pypy/extradoc/talk/dyla2007/dyla.pdf ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/dyla2007/dyla.tex ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.tex (original) +++ pypy/extradoc/talk/dyla2007/dyla.tex Sun May 27 19:37:46 2007 @@ -41,7 +41,7 @@ virtual machines instead of writing them by hand. This approach has been validated by the PyPy project, with which we can automatically insert many features into the -generated virtual machines ? including good just-in-time compilers tuned +generated virtual machines -- including good just-in-time compilers tuned to the dynamic language at hand. \footnote{This research is partially supported by the EU funded %research project: @@ -73,7 +73,7 @@ the implementation of domain-specific-languages where it is important to keep implementation effort small. -For these reasons writing a virtual machine in C has many problems because it +For these reasons writing a virtual machine in C is problematic because it forces the language implementer to deal with many low-level details. Limitations of the C implementation lead to alternative implementations which draw work-power from the reference implementation. An alternative to writing @@ -84,7 +84,7 @@ implementation of such a VM is started, this enters in conflict with the goal of having to maintain essentially a single, simple enough implementation for a given programming language: as the language becomes popular, there will be a -demand for having it run on various platforms ? high-level VMs as well as +demand for having it run on various platforms -- high-level VMs as well as C-level environments. The argument we will make in the present paper is that it is possible to @@ -94,7 +94,7 @@ high-level but less dynamic language. This interpreter plays the role of a specification for the dynamic language. With a good enough translation toolchain we can then generate whole virtual machines from -this specification ? either full custom VMs for C-level operating +this specification -- either full custom VMs for C-level operating systems, or layers on top of various OO VMs. In other words, meta-programming techniques can be used to successfully replace a foreseeable one-VM-fits-all standardization attempt. From cfbolz at codespeak.net Sun May 27 19:43:37 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 27 May 2007 19:43:37 +0200 (CEST) Subject: [pypy-svn] r43740 - pypy/extradoc/talk/dyla2007 Message-ID: <20070527174337.B23FE8099@code0.codespeak.net> Author: cfbolz Date: Sun May 27 19:43:37 2007 New Revision: 43740 Modified: pypy/extradoc/talk/dyla2007/dyla.tex Log: the proper magic to be able to use utf-8 Modified: pypy/extradoc/talk/dyla2007/dyla.tex ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.tex (original) +++ pypy/extradoc/talk/dyla2007/dyla.tex Sun May 27 19:43:37 2007 @@ -1,7 +1,8 @@ -\documentclass{llncs} +\documentclass[utf8x]{llncs} \usepackage{makeidx} \usepackage{graphicx} +\usepackage[utf8x]{inputenc} \begin{document} From cfbolz at codespeak.net Sun May 27 19:46:20 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 27 May 2007 19:46:20 +0200 (CEST) Subject: [pypy-svn] r43741 - pypy/extradoc/talk/dyla2007 Message-ID: <20070527174620.1061580BC@code0.codespeak.net> Author: cfbolz Date: Sun May 27 19:46:19 2007 New Revision: 43741 Modified: pypy/extradoc/talk/dyla2007/dyla.tex Log: okok. revert the last one since armin's latex doesn't seem to like it. Modified: pypy/extradoc/talk/dyla2007/dyla.tex ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.tex (original) +++ pypy/extradoc/talk/dyla2007/dyla.tex Sun May 27 19:46:19 2007 @@ -1,8 +1,7 @@ -\documentclass[utf8x]{llncs} +\documentclass{llncs} \usepackage{makeidx} \usepackage{graphicx} -\usepackage[utf8x]{inputenc} \begin{document} From hpk at codespeak.net Sun May 27 19:46:32 2007 From: hpk at codespeak.net (hpk at codespeak.net) Date: Sun, 27 May 2007 19:46:32 +0200 (CEST) Subject: [pypy-svn] r43742 - pypy/extradoc/talk/dyla2007 Message-ID: <20070527174632.5056C80BF@code0.codespeak.net> Author: hpk Date: Sun May 27 19:46:31 2007 New Revision: 43742 Modified: pypy/extradoc/talk/dyla2007/dyla.tex Log: a quick try describing PyPy's approach without contrasting it to .NET/CLI too much - rather focusing on presenting PyPy as the next evolutional step to raise the level of language implementations, and still taking advantage of existing new approaches/VMs. Modified: pypy/extradoc/talk/dyla2007/dyla.tex ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.tex (original) +++ pypy/extradoc/talk/dyla2007/dyla.tex Sun May 27 19:46:31 2007 @@ -7,7 +7,7 @@ \pagestyle{headings} -\title{How to Not write Virtual Machines for Dynamic Languages} +\title{PyPy: raising the level for Dynamic Language implementations} %\titlerunning{XXX} % abbreviated title (for running head) % also used for the TOC unless @@ -28,21 +28,30 @@ \begin{abstract} -Dynamic languages are traditionally implemented by writing a virtual -machine for them, centered around an interpreter and/or a built-in -compiler and providing the object model and memory management. When -a language becomes more successful, the limitations of such an implementation -lead to the emergence of alternative implementations that try to solve some -of the problems. Another reason for new implementations is the desire to have -the language integrate well with existing, well-tuned object-oriented virtual -machine like Java's. In this paper, we describe the mechanisms that lead to an -abundance of implementations, and explore some of the limitations of standard -VMs. We propose a different complementary alternative: flexibly generating -virtual machines instead -of writing them by hand. This approach has been validated by the PyPy -project, with which we can automatically insert many features into the -generated virtual machines -- including good just-in-time compilers tuned -to the dynamic language at hand. +Dynamic languages are usually implemented in lower level environments, +traditionally in C/Posix. Recently .NET and the JVM are emerging as +new higher-level environments, offering more built-in support for +memory management, threading models, dynamic optimization techniques +and also offering an abundance of libraries for application development. +However, manually implementing languages for a standardized runtime environment +may not be the last word on bringing dynamic language implementations to a higher level. +The PyPy project successfully experimented with an approach that +uses extensive meta-programming techniques to analyse and transform the +single source of an language "specification" and automatically adds +lower level and runtime-dependent aspects, successfully producing +efficent virtual machines for an runtime environment of choice. +The meta-programming facilities are largely re-useable for implementing +different computer languages. We thus argue in this paper that +one should not write interpreters manually and dependent on lower +level and runtime details - but rather explore and exploit +meta-programming facilities which themselves can be written +and tested in a very high language, overall raising the level +of implementing dynamic languages. While there is considerable +complexity liying in such a meta-programming tool chain, it is +re-usable for different language implementations and generally +helps to avoid having to implement language semantics and +optimizations redundantly. + \footnote{This research is partially supported by the EU funded %research project: IST 004779 PyPy (PyPy: Implementing Python in Python).} \\ From arigo at codespeak.net Sun May 27 20:36:10 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 27 May 2007 20:36:10 +0200 (CEST) Subject: [pypy-svn] r43743 - pypy/extradoc/talk/dyla2007 Message-ID: <20070527183610.52E4180BB@code0.codespeak.net> Author: arigo Date: Sun May 27 20:36:09 2007 New Revision: 43743 Modified: pypy/extradoc/talk/dyla2007/dyla.tex Log: Shrink shrink shrink the start of section 2. It says less, but I think that the point that we want implementations on top of OO VMs was made in the introduction already. Modified: pypy/extradoc/talk/dyla2007/dyla.tex ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.tex (original) +++ pypy/extradoc/talk/dyla2007/dyla.tex Sun May 27 20:36:09 2007 @@ -79,8 +79,8 @@ techniques even if that entails lower execution speed. Similarly fragmentation (for example because of other implementations of the same language) is a problem because it divides available resources. All these points also apply to -the implementation of domain-specific-languages where it is important to keep -implementation effort small. +the implementation of domain-specific languages where it is important to keep +the implementation effort small. For these reasons writing a virtual machine in C is problematic because it forces the language implementer to deal with many low-level details. Limitations @@ -125,71 +125,32 @@ \section{Approaches to Dynamic Language Implementation} \label{sect:approaches} -XXX the next paragraph is also the start of the introduction right now! - -Dynamic languages are traditionally implemented by writing a virtual -machine for them in a low-level language like C, or in a language that -can relatively easily be turned into C. The machine implements an -object model supporting the high level dynamic language's objects. It -typically provides features like automatic garbage collection. Recent -languages like Python, Ruby, Perl and JavaScript have complicated -semantics which are most easily mapped to a simple interpreter operating -on syntax trees or bytecode. Simpler languages like Lisp and Self -typically have more efficient implementations based on just-in-time code -generation. - -There are many good reasons for choosing C as an implementation language. It -gives a decent base performance, is highly portable across different hardware -architectures and operating system if some care is taken and it allows the reuse -of many existing libraries. - -On the other hand the use of C comes with a set of problems. Since C has no ways -to do automatic memory management and since the implemented languages are -garbage collected, the VM needs to contain a garbage collector. Implementing a -high-performance GC is a tedious task and so many implementations resort to -relatively simple algorithms like mark-and-sweep or reference counting. Also the -choice of garbage collector is fixed relatively early in the life of the project -and then manifests itself throughout the complete VM source code. Thus it -becomes extremely tedious to change it later. - -%A similar problem is the threading model the VM uses -% - simpler threading models -% - don't make single-threaded apps slower due to locks -% - global locks -% - green threading - -% - wrapping with C libraries simple but tedious - -These limitations of a C-based implementation often lead to more implementations -of the language. For example in addition to the main C implementation of Python -(``CPython'' \cite{cpy251}) there are a number of additional implementations -that want to fix some of the limitations of CPython. One of them is ``Stackless -Python'' \cite{stackless} which is a fork of CPython that adds micro-threading -capabilities to Python. One of the reasons for not incorporating it back into -CPython was that it was felt that they would make the implementation too -complex. Another implementation of the Python language is the Psyco -\cite{psyco-software} -project which adds a JIT-compiler to CPython. As all other re-implementations, -Psyco contains the Python semantics and needs to be kept synchronized with -CPython manually. - -% - free threading attempts for Python - -As a language becomes more popular, yet another reason for different -implementations emerges: People want to use it together with Java and C\# -libraries and tools. Although it is in principle possible to do that by -bridging the C-VM with the respective VMs of these languages (for example the -``PythonNet'' project \cite{pythonnet} that brigdes CPython with a .NET VM) -this is an incomplete solution since it prevents many use-cases. -Therefore in the end implementations of the dynamic language on top of the -object-oriented VMs are created to properly integrate with the complete -environment. +\def\implname#1{\emph{#1}} +The fact that limitations of a C-based implementation of a dynamic +language leads to the emergence of additional implementations is clear +in the case of Python. The reference implementation, \implname{CPython} +\cite{cpy251}, is a simple recursive interpreter. \implname{Stackless +Python} \cite{stackless} is a fork that adds micro-threading +capabilities to Python. One of the reasons for not incorporating it back +into CPython was that it was felt that they would make the +implementation too complex. Another implementation of the Python +language is \implname{Psyco} \cite{psyco-software}, which adds a +JIT-compiler to CPython. Finally, \implname{Jython} is a +re-implementation for the Java VM and \implname{IronPython} one for +.NET. All of these need to be kept in sync with the relatively fast +evolution of the language. + +With the emergence of .NET and the JVM as interesting language +implementation platforms, an argument that is sometimes made is that +communities should only develop an implementation of their language for +one of these platforms.\footnote{Preferably the argument author's favourite +one.} \subsection{Assessing the Advantages of Implementing a Language on Top of OO VMs} -Implementing a language on top of an OO VM is in many ways easier than +Implementing a language on top of an existing OO VM is in many ways easier than implementing it in C. Let's take a look at the advantages that are usually cited for basing a language implementation of a dynamic language on a standard object oriented @@ -249,7 +210,7 @@ \item \emph{Better performance:} So far it seems that performance of highly dynamic languages is not actually significantly improved on OO VMs. -Jython\footnote{Python on the Java VM} is around 5 +Jython is around 5 times slower than CPython, for IronPython\footnote{Python on .NET, which gives up on some features to improve performance} the figures vary but it is mostly From cfbolz at codespeak.net Sun May 27 20:39:41 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 27 May 2007 20:39:41 +0200 (CEST) Subject: [pypy-svn] r43744 - pypy/extradoc/talk/dyla2007 Message-ID: <20070527183941.E63AE80BB@code0.codespeak.net> Author: cfbolz Date: Sun May 27 20:39:41 2007 New Revision: 43744 Modified: pypy/extradoc/talk/dyla2007/dyla.pdf pypy/extradoc/talk/dyla2007/dyla.tex Log: try to steal the good ideas of holgers abstract Modified: pypy/extradoc/talk/dyla2007/dyla.pdf ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/dyla2007/dyla.tex ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.tex (original) +++ pypy/extradoc/talk/dyla2007/dyla.tex Sun May 27 20:39:41 2007 @@ -28,33 +28,29 @@ \begin{abstract} -Dynamic languages are usually implemented in lower level environments, -traditionally in C/Posix. Recently .NET and the JVM are emerging as -new higher-level environments, offering more built-in support for -memory management, threading models, dynamic optimization techniques -and also offering an abundance of libraries for application development. -However, manually implementing languages for a standardized runtime environment -may not be the last word on bringing dynamic language implementations to a higher level. -The PyPy project successfully experimented with an approach that -uses extensive meta-programming techniques to analyse and transform the -single source of an language "specification" and automatically adds -lower level and runtime-dependent aspects, successfully producing -efficent virtual machines for an runtime environment of choice. -The meta-programming facilities are largely re-useable for implementing -different computer languages. We thus argue in this paper that -one should not write interpreters manually and dependent on lower -level and runtime details - but rather explore and exploit -meta-programming facilities which themselves can be written -and tested in a very high language, overall raising the level -of implementing dynamic languages. While there is considerable -complexity liying in such a meta-programming tool chain, it is -re-usable for different language implementations and generally -helps to avoid having to implement language semantics and -optimizations redundantly. - -\footnote{This research is partially supported by the EU funded %research - project: -IST 004779 PyPy (PyPy: Implementing Python in Python).} \\ +Dynamic languages are traditionally implemented by writing a virtual machine for +them, centered around an interpreter and/or a built-in compiler and providing +the object model and memory management. When a language becomes more successful, +the limitations of such an implementation lead to the emergence of alternative +implementations that try to solve some of the problems. Another reason for new +implementations is the desire to have the language integrate well with existing, +well-tuned object-oriented virtual machine like Java's. In this paper, we +describe the mechanisms that lead to an abundance of implementations, and +explore some of the limitations of standard VMs. We propose a different +complementary alternative to writing VMs by hand and dealing with low-level +details: flexibly generating virtual machines from a single abstract language +``specification''. + +This approach has been validated by the PyPy project that tries to explore +meta-programming facilities with which we can automatically insert many features +and low-level details into an automatically generated virtual machines -- +including good just-in-time compilers tuned to the dynamic language at hand. +We thus argue in this paper that one should not write interpreters manually but +rather use meta-programming techniques and raise the overall level at which +dynamic languages are implemented. We believe this to be ultimately a better +investment of efforts than the development of more advanced general-purpose OO +VMs. \footnote{This research is partially supported by the EU funded + project: IST 004779 PyPy (PyPy: Implementing Python in Python).} \\ \end{abstract} From arigo at codespeak.net Sun May 27 20:46:34 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 27 May 2007 20:46:34 +0200 (CEST) Subject: [pypy-svn] r43745 - pypy/extradoc/talk/dyla2007 Message-ID: <20070527184634.E2B7180BB@code0.codespeak.net> Author: arigo Date: Sun May 27 20:46:34 2007 New Revision: 43745 Modified: pypy/extradoc/talk/dyla2007/dyla.tex Log: Swap the two paras of the abstract. Experimental. Modified: pypy/extradoc/talk/dyla2007/dyla.tex ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.tex (original) +++ pypy/extradoc/talk/dyla2007/dyla.tex Sun May 27 20:46:34 2007 @@ -28,6 +28,12 @@ \begin{abstract} +We argue in this paper that one should not write interpreters manually but +rather use meta-programming techniques and raise the overall level at which +dynamic languages are implemented. We believe this to be ultimately a better +investment of efforts than the development of more advanced general-purpose OO +VMs. + Dynamic languages are traditionally implemented by writing a virtual machine for them, centered around an interpreter and/or a built-in compiler and providing the object model and memory management. When a language becomes more successful, @@ -38,18 +44,11 @@ describe the mechanisms that lead to an abundance of implementations, and explore some of the limitations of standard VMs. We propose a different complementary alternative to writing VMs by hand and dealing with low-level -details: flexibly generating virtual machines from a single abstract language -``specification''. - -This approach has been validated by the PyPy project that tries to explore -meta-programming facilities with which we can automatically insert many features -and low-level details into an automatically generated virtual machines -- -including good just-in-time compilers tuned to the dynamic language at hand. -We thus argue in this paper that one should not write interpreters manually but -rather use meta-programming techniques and raise the overall level at which -dynamic languages are implemented. We believe this to be ultimately a better -investment of efforts than the development of more advanced general-purpose OO -VMs. \footnote{This research is partially supported by the EU funded +details, validated by the PyPy project: flexibly generating virtual machines +from a single abstract language ``specification'', inserting features and +low-level details automatically -- including good just-in-time compilers tuned +to the dynamic language at hand. +\footnote{This research is partially supported by the EU funded project: IST 004779 PyPy (PyPy: Implementing Python in Python).} \\ \end{abstract} From cfbolz at codespeak.net Sun May 27 20:51:40 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 27 May 2007 20:51:40 +0200 (CEST) Subject: [pypy-svn] r43746 - pypy/extradoc/talk/dyla2007 Message-ID: <20070527185140.6A83680B4@code0.codespeak.net> Author: cfbolz Date: Sun May 27 20:51:39 2007 New Revision: 43746 Modified: pypy/extradoc/talk/dyla2007/dyla.tex Log: slight rewrite of the first paragraph of the abstract. adding a parenthetical comment about GCs. Modified: pypy/extradoc/talk/dyla2007/dyla.tex ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.tex (original) +++ pypy/extradoc/talk/dyla2007/dyla.tex Sun May 27 20:51:39 2007 @@ -28,11 +28,11 @@ \begin{abstract} -We argue in this paper that one should not write interpreters manually but -rather use meta-programming techniques and raise the overall level at which -dynamic languages are implemented. We believe this to be ultimately a better -investment of efforts than the development of more advanced general-purpose OO -VMs. +We argue in this paper that one should not write interpreters for dynamic +languages manually but rather use meta-programming techniques and raise the +overall level at which they are are implemented. We believe this to be +ultimately a better investment of efforts than the development of more advanced +general-purpose object oriented VMs. Dynamic languages are traditionally implemented by writing a virtual machine for them, centered around an interpreter and/or a built-in compiler and providing @@ -78,7 +78,8 @@ the implementation effort small. For these reasons writing a virtual machine in C is problematic because it -forces the language implementer to deal with many low-level details. Limitations +forces the language implementer to deal with many low-level details (like +garbage collection and threading-issues). Limitations of the C implementation lead to alternative implementations which draw work-power from the reference implementation. An alternative to writing implementations in C is to build them on top of one of the newer object oriented From cfbolz at codespeak.net Sun May 27 20:55:49 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 27 May 2007 20:55:49 +0200 (CEST) Subject: [pypy-svn] r43747 - pypy/extradoc/talk/dyla2007 Message-ID: <20070527185549.6330180BB@code0.codespeak.net> Author: cfbolz Date: Sun May 27 20:55:49 2007 New Revision: 43747 Modified: pypy/extradoc/talk/dyla2007/dyla.pdf pypy/extradoc/talk/dyla2007/dyla.tex Log: I find "fact" a bit strong here. Modified: pypy/extradoc/talk/dyla2007/dyla.pdf ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/dyla2007/dyla.tex ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.tex (original) +++ pypy/extradoc/talk/dyla2007/dyla.tex Sun May 27 20:55:49 2007 @@ -123,7 +123,7 @@ \def\implname#1{\emph{#1}} -The fact that limitations of a C-based implementation of a dynamic +The observation that limitations of a C-based implementation of a dynamic language leads to the emergence of additional implementations is clear in the case of Python. The reference implementation, \implname{CPython} \cite{cpy251}, is a simple recursive interpreter. \implname{Stackless From cfbolz at codespeak.net Sun May 27 20:59:31 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 27 May 2007 20:59:31 +0200 (CEST) Subject: [pypy-svn] r43748 - pypy/extradoc/talk/dyla2007 Message-ID: <20070527185931.29DB180C1@code0.codespeak.net> Author: cfbolz Date: Sun May 27 20:59:30 2007 New Revision: 43748 Modified: pypy/extradoc/talk/dyla2007/dyla.pdf pypy/extradoc/talk/dyla2007/dyla.tex Log: add two noindents so make the single line between to itemizes stand out more Modified: pypy/extradoc/talk/dyla2007/dyla.pdf ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/dyla2007/dyla.tex ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.tex (original) +++ pypy/extradoc/talk/dyla2007/dyla.tex Sun May 27 20:59:30 2007 @@ -200,6 +200,7 @@ \end{itemize} +\noindent At a closer look, some of these advantages are only partially true in practice: \begin{itemize} @@ -527,6 +528,7 @@ true for general-purpose VMs.) \end{itemize} +\noindent As a better alternative, we advocate a more general usage of meta-programming: From arigo at codespeak.net Sun May 27 21:03:24 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 27 May 2007 21:03:24 +0200 (CEST) Subject: [pypy-svn] r43749 - pypy/extradoc/talk/dyla2007 Message-ID: <20070527190324.BC20C80C0@code0.codespeak.net> Author: arigo Date: Sun May 27 21:03:24 2007 New Revision: 43749 Modified: pypy/extradoc/talk/dyla2007/dyla.tex Log: Typo; spacing. Modified: pypy/extradoc/talk/dyla2007/dyla.tex ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.tex (original) +++ pypy/extradoc/talk/dyla2007/dyla.tex Sun May 27 21:03:24 2007 @@ -34,6 +34,8 @@ ultimately a better investment of efforts than the development of more advanced general-purpose object oriented VMs. +\smallskip + Dynamic languages are traditionally implemented by writing a virtual machine for them, centered around an interpreter and/or a built-in compiler and providing the object model and memory management. When a language becomes more successful, @@ -79,7 +81,7 @@ For these reasons writing a virtual machine in C is problematic because it forces the language implementer to deal with many low-level details (like -garbage collection and threading-issues). Limitations +garbage collection and threading issues). Limitations of the C implementation lead to alternative implementations which draw work-power from the reference implementation. An alternative to writing implementations in C is to build them on top of one of the newer object oriented From cfbolz at codespeak.net Sun May 27 21:50:40 2007 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 27 May 2007 21:50:40 +0200 (CEST) Subject: [pypy-svn] r43751 - pypy/extradoc/talk/dyla2007 Message-ID: <20070527195040.2599B80BC@code0.codespeak.net> Author: cfbolz Date: Sun May 27 21:50:39 2007 New Revision: 43751 Modified: pypy/extradoc/talk/dyla2007/dyla.pdf pypy/extradoc/talk/dyla2007/dyla.tex Log: change to the old title but italicize the "not". is -> was in the footnote. Modified: pypy/extradoc/talk/dyla2007/dyla.pdf ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/dyla2007/dyla.tex ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.tex (original) +++ pypy/extradoc/talk/dyla2007/dyla.tex Sun May 27 21:50:39 2007 @@ -7,7 +7,7 @@ \pagestyle{headings} -\title{PyPy: raising the level for Dynamic Language implementations} +\title{How to \emph{not} write Virtual Machines for Dynamic Languages} %\titlerunning{XXX} % abbreviated title (for running head) % also used for the TOC unless @@ -50,7 +50,7 @@ from a single abstract language ``specification'', inserting features and low-level details automatically -- including good just-in-time compilers tuned to the dynamic language at hand. -\footnote{This research is partially supported by the EU funded +\footnote{This research was partially supported by the EU funded project: IST 004779 PyPy (PyPy: Implementing Python in Python).} \\ \end{abstract} From arigo at codespeak.net Sun May 27 21:52:50 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 27 May 2007 21:52:50 +0200 (CEST) Subject: [pypy-svn] r43752 - pypy/extradoc/talk/dyla2007 Message-ID: <20070527195250.5151980BB@code0.codespeak.net> Author: arigo Date: Sun May 27 21:52:50 2007 New Revision: 43752 Modified: pypy/extradoc/talk/dyla2007/dyla.tex Log: Increase the skip. Seems to have no visible effect when Carl Friedrich compiles the pdf... Modified: pypy/extradoc/talk/dyla2007/dyla.tex ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.tex (original) +++ pypy/extradoc/talk/dyla2007/dyla.tex Sun May 27 21:52:50 2007 @@ -34,7 +34,7 @@ ultimately a better investment of efforts than the development of more advanced general-purpose object oriented VMs. -\smallskip +\medskip Dynamic languages are traditionally implemented by writing a virtual machine for them, centered around an interpreter and/or a built-in compiler and providing From santagada at codespeak.net Sun May 27 22:30:46 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Sun, 27 May 2007 22:30:46 +0200 (CEST) Subject: [pypy-svn] r43753 - in pypy/dist/pypy/lang/js: . test Message-ID: <20070527203046.9622880BB@code0.codespeak.net> Author: santagada Date: Sun May 27 22:30:45 2007 New Revision: 43753 Modified: pypy/dist/pypy/lang/js/astbuilder.py pypy/dist/pypy/lang/js/operations.py pypy/dist/pypy/lang/js/test/test_interp.py Log: made for in works and made a simple test to the ; insertion Modified: pypy/dist/pypy/lang/js/astbuilder.py ============================================================================== --- pypy/dist/pypy/lang/js/astbuilder.py (original) +++ pypy/dist/pypy/lang/js/astbuilder.py Sun May 27 22:30:45 2007 @@ -306,7 +306,13 @@ right = self.dispatch(node.children[2]) body= self.dispatch(node.children[3]) return operations.ForIn(pos, left, right, body) - visit_invarfor = visit_infor + + def visit_invarfor(self, node): + pos = self.get_pos(node) + left = self.dispatch(node.children[1]) + right = self.dispatch(node.children[2]) + body= self.dispatch(node.children[3]) + return operations.ForVarIn(pos, left, right, body) def get_next_expr(self, node, i): if isinstance(node.children[i], Symbol) and \ Modified: pypy/dist/pypy/lang/js/operations.py ============================================================================== --- pypy/dist/pypy/lang/js/operations.py (original) +++ pypy/dist/pypy/lang/js/operations.py Sun May 27 22:30:45 2007 @@ -878,6 +878,7 @@ ctx.variable.Put(name, w_Undefined) else: ctx.variable.Put(name, self.expr.eval(ctx).GetValue()) + return self.identifier class VariableDeclList(Expression): @@ -959,8 +960,33 @@ continue +class ForVarIn(Statement): + def __init__(self, pos, vardecl, lobject, body): + self.pos = pos + self.vardecl = vardecl + self.object = lobject + self.body = body + + def execute(self, ctx): + identifier = self.vardecl.eval(ctx) + obj = self.object.eval(ctx).GetValue().ToObject() + for prop in obj.propdict.values(): + if prop.de: + continue + iterator = identifier.eval(ctx) + iterator.PutValue(prop.value, ctx) + try: + result = self.body.execute(ctx) + except ExecutionReturned, e: + if e.type == 'break': + break + elif e.type == 'continue': + continue + + class ForIn(Statement): def __init__(self, pos, iterator, lobject, body): + self.pos = pos self.iterator = iterator self.object = lobject self.body = body @@ -980,6 +1006,7 @@ elif e.type == 'continue': continue + class For(Statement): def __init__(self, pos, setup, condition, update, body): self.pos = pos Modified: pypy/dist/pypy/lang/js/test/test_interp.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_interp.py (original) +++ pypy/dist/pypy/lang/js/test/test_interp.py Sun May 27 22:30:45 2007 @@ -456,6 +456,14 @@ } """, ['5',]) +def test_forinvar(): + assertp(""" + var x = {a:5}; + for(var y in x){ + print(y); + } + """, ['5',]) + def test_stricteq(): yield assertv, "2 === 2;", True yield assertv, "2 === 3;", False @@ -536,3 +544,6 @@ a2[0] = 2; print(a1[0]); """, ['1', '1']) + +def test_semicolon(): + assertv("1", 1) From santagada at codespeak.net Sun May 27 23:17:55 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Sun, 27 May 2007 23:17:55 +0200 (CEST) Subject: [pypy-svn] r43754 - in pypy/dist/pypy/lang/js: . test Message-ID: <20070527211755.7EBF380BB@code0.codespeak.net> Author: santagada Date: Sun May 27 23:17:53 2007 New Revision: 43754 Modified: pypy/dist/pypy/lang/js/operations.py pypy/dist/pypy/lang/js/test/test_interp.py Log: made return work properly and remove some code from call that was masking the problem in sourceelements and block that shoudl be the place to threat a return statement. Modified: pypy/dist/pypy/lang/js/operations.py ============================================================================== --- pypy/dist/pypy/lang/js/operations.py (original) +++ pypy/dist/pypy/lang/js/operations.py Sun May 27 23:17:53 2007 @@ -219,11 +219,7 @@ else: r7 = r6 - try: - retval = r3.Call(ctx=ctx, args=r2.get_args(), this=r7) - return retval - except ExecutionReturned, e: - return e.value + return r3.Call(ctx=ctx, args=r2.get_args(), this=r7) class Comma(BinaryOp): @@ -789,7 +785,7 @@ return last except Exception, e: if isinstance(e, ExecutionReturned) and e.type == 'return': - raise + return e.value else: # TODO: proper exception handling print "exception in line: %s, on: %s%s"%(node.pos.lineno, node, os.linesep) Modified: pypy/dist/pypy/lang/js/test/test_interp.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_interp.py (original) +++ pypy/dist/pypy/lang/js/test/test_interp.py Sun May 27 23:17:53 2007 @@ -85,8 +85,9 @@ assertp('x=function(){print(3);}; x();', '3') def test_function_returns(): - yield assertp, 'x=function(){return 1;}; print(x()+x());', '2' + yield assertv, 'x=function(){return 1;}; x()+x();', 2 yield assertp, 'function x() { return; };', [] + yield assertv, 'function x() { d=2; return d;}; x()', 2 def test_var_declaration(): yield assertv, 'var x = 3; x;', 3 From mwh at codespeak.net Sun May 27 23:22:41 2007 From: mwh at codespeak.net (mwh at codespeak.net) Date: Sun, 27 May 2007 23:22:41 +0200 (CEST) Subject: [pypy-svn] r43755 - pypy/extradoc/talk/dyla2007 Message-ID: <20070527212241.F290E80BB@code0.codespeak.net> Author: mwh Date: Sun May 27 23:22:41 2007 New Revision: 43755 Modified: pypy/extradoc/talk/dyla2007/dyla.tex Log: 53% of a review Modified: pypy/extradoc/talk/dyla2007/dyla.tex ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.tex (original) +++ pypy/extradoc/talk/dyla2007/dyla.tex Sun May 27 23:22:41 2007 @@ -31,18 +31,18 @@ We argue in this paper that one should not write interpreters for dynamic languages manually but rather use meta-programming techniques and raise the overall level at which they are are implemented. We believe this to be -ultimately a better investment of efforts than the development of more advanced +ultimately a better investment of efforts than the development of more and more advanced general-purpose object oriented VMs. \medskip Dynamic languages are traditionally implemented by writing a virtual machine for them, centered around an interpreter and/or a built-in compiler and providing -the object model and memory management. When a language becomes more successful, +the object model and memory management. When a language becomes more popular, the limitations of such an implementation lead to the emergence of alternative implementations that try to solve some of the problems. Another reason for new implementations is the desire to have the language integrate well with existing, -well-tuned object-oriented virtual machine like Java's. In this paper, we +well-tuned object-oriented virtual machine like the Java Virtual Machine. In this paper, we describe the mechanisms that lead to an abundance of implementations, and explore some of the limitations of standard VMs. We propose a different complementary alternative to writing VMs by hand and dealing with low-level @@ -68,8 +68,8 @@ typically have more efficient implementations based on just-in-time code generation. -The efforts required to build a new virtual machine are relatively -important. This is particularly true for languages which are complex +The effort required to build a new virtual machine are relatively +large. This is particularly true for languages which are complex and in constant evolution. Language implementation communities from an open-source or academic context have only limited resources. Therefore they cannot afford to have a highly complex implementation and often chose simpler @@ -86,9 +86,9 @@ work-power from the reference implementation. An alternative to writing implementations in C is to build them on top of one of the newer object oriented virtual machines (``OO VM'') such as the JVM or the CLR. This is often wanted by -the community anyway, since it leads to better re-usability of libraries of +the community anyway, since it leads to the ability to re-use the libraries of these platforms. However, if a C implementation existed before the -implementation of such a VM is started, this enters in conflict with the goal of +implementation of such a VM is started, this enters into conflict with the goal of having to maintain essentially a single, simple enough implementation for a given programming language: as the language becomes popular, there will be a demand for having it run on various platforms -- high-level VMs as well as @@ -141,9 +141,9 @@ With the emergence of .NET and the JVM as interesting language implementation platforms, an argument that is sometimes made is that -communities should only develop an implementation of their language for -one of these platforms.\footnote{Preferably the argument author's favourite -one.} +communities should only develop an implementation of their language +for one of these platforms (preferably the argument author's favourite +one). \subsection{Assessing the Advantages of Implementing a Language on Top of OO VMs} @@ -152,9 +152,7 @@ implementing it in C. Let's take a look at the advantages that are usually cited for basing a language implementation of a dynamic language on a standard object oriented -virtual machine, for example the JVM or the CLR. The central theme of the -benefits of OO VMs is the ability to implement certain hard things only once -and share the benefits between all language implementations on top of the OO VM. +virtual machine, for example the JVM or the CLR. \begin{itemize} \item @@ -191,7 +189,7 @@ \emph{Ease of implementation:} The implementation of a language on top of an OO VM is easier because it starts at a higher level than C. Usually a high-level language like Java or C\# is used for the language implementation, -which both offer the language implementer a much higher level of abstraction +both of which offer the language implementer a much higher level of abstraction than when implementing in C. \item @@ -203,7 +201,10 @@ \noindent -At a closer look, some of these advantages are only partially true in practice: +The central theme of the benefits of OO VMs is the ability to +implement certain hard things only once and share the benefits between +all language implementations on top of the OO VM. At a closer look, +some of these advantages are only partially true in practice: \begin{itemize} \item @@ -266,7 +267,7 @@ language communities. Although most individual implementations exist for good reasons, the sum of all of them and the need to keep them synchronized with the reference implementations lead to a lot of duplicated work and division of -efforts. This is especially true for open source languages which tend to evolve +effort. This is especially true for open source languages which tend to evolve quickly. At any one point in time some of the implementations will lag behind which makes writing code which can work on all of the implementations harder. @@ -277,7 +278,7 @@ issues are involved in such a decision. On the other hand deciding on one single object and execution model would not be an equally good fit for all languages. -In the next section we are exploring a different approach for implementing +In the next section we explore a different approach for implementing dynamic languages that we hope is able to solve many of the problems of implementing a language, in particular the problem of an explosion of the number of implementations. From arigo at codespeak.net Sun May 27 23:28:22 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 27 May 2007 23:28:22 +0200 (CEST) Subject: [pypy-svn] r43756 - pypy/extradoc/talk/dyla2007 Message-ID: <20070527212822.2185D80BB@code0.codespeak.net> Author: arigo Date: Sun May 27 23:28:16 2007 New Revision: 43756 Modified: pypy/extradoc/talk/dyla2007/dyla.tex Log: Review by exarkun. A couple of open issues, intermediate check-in. Modified: pypy/extradoc/talk/dyla2007/dyla.tex ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.tex (original) +++ pypy/extradoc/talk/dyla2007/dyla.tex Sun May 27 23:28:16 2007 @@ -30,21 +30,21 @@ We argue in this paper that one should not write interpreters for dynamic languages manually but rather use meta-programming techniques and raise the -overall level at which they are are implemented. We believe this to be +overall level at which they are implemented. We believe this to be ultimately a better investment of efforts than the development of more and more advanced general-purpose object oriented VMs. \medskip -Dynamic languages are traditionally implemented by writing a virtual machine for -them, centered around an interpreter and/or a built-in compiler and providing +Dynamic languages are traditionally implemented by writing a virtual machine +centered around an interpreter and/or a built-in compiler and providing the object model and memory management. When a language becomes more popular, the limitations of such an implementation lead to the emergence of alternative implementations that try to solve some of the problems. Another reason for new implementations is the desire to have the language integrate well with existing, well-tuned object-oriented virtual machine like the Java Virtual Machine. In this paper, we -describe the mechanisms that lead to an abundance of implementations, and -explore some of the limitations of standard VMs. We propose a different +describe the mechanisms that lead to an abundance of implementations and +explore some of the limitations of standard VMs. We propose a complementary alternative to writing VMs by hand and dealing with low-level details, validated by the PyPy project: flexibly generating virtual machines from a single abstract language ``specification'', inserting features and @@ -58,22 +58,22 @@ \section{Introduction} Dynamic languages are traditionally implemented by writing a virtual -machine for them in a low-level language like C, or in a language that +machine for them in a low-level language like C or in a language that can relatively easily be turned into C. The machine implements an object model supporting the high level dynamic language's objects. It typically provides features like automatic garbage collection. Recent languages like Python, Ruby, Perl and JavaScript have complicated semantics which are most easily mapped to a simple interpreter operating on syntax trees or bytecode; simpler languages like Lisp and Self -typically have more efficient implementations based on just-in-time code +typically have more efficient implementations based on code generation. -The effort required to build a new virtual machine are relatively +The effort required to build a new virtual machine is relatively large. This is particularly true for languages which are complex and in constant evolution. Language implementation communities from an open-source or academic context have only limited resources. Therefore they -cannot afford to have a highly complex implementation and often chose simpler -techniques even if that entails lower execution speed. Similarly fragmentation +cannot afford to have a highly complex implementation and often choose simpler +techniques even if that entails lower execution speed. Similarly, fragmentation (for example because of other implementations of the same language) is a problem because it divides available resources. All these points also apply to the implementation of domain-specific languages where it is important to keep @@ -83,7 +83,7 @@ forces the language implementer to deal with many low-level details (like garbage collection and threading issues). Limitations of the C implementation lead to alternative implementations which draw -work-power from the reference implementation. An alternative to writing +resources from the reference implementation. An alternative to writing implementations in C is to build them on top of one of the newer object oriented virtual machines (``OO VM'') such as the JVM or the CLR. This is often wanted by the community anyway, since it leads to the ability to re-use the libraries of @@ -91,24 +91,24 @@ implementation of such a VM is started, this enters into conflict with the goal of having to maintain essentially a single, simple enough implementation for a given programming language: as the language becomes popular, there will be a -demand for having it run on various platforms -- high-level VMs as well as +demand to have it run on various platforms -- high-level VMs as well as C-level environments. -The argument we will make in the present paper is that it is possible to +In this paper, we will argue that it is possible to benefit from and integrate with OO VMs while keeping the dynamic -language implemented by a single, simple source code base. The idea is +language implemented with a single, simple source code base. The idea is to write an interpreter for that language in another sufficiently high-level but less dynamic language. This interpreter plays the role -of a specification for the dynamic language. With a good enough +of a specification for the dynamic language. With a sufficiently capable translation toolchain we can then generate whole virtual machines from -this specification -- either full custom VMs for C-level operating -systems, or layers on top of various OO VMs. In other words, +this specification -- either wholely custom VMs for C-level operating +systems or VMs layered on top of various OO VMs. In other words, meta-programming techniques can be used to successfully replace a foreseeable one-VM-fits-all standardization attempt. -The argument boils down to: VMs for dynamic languages should not be -written by hand! The justification is based on the -PyPy project, which proves that the approach is +The crux of the argument is that VMs for dynamic languages should not be +written by hand! The PyPy project is the justification, +proving that the approach is feasible in practice. Just as importantly, it also brings new insights and concrete benefits in term of flexibility and performance that go beyond the state of the art. @@ -131,16 +131,16 @@ \cite{cpy251}, is a simple recursive interpreter. \implname{Stackless Python} \cite{stackless} is a fork that adds micro-threading capabilities to Python. One of the reasons for not incorporating it back -into CPython was that it was felt that they would make the +into CPython was that it was felt that this would make the implementation too complex. Another implementation of the Python language is \implname{Psyco} \cite{psyco-software}, which adds a JIT-compiler to CPython. Finally, \implname{Jython} is a -re-implementation for the Java VM and \implname{IronPython} one for -.NET. All of these need to be kept in sync with the relatively fast +re-implementation for the Java VM and \implname{IronPython} for +the CLR. All of these need to be kept in sync with the relatively fast evolution of the language. -With the emergence of .NET and the JVM as interesting language -implementation platforms, an argument that is sometimes made is that +With the emergence of the CLR and the JVM as interesting language +implementation platforms, it is sometimes argued that communities should only develop an implementation of their language for one of these platforms (preferably the argument author's favourite one). @@ -160,13 +160,13 @@ object model and all the languages implemented on top of it are using it, it is easier to integrate the languages that are running on top of the VM. This allows reuse of libraries between all the implemented languages. This is -typically the most important reason for wanting an implementation on the VM in +typically the most important reason to want an implementation on the VM in the first place. \item \emph{Cross-platform portability:} Only the underlying VM has to be ported to various hardware architectures and operating systems. The languages implemented -on top can then be run without change in various environments. +on top of it can then be run without change in various environments. \item \emph{Better tools:} Better IDEs, debuggers and profilers. @@ -181,8 +181,8 @@ \item \emph{Better performance:} Similarly, object-oriented VMs usually come with a highly tuned just-in-time compiler to make them perform well without requiring -ahead-of-time compilation to machine language. This in addition with the -previous point leads to much better performance of the languages running on top +ahead-of-time compilation to machine language. This, in addition to the +previous point, leads to much better performance of the languages running on top of the VM. \item @@ -193,8 +193,8 @@ than when implementing in C. \item -\emph{A single unified implementation base:} The .NET and Java VMs are trying -to position themselves as all-encompassing platforms; if one succeeds, then +\emph{A single unified implementation base:} The CLR and JVM are trying +to position themselves as all-encompassing platforms; if one succeeds, implementations of the dynamic language for other platforms might no longer be required. \end{itemize} @@ -211,12 +211,12 @@ \emph{Better performance:} So far it seems that performance of highly dynamic languages is not actually significantly improved on OO VMs. Jython is around 5 -times slower than CPython, for IronPython\footnote{Python on .NET, which -gives up on some features to improve performance} +times slower than CPython, for IronPython (which +gives up on at least one feature -- frame objects -- to improve performance) the figures vary but it is mostly within the same order of magnitude as CPython. The most important reason for this is that the VM's JIT compilers are optimized for specific usage patterns -that are common in the main language of the OO VM. To get good speeds the +that are common in the primary language of the OO VM. To achieve good speeds, the language implementers would have to carefully produce code that matches these usage patterns, which is not a simple task. @@ -225,60 +225,60 @@ higher memory overhead to start with (XXX ref) \item -\emph{Cross-platform portability:} While this is true to some extend, the +\emph{Cross-platform portability:} While this is true to some extent, the situation with regard to portability is not significantly improved compared to -e.g. C/Posix, which is relatively portable too. Also portability sometimes +e.g. C/POSIX, which is relatively portable as well. Also, portability sometimes comes at the price of performance, because even if the OO VM is running on a particular hardware architecture it is not clear that the JIT is tuned for this -architecture too or working at all, which leads to significantly less +architecture (or working at all), leading to significantly reduced speed. \item \emph{Ease of implementation:} This point is disputable. On the one hand, OO -VMs typically allow the language implementor to start at a higher level. On the -other hand they also enforce a specific object and execution model. This means +VMs typically allow the language implementer to start at a higher level. On the +other hand, they also enforce a specific object and execution model. This means that the concepts of the implemented language need to be mapped to the -execution model of the underlying VM, which may be easy or not, depending very +execution model of the underlying VM, which may or may not be easy, depending very much on the language in question. -An example where this mapping does not work too well is Prolog. While there +An example where this mapping does not work very well is Prolog. While there exist several implementations of Prolog on top of the JVM \cite{prologcafe} -\cite{InterProlog} and also one on .NET \cite{psharp}, +\cite{InterProlog} and one on .NET \cite{psharp}, they are not particular efficient, especially when compared to good Prolog VMs -in written in C. This is mostly because the Prolog execution model, which -involves backtracking and deep recursion does not fit the JVM and .NET very +written in C. This is mostly because the Prolog execution model, which +involves backtracking and deep recursion, does not fit the JVM and .NET very well. Therefore the Prolog implementations on top of OO VMs resort to models -that is quite unnatural both for the OO VM and for Prolog. +that are quite unnatural both for the OO VM and for Prolog. -Another important point that make implementations of languages on top of OO VMs -harder is that typically they don't support meta-programming very well, or only +Another important point that makes implementation of languages on top of OO VMs +harder is that typically OO VMs don't support meta-programming very well, or do so only at the bytecode level. \end{itemize} -On the other hand some of the benefits are real and very useful, the most -prominent being the easy interaction with the rest of the VM. Furthermore there -is better tool support and better GCs. Also for languages where the execution +Nevertheless, some of the benefits are real and very useful, the most +prominent of which being easy interaction with the rest of the VM. Furthermore, there +is better tool support and better GCs. Also, for languages where the execution model fits the OO VM well, many of the disadvantages disappear. \subsection{The Cost of Implementation-Proliferation} -The described proliferation of language implementations is a big problem for +The described proliferation of language implementations is a large problem for language communities. Although most individual implementations exist for good reasons, the sum of all of them and the need to keep them synchronized with the -reference implementations lead to a lot of duplicated work and division of -effort. This is especially true for open source languages which tend to evolve +reference implementations leads to a significant amount of duplicated work and division of +effort; this is especially true for open source languages which tend to evolve quickly. At any one point in time some of the implementations will lag behind which makes writing code which can work on all of the implementations harder. Implementing a language on top of a OO VM has many advantages, so some people propose the solution of standardizing on one particular OO VM to not have -to maintain implementations for several of them. While this would in theory -alleviate the problem it is unlikely to happen. On the one hand many political -issues are involved in such a decision. On the other hand deciding on one single +to maintain implementations for several of them. While this would, in theory, +alleviate the problem it is unlikely to happen. On the one hand, many political +issues are involved in such a decision. On the other hand, deciding on a single object and execution model would not be an equally good fit for all languages. -In the next section we explore a different approach for implementing +In the next section, we explore a different approach for implementing dynamic languages that we hope is able to solve many of the problems of implementing a language, in particular the problem of an explosion of the number of implementations. @@ -293,13 +293,13 @@ argue that this approach gives many of the benefits usually expected by an implementer when he decides to target an existing object-oriented virtual machine. It also gives other benefits that we will describe -- -mostly in term of flexibility. But most importantly, it lets a +mostly in term of flexibility. Most importantly, it lets a community write a single source implementation of the language, avoiding -the time-consuming task of keeping multiple ones in sync. The single +the time-consuming task of keeping several of them in sync. The single source can be used to generate either custom VMs for C-like -environments, or interpreters running on top of OO VMs. It makes it +environments or interpreters running on top of OO VMs. This makes it practical to experiment with large changes to the language and with -entirely new languages, like domain-specific languages, while at any +entirely new languages, such as domain-specific languages, while at any time being able to run the implemented language in a variety of environments, from C/Posix to the JVM to .NET. @@ -308,10 +308,10 @@ We implemented this idea in the PyPy project \cite{pypy}. The dynamic language for which we wrote an interpreter is Python. It is a language which, because of its size and rather intricate semantics, is a good target for -our approach, in the following sense: its previous reimplementations +our approach in the following sense: its previous reimplementations (Jython for the JVM and IronPython for .NET) have each proved to be very -time-consuming to maintain. Our implementation is by construction -easier to maintain, and extremely portable (including to C/Posix, to the +time-consuming to maintain. Our implementation is, by construction, +easier to maintain and extremely portable (including to C/Posix, to the JVM and to .NET). In meta-programming terms, the PyPy architecture is as follows: @@ -319,7 +319,7 @@ \begin{itemize} \item -we use a very expressive \emph{object language} (RPython -- an analyzable +We use a very expressive \emph{object language} (RPython -- an analyzable subset of Python) as the language in which the complete Python interpreter is written, together with the implementation of its built-in types. The language is still close to Python, e.g. it is @@ -329,18 +329,18 @@ management, no pieces of C or C-level code. \item -we use a very expressive metalanguage (namely regular Python) to +We use a very expressive metalanguage (namely regular Python) to perform the analysis of RPython code (control flow and data flow construction, type inference, etc.) and its successive transformations. \item -this meta-programming component of PyPy is called the \emph{translation +This meta-programming component of PyPy is called the \emph{translation framework}, as it translates RPython source code (i.e. the full Python interpreter) into lower-level code. Its purpose is to add aspects to and specialize the interpreter to fit a selectable virtual or hardware runtime environment. This either turns the interpreter into a -standalone virtual machine, or integrates it into an existing OO VM. +standalone virtual machine or integrates it into an existing OO VM. The necessary support code -- e.g. the garbage collector when targeting C -- is itself written in RPython in much the same spirit that the Jikes RVM's GCs are written in Java \cite{JikesGC}; as needed, it is @@ -358,8 +358,8 @@ include the spectacular speed-ups obtained in some cases by the JIT compiler described in section \ref{subsect:dynamic_compilers}. -In the sequel, we will focus on the relative advantages and -inconvenients of the PyPy approach compared to the approach of +In the sequel (??? - what is the sequel), we will focus on the relative advantages and +inconveniences of the PyPy approach compared to the approach of hand-writing a language implementation on top of an OO VM. @@ -367,12 +367,12 @@ Our approach -- a single ``meta-written'' implementation -- naturally leads to language implementations that have various advantages over the -``hand-written'' implementations. First of all, it is a single-source +``hand-written'' implementations. Firstly, it is a single-source approach -- we explicitly seek to solve the problem of proliferation of -implementations. In the sequel, we will show that this goal can be -achieved without giving up on the advantages of hand-written -implementations for OO VMs. Moreover, there are additional advantages --- in our opinion significant enough to hint that meta-programming, +implementations. In the sequel (???), we will show that this goal can be +achieved without giving up the advantages of hand-written +implementations for OO VMs. Moreover, there are additional advantages which, +in our opinion, are significant enough to hint that meta-programming, though not widely used in general-purpose programming, is an essential tool in a language implementer's toolbox. @@ -381,7 +381,7 @@ A first point is that it makes interpreters easy to write, update and generally experiment with. More expressiveness helps at all levels: our Python interpreter is written in RPython as a relatively simple -interpreter, in some respects easier to understand than CPython. We are +interpreter and is, in some respects, easier to understand than CPython. We are using its high level and flexibility to quickly experiment with features or implementation techniques in ways that would, in a traditional approach, require pervasive changes to the source code. For example, @@ -427,7 +427,7 @@ of analyzing and transforming the high-level source code and generating lower-level output in various languages? -Although it is able to generate, among other things, a complete custom +Although it is able to generate, among other things, a complete, custom VM for C-like environments, we found that the required effort that must be put into the translation toolchain was still much lower than that of writing a good-quality OO VM. A reason is that a translation toolchain @@ -458,12 +458,12 @@ the Jikes RVM \cite{JikesGC}. As they are in Java, it should be relatively straightforward to add a translation step that turns one of them into RPython (or directly into our RPython-level intermediate -representation) and integrate it with the rest of the program being +representation) and integrates it with the rest of the program being translated. In summary, developing a meta-programming translation toolchain requires work, but it can be done incrementally, it can reuse existing code, and -it gives a toolchain that is itself highly reusable and flexible in +it results in a toolchain that is itself highly reusable and flexible in nature. \subsection{Dynamic compilers} @@ -480,12 +480,12 @@ The deeper problem with the otherwise highly-tuned JIT compilers of the OO VMs is that they are not a very good match for running dynamic languages. It might be possible to tune a general-purpose JIT compiler -enough, and write the dynamic language implementation accordingly, so +enough and write the dynamic language implementation accordingly so that most of the bookkeeping work involved in running the dynamic language can be removed -- dispatching, boxing, unboxing... However this has not been demonstrated yet. -By far the fastest Python implementation, Psyco \cite{psyco-software} contains a +By far the fastest Python implementation, Psyco \cite{psyco-software}, contains a hand-written language-specific dynamic compiler. PyPy's translation tool-chain is able to extend the generated VMs with an automatically generated dynamic compiler that uses techniques similar to those of Psyco @@ -514,13 +514,13 @@ %XXX doesn't look entirely nice \begin{itemize} \item \emph{Do not write dynamic language implementations ``by hand''.} -Writing them more abstractly, at a higher level, has mostly only -advantages, among them avoiding a proliferation of implementations +Writing them more abstractly, at a higher level, has primarily only +advantages, among them the avoidance of a proliferation of implementations growing out of sync. Writing interpreters both flexibly and efficiently -is difficult, and meta-programming is a good way to achieve it. +is difficult and meta-programming is a good way to achieve it. Moreover, this is not incompatible with targetting and benefiting from -existing good object-oriented virtual machines like the Java and .NET -ones. +existing high-quality object-oriented virtual machines like those of the Java and .NET. + \item \emph{Do not write VMs ``by hand''.} Writing language-specific virtual machines is a time-consuming task for @@ -539,7 +539,7 @@ \item \emph{Let's write more meta-programming translation toolchains.} Aside from the advantages described in section \ref{sect:metaprogramming}, a translation toolchain need not be -standardized for inter-operability, but can be tailored to the needs of +standardized for inter-operability but can be tailored to the needs of each project. Diversity is good; there is no need to attempt to standardize on a single OO VM. \end{itemize} From fijal at codespeak.net Sun May 27 23:29:53 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 27 May 2007 23:29:53 +0200 (CEST) Subject: [pypy-svn] r43757 - in pypy/branch/kill-ctypes/pypy/rpython/lltypesystem: . test Message-ID: <20070527212953.C19D680BB@code0.codespeak.net> Author: fijal Date: Sun May 27 23:29:53 2007 New Revision: 43757 Added: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rfficache.py pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rfficache.py Log: Add a sizeof gcc invoker Added: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rfficache.py ============================================================================== --- (empty file) +++ pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rfficache.py Sun May 27 23:29:53 2007 @@ -0,0 +1,31 @@ + +""" This file creates and maintains _cache/rtypes.py, which +keeps information about C type sizes on various platforms +""" + +import py +from pypy.translator.tool.cbuild import build_executable +from subprocess import PIPE, Popen +from pypy.tool.udir import udir + +def sizeof_c_type(c_typename, includes={}): + includes['stdio.h'] = True + include_string = "\n".join(["#include <%s>" % i for i in includes.keys()]) + c_source = py.code.Source(''' + // includes + %s + + // checking code + int main(void) + { + printf("%%d\\n", sizeof(%s)); + return (0); + } + ''' % (include_string, c_typename)) + c_file = udir.join("typetest.c") + c_file.write(c_source) + + c_exec = build_executable([str(c_file)]) + pipe = Popen(c_exec, stdout=PIPE) + pipe.wait() + return int(pipe.stdout.read()) * 8 Added: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rfficache.py ============================================================================== --- (empty file) +++ pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rfficache.py Sun May 27 23:29:53 2007 @@ -0,0 +1,6 @@ + +from pypy.rpython.lltypesystem.rfficache import sizeof_c_type + +def test_sizeof_c_type(): + sizeofchar = sizeof_c_type('char') + assert sizeofchar == 8 From mwh at codespeak.net Sun May 27 23:36:33 2007 From: mwh at codespeak.net (mwh at codespeak.net) Date: Sun, 27 May 2007 23:36:33 +0200 (CEST) Subject: [pypy-svn] r43758 - pypy/extradoc/talk/dyla2007 Message-ID: <20070527213633.BF41280BC@code0.codespeak.net> Author: mwh Date: Sun May 27 23:36:33 2007 New Revision: 43758 Modified: pypy/extradoc/talk/dyla2007/dyla.tex Log: spell check and small remaining fixes Modified: pypy/extradoc/talk/dyla2007/dyla.tex ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.tex (original) +++ pypy/extradoc/talk/dyla2007/dyla.tex Sun May 27 23:36:33 2007 @@ -101,7 +101,7 @@ high-level but less dynamic language. This interpreter plays the role of a specification for the dynamic language. With a sufficiently capable translation toolchain we can then generate whole virtual machines from -this specification -- either wholely custom VMs for C-level operating +this specification -- either wholly custom VMs for C-level operating systems or VMs layered on top of various OO VMs. In other words, meta-programming techniques can be used to successfully replace a foreseeable one-VM-fits-all standardization attempt. @@ -301,7 +301,7 @@ practical to experiment with large changes to the language and with entirely new languages, such as domain-specific languages, while at any time being able to run the implemented language in a variety of -environments, from C/Posix to the JVM to .NET. +environments, from C/POSIX to the JVM to .NET. \subsection{PyPy architecture} @@ -311,7 +311,7 @@ our approach in the following sense: its previous reimplementations (Jython for the JVM and IronPython for .NET) have each proved to be very time-consuming to maintain. Our implementation is, by construction, -easier to maintain and extremely portable (including to C/Posix, to the +easier to maintain and extremely portable (including to C/POSIX, to the JVM and to .NET). In meta-programming terms, the PyPy architecture is as follows: @@ -350,7 +350,7 @@ A detailed description of this translation process is beyond the scope of the present paper; it can be found in \cite{pypyvmconstruction}. The actual Python interpreter of PyPy and the results we achieved by translating it to C, LLVM -\cite{LLVM} and .NET are described in \cite{architecture} \cite{translationdoc} +\cite{LLVM} and .NET are described in \cite{architecture} \cite{translationdoc}. These results show that the approach is practical and gives results whose performance is within the same order of magnitude (within a factor of 2 and improving) of the hand-written, @@ -518,7 +518,7 @@ advantages, among them the avoidance of a proliferation of implementations growing out of sync. Writing interpreters both flexibly and efficiently is difficult and meta-programming is a good way to achieve it. -Moreover, this is not incompatible with targetting and benefiting from +Moreover, this is not incompatible with targeting and benefiting from existing high-quality object-oriented virtual machines like those of the Java and .NET. @@ -548,7 +548,7 @@ unexplored design space; it is likely that some of the choices made in PyPy will turn out to be suboptimal. We are hoping that other toolchains will emerge over time, exploring other aspects and proposing -other solutions. By their ``meta'' nature, these multiple approachs +other solutions. By their ``meta'' nature, these multiple approaches should be easier to bridge together than, say, multiple OO VMs with different object and runtime models. We believe that further research in this area might open the door to better solutions for From santagada at codespeak.net Sun May 27 23:36:57 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Sun, 27 May 2007 23:36:57 +0200 (CEST) Subject: [pypy-svn] r43759 - pypy/dist/pypy/lang/js/test Message-ID: <20070527213657.5172380BB@code0.codespeak.net> Author: santagada Date: Sun May 27 23:36:56 2007 New Revision: 43759 Added: pypy/dist/pypy/lang/js/test/test_operations.py Log: new unittest module Added: pypy/dist/pypy/lang/js/test/test_operations.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/lang/js/test/test_operations.py Sun May 27 23:36:56 2007 @@ -0,0 +1,24 @@ +import py +from pypy.lang.js import interpreter +from pypy.lang.js.operations import * +from pypy.lang.js.jsobj import W_Number, empty_context + +class MOCKNode(Node): + def __init__(self, pos, ret): + self.pos = pos + self.ret = ret + + def eval(self, ctx): + return self.ret + +POSDEF = Position() +VALDEF = MOCKNode(POSDEF, 1) + +def test_return(): + ctx = empty_context() + r = Return(POSDEF, VALDEF) + block = Block(POSDEF, [r]) + selements = SourceElements(POSDEF, [], {}, [r]) + br = block.execute(ctx) + sr = selements.execute(ctx) + assert br == 1 and sr == 1 From arigo at codespeak.net Sun May 27 23:39:15 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 27 May 2007 23:39:15 +0200 (CEST) Subject: [pypy-svn] r43760 - pypy/extradoc/talk/dyla2007 Message-ID: <20070527213915.99EE280BF@code0.codespeak.net> Author: arigo Date: Sun May 27 23:39:15 2007 New Revision: 43760 Modified: pypy/extradoc/talk/dyla2007/dyla.tex Log: Small fixes. Modified: pypy/extradoc/talk/dyla2007/dyla.tex ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.tex (original) +++ pypy/extradoc/talk/dyla2007/dyla.tex Sun May 27 23:39:15 2007 @@ -102,7 +102,7 @@ of a specification for the dynamic language. With a sufficiently capable translation toolchain we can then generate whole virtual machines from this specification -- either wholly custom VMs for C-level operating -systems or VMs layered on top of various OO VMs. In other words, +systems or as a layer on top of various OO VMs. In other words, meta-programming techniques can be used to successfully replace a foreseeable one-VM-fits-all standardization attempt. @@ -358,7 +358,7 @@ include the spectacular speed-ups obtained in some cases by the JIT compiler described in section \ref{subsect:dynamic_compilers}. -In the sequel (??? - what is the sequel), we will focus on the relative advantages and +In the sequel, we will focus on the relative advantages and inconveniences of the PyPy approach compared to the approach of hand-writing a language implementation on top of an OO VM. @@ -369,7 +369,7 @@ leads to language implementations that have various advantages over the ``hand-written'' implementations. Firstly, it is a single-source approach -- we explicitly seek to solve the problem of proliferation of -implementations. In the sequel (???), we will show that this goal can be +implementations. In the sequel, we will show that this goal can be achieved without giving up the advantages of hand-written implementations for OO VMs. Moreover, there are additional advantages which, in our opinion, are significant enough to hint that meta-programming, From arigo at codespeak.net Sun May 27 23:42:23 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 27 May 2007 23:42:23 +0200 (CEST) Subject: [pypy-svn] r43761 - pypy/extradoc/talk/dyla2007 Message-ID: <20070527214223.2654180BB@code0.codespeak.net> Author: arigo Date: Sun May 27 23:42:22 2007 New Revision: 43761 Modified: pypy/extradoc/talk/dyla2007/dyla.tex Log: Define the abbreviation "VM". Modified: pypy/extradoc/talk/dyla2007/dyla.tex ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.tex (original) +++ pypy/extradoc/talk/dyla2007/dyla.tex Sun May 27 23:42:22 2007 @@ -32,7 +32,7 @@ languages manually but rather use meta-programming techniques and raise the overall level at which they are implemented. We believe this to be ultimately a better investment of efforts than the development of more and more advanced -general-purpose object oriented VMs. +general-purpose object oriented virtual machines (VMs). \medskip From arigo at codespeak.net Sun May 27 23:48:11 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 27 May 2007 23:48:11 +0200 (CEST) Subject: [pypy-svn] r43762 - pypy/extradoc/talk/dyla2007 Message-ID: <20070527214811.5357C80BB@code0.codespeak.net> Author: arigo Date: Sun May 27 23:48:10 2007 New Revision: 43762 Modified: pypy/extradoc/talk/dyla2007/dyla.pdf pypy/extradoc/talk/dyla2007/dyla.tex Log: Avoid "in sync" and "out of sync". Modified: pypy/extradoc/talk/dyla2007/dyla.pdf ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/dyla2007/dyla.tex ============================================================================== --- pypy/extradoc/talk/dyla2007/dyla.tex (original) +++ pypy/extradoc/talk/dyla2007/dyla.tex Sun May 27 23:48:10 2007 @@ -136,7 +136,7 @@ language is \implname{Psyco} \cite{psyco-software}, which adds a JIT-compiler to CPython. Finally, \implname{Jython} is a re-implementation for the Java VM and \implname{IronPython} for -the CLR. All of these need to be kept in sync with the relatively fast +the CLR. All of these need to be kept synchronized with the relatively fast evolution of the language. With the emergence of the CLR and the JVM as interesting language @@ -295,7 +295,7 @@ virtual machine. It also gives other benefits that we will describe -- mostly in term of flexibility. Most importantly, it lets a community write a single source implementation of the language, avoiding -the time-consuming task of keeping several of them in sync. The single +the time-consuming task of keeping several of them synchronized. The single source can be used to generate either custom VMs for C-like environments or interpreters running on top of OO VMs. This makes it practical to experiment with large changes to the language and with @@ -497,8 +497,9 @@ with the necessary support code and the rest of the regular interpreter. This result was one of the major goals and motivations for the whole -approach. By construction, the JIT does not get out of sync when the -language evolves, and any code written in the dynamic language runs +approach. By construction, the JIT stays synchronized with its VM +and with the language when it evolves, +and any code written in the dynamic language runs correctly under the JIT. Some very simple Python examples run more than 100 times faster. At the time of this writing this is still rather experimental, and the techniques involved are well beyond the scope of @@ -515,8 +516,8 @@ \begin{itemize} \item \emph{Do not write dynamic language implementations ``by hand''.} Writing them more abstractly, at a higher level, has primarily only -advantages, among them the avoidance of a proliferation of implementations -growing out of sync. Writing interpreters both flexibly and efficiently +advantages, among them the avoidance of a proliferation of diverging +implementations. Writing interpreters both flexibly and efficiently is difficult and meta-programming is a good way to achieve it. Moreover, this is not incompatible with targeting and benefiting from existing high-quality object-oriented virtual machines like those of the Java and .NET. From fijal at codespeak.net Mon May 28 00:20:23 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 28 May 2007 00:20:23 +0200 (CEST) Subject: [pypy-svn] r43763 - in pypy/branch/kill-ctypes/pypy/rpython/lltypesystem: . test Message-ID: <20070527222023.3E10380BC@code0.codespeak.net> Author: fijal Date: Mon May 28 00:20:22 2007 New Revision: 43763 Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rfficache.py pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rfficache.py Log: Add caching mechanism Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rfficache.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rfficache.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rfficache.py Mon May 28 00:20:22 2007 @@ -8,7 +8,7 @@ from subprocess import PIPE, Popen from pypy.tool.udir import udir -def sizeof_c_type(c_typename, includes={}): +def sizeof_c_type(c_typename, includes={}, compiler_exe=None): includes['stdio.h'] = True include_string = "\n".join(["#include <%s>" % i for i in includes.keys()]) c_source = py.code.Source(''' @@ -25,7 +25,39 @@ c_file = udir.join("typetest.c") c_file.write(c_source) - c_exec = build_executable([str(c_file)]) + c_exec = build_executable([str(c_file)], compiler_exe=compiler_exe) pipe = Popen(c_exec, stdout=PIPE) pipe.wait() return int(pipe.stdout.read()) * 8 + +def machine_key(): + """ Key unique to machine type, but general enough to share + it between eg different kernels + """ + import platform + return platform.processor(), platform.architecture(), platform.system() + +TYPES = [] +for _name in 'char short int long'.split(): + for name in (_name, 'unsigned ' + _name): + TYPES.append(name) +TYPES.append('long long') +TYPES.append('unsigned long long') +TYPES.append('size_t') + +def get_type_sizes(filename, platform_key=machine_key(), types=TYPES, + compiler_exe=None): + try: + mod = py.path.local(filename).pyimport() + platforms = mod.platforms + except (ImportError, py.error.ENOENT): + platforms = {} + try: + return platforms[platform_key] + except KeyError: + value = dict([(i, sizeof_c_type(i, compiler_exe=compiler_exe)) + for i in types]) + platforms[platform_key] = value + py.path.local(filename).write('platforms = ' + repr(platforms)) + return value + Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rfficache.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rfficache.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rfficache.py Mon May 28 00:20:22 2007 @@ -1,6 +1,21 @@ -from pypy.rpython.lltypesystem.rfficache import sizeof_c_type +from pypy.rpython.lltypesystem.rfficache import * +from pypy.tool.udir import udir def test_sizeof_c_type(): sizeofchar = sizeof_c_type('char') assert sizeofchar == 8 + +def test_gettypesizes(): + tmpfile = udir.join("somecrappyfile.py") + assert get_type_sizes(tmpfile)['char'] == 8 + # this should not invoke a compiler + assert get_type_sizes(tmpfile, compiler_exe='xxx')['char'] == 8 + +def test_gettypesizes_platforms(): + tmpfile = udir.join("plat.py") + tmpfile.write(py.code.Source(""" + platforms = {'xxx':{'char':4}} + """)) + assert get_type_sizes(tmpfile)['char'] == 8 + assert get_type_sizes(tmpfile, platform_key='xxx', compiler_exe='xxx')['char'] == 4 From fijal at codespeak.net Mon May 28 00:32:27 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 28 May 2007 00:32:27 +0200 (CEST) Subject: [pypy-svn] r43764 - pypy/branch/kill-ctypes/pypy/rpython/lltypesystem Message-ID: <20070527223227.42C3A80BC@code0.codespeak.net> Author: fijal Date: Mon May 28 00:32:26 2007 New Revision: 43764 Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rfficache.py Log: Add cache Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rfficache.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rfficache.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rfficache.py Mon May 28 00:32:26 2007 @@ -37,6 +37,8 @@ import platform return platform.processor(), platform.architecture(), platform.system() +# XXX add float types as well here + TYPES = [] for _name in 'char short int long'.split(): for name in (_name, 'unsigned ' + _name): @@ -58,6 +60,12 @@ value = dict([(i, sizeof_c_type(i, compiler_exe=compiler_exe)) for i in types]) platforms[platform_key] = value - py.path.local(filename).write('platforms = ' + repr(platforms)) + comment = "# this is automatically generated cache files for c types\n" + py.path.local(filename).write(comment + 'platforms = ' + + repr(platforms) + "\n") return value +from pypy.tool import autopath +CACHE = py.path.local(autopath.pypydir).join('_cache').join('typecache.py') +platform = get_type_sizes(CACHE) + From fijal at codespeak.net Mon May 28 00:34:36 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 28 May 2007 00:34:36 +0200 (CEST) Subject: [pypy-svn] r43765 - pypy/branch/kill-ctypes/pypy/rpython/lltypesystem Message-ID: <20070527223436.2B63080BC@code0.codespeak.net> Author: fijal Date: Mon May 28 00:34:35 2007 New Revision: 43765 Added: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/typecache.py Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rfficache.py Log: Really add cache Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rfficache.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rfficache.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rfficache.py Mon May 28 00:34:35 2007 @@ -66,6 +66,6 @@ return value from pypy.tool import autopath -CACHE = py.path.local(autopath.pypydir).join('_cache').join('typecache.py') +CACHE = py.magic.autopath().dirpath().join('typecache.py') platform = get_type_sizes(CACHE) Added: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/typecache.py ============================================================================== --- (empty file) +++ pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/typecache.py Mon May 28 00:34:35 2007 @@ -0,0 +1,2 @@ +# this is automatically generated cache files for c types +platforms = {('', ('32bit', 'ELF'), 'Linux'): {'short': 16, 'int': 32, 'unsigned char': 8, 'long': 32, 'char': 8, 'unsigned short': 16, 'unsigned long': 32, 'long long': 64, 'unsigned long long': 64, 'size_t': 32, 'unsigned int': 32}} From fijal at codespeak.net Mon May 28 00:58:57 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 28 May 2007 00:58:57 +0200 (CEST) Subject: [pypy-svn] r43766 - in pypy/branch/kill-ctypes/pypy/rpython/lltypesystem: . test Message-ID: <20070527225857.728A080C0@code0.codespeak.net> Author: fijal Date: Mon May 28 00:58:55 2007 New Revision: 43766 Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rfficache.py Log: Add those type to be available Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rffi.py Mon May 28 00:58:55 2007 @@ -3,6 +3,7 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.annotation.model import lltype_to_annotation from pypy.rlib.objectmodel import Symbolic, CDefinedIntSymbolic +from pypy.rlib import rarithmetic class CConstant(Symbolic): """ A C-level constant, maybe #define, rendered directly. @@ -23,6 +24,24 @@ sources=tuple(sources), includes=tuple(includes)) +def setup(): + """ creates necessary c-level types + """ + from pypy.rpython.lltypesystem.rfficache import platform + for name, bits in platform.items(): + if name.startswith('unsigned'): + name = 'u' + name[9:] + signed = False + else: + signed = True + name = name.replace(' ', '') + llname = name.upper() + inttype = rarithmetic.build_int('r_' + name, signed, bits) + globals()['r_' + name] = inttype + globals()[llname] = lltype.build_number(llname, inttype) + +setup() + def CStruct(name, *fields, **kwds): """ A small helper to create external C structure, not the pypy one @@ -72,4 +91,3 @@ lltype.free(next[i], flavor='raw') i += 1 lltype.free(ref, flavor='raw') - Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rfficache.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rfficache.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/test/test_rfficache.py Mon May 28 00:58:55 2007 @@ -1,5 +1,6 @@ from pypy.rpython.lltypesystem.rfficache import * +from pypy.rpython.lltypesystem import rffi from pypy.tool.udir import udir def test_sizeof_c_type(): @@ -19,3 +20,12 @@ """)) assert get_type_sizes(tmpfile)['char'] == 8 assert get_type_sizes(tmpfile, platform_key='xxx', compiler_exe='xxx')['char'] == 4 + +def test_types_present(): + for name in TYPES: + if name.startswith('unsigned'): + name = 'u' + name[9:] + name = name.replace(' ', '') + assert hasattr(rffi, 'r_' + name) + assert hasattr(rffi, name.upper()) + From fijal at codespeak.net Mon May 28 01:38:46 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 28 May 2007 01:38:46 +0200 (CEST) Subject: [pypy-svn] r43767 - in pypy/branch/kill-ctypes/pypy: rlib rpython/module rpython/module/test translator/c/test Message-ID: <20070527233846.6861780C0@code0.codespeak.net> Author: fijal Date: Mon May 28 01:38:46 2007 New Revision: 43767 Modified: pypy/branch/kill-ctypes/pypy/rlib/ros.py pypy/branch/kill-ctypes/pypy/rpython/module/ll_os.py pypy/branch/kill-ctypes/pypy/rpython/module/test/test_ll_os.py pypy/branch/kill-ctypes/pypy/translator/c/test/test_extfunc.py Log: exarkun: present for you, RPython level implementation of utime, still doesn't work on windows nor provide utimes support, but kind of proves how well rffi works. Modified: pypy/branch/kill-ctypes/pypy/rlib/ros.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rlib/ros.py (original) +++ pypy/branch/kill-ctypes/pypy/rlib/ros.py Mon May 28 01:38:46 2007 @@ -54,3 +54,11 @@ def opendir(dirname): return DIR(dirname) opendir._annenforceargs_ = (str,) + +# probably we can get an annotation support for not having both implementations +# here, but let's leave it for now +def utime_null(path): + os.utime(path, None) + +def utime_tuple(path, tp): + os.utime(path, tp) Modified: pypy/branch/kill-ctypes/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/module/ll_os.py Mon May 28 01:38:46 2007 @@ -13,7 +13,7 @@ # 'suggested_primitive' flag is set to another function, if the conversion # and buffer preparation stuff is not useful. -import os, errno +import os from pypy.rpython.module.support import ll_strcpy, _ll_strfill from pypy.rpython.module.support import to_opaque_object, from_opaque_object from pypy.rlib import ros @@ -24,12 +24,8 @@ from pypy.annotation.model import SomeString, SomeInteger, s_ImpossibleValue, \ s_None from pypy.annotation.listdef import s_list_of_strings -import ctypes -import pypy.rpython.rctypes.implementation -from pypy.rpython.rctypes.tool.libc import libc from pypy.rpython.lltypesystem import rffi from pypy.rpython.lltypesystem import lltype -from pypy.rpython.rctypes.aerrno import geterrno if hasattr(os, 'execv'): @@ -46,7 +42,7 @@ os_execv(l_path, l_args) rffi.free_charpp(l_args) lltype.free(l_path, flavor='raw') - raise OSError(geterrno(), "execv failed") + raise OSError(rffi.c_errno, "execv failed") register_external(os.execv, [str, [str]], s_ImpossibleValue, llimpl= execv_lltypeimpl, export_name="ll_os.ll_os_execv") @@ -60,7 +56,7 @@ def dup_lltypeimpl(fd): newfd = os_dup(fd) if newfd == -1: - raise OSError(geterrno(), "dup failed") + raise OSError(rffi.c_errno, "dup failed") return newfd register_external(os.dup, [int], int, llimpl=dup_lltypeimpl, export_name="ll_os.ll_os_dup") @@ -74,10 +70,41 @@ def dup2_lltypeimpl(fd, newfd): error = os_dup2(fd, newfd) if error == -1: - raise OSError(geterrno(), "dup2 failed") + raise OSError(rffi.c_errno, "dup2 failed") register_external(os.dup2, [int, int], s_None, llimpl=dup2_lltypeimpl, export_name="ll_os.ll_os_dup2") + +UTIMEBUFP = rffi.CStruct('utimbuf', ('actime', rffi.SIZE_T), + ('modtime', rffi.SIZE_T)) + +# XXX sys/types.h is not portable at all +ros_utime = rffi.llexternal('utime', [rffi.CCHARP, UTIMEBUFP], lltype.Signed, + includes=['utime.h', 'sys/types.h']) + +def utime_null_lltypeimpl(path): + l_path = rffi.str2charp(path) + error = ros_utime(l_path, lltype.nullptr(UTIMEBUFP.TO)) + lltype.free(l_path, flavor='raw') + if error == -1: + raise OSError(rffi.c_errno, "utime_null failed") +register_external(ros.utime_null, [str], s_None, llimpl=utime_null_lltypeimpl) + +def utime_tuple_lltypeimpl(path, tp): + # XXX right now they're all ints, might change in future + # XXX does not use utimes, even when available + l_path = rffi.str2charp(path) + l_utimebuf = lltype.malloc(UTIMEBUFP.TO, flavor='raw') + l_utimebuf.c_actime, l_utimebuf.c_modtime = tp + error = ros_utime(l_path, l_utimebuf) + lltype.free(l_path, flavor='raw') + lltype.free(l_utimebuf, flavor='raw') + if error == -1: + raise OSError(rffi.c_errno, "utime_tuple failed") +register_external(ros.utime_tuple, [str, (int, int)], + llimpl=utime_tuple_lltypeimpl) + + class BaseOS: __metaclass__ = ClassMethods Modified: pypy/branch/kill-ctypes/pypy/rpython/module/test/test_ll_os.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/module/test/test_ll_os.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/module/test/test_ll_os.py Mon May 28 01:38:46 2007 @@ -1,7 +1,6 @@ import os from pypy.tool.udir import udir from pypy.tool.pytest.modcheck import skipimporterror -skipimporterror("ctypes") from pypy.rpython.lltypesystem.module.ll_os import Implementation as impl import sys Modified: pypy/branch/kill-ctypes/pypy/translator/c/test/test_extfunc.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/translator/c/test/test_extfunc.py (original) +++ pypy/branch/kill-ctypes/pypy/translator/c/test/test_extfunc.py Mon May 28 01:38:46 2007 @@ -749,3 +749,23 @@ func() assert open(filename).read() == "42" +def test_utime(): + # XXX utimes & float support + path = str(udir.ensure("test_utime.txt")) + from time import time, sleep + t0 = time() + sleep(1) + + def does_stuff(): + ros.utime_null(path) + + func = compile(does_stuff, []) + func() + assert os.stat(path).st_atime > t0 + + def utime_tuple(): + ros.utime_tuple(path, (int(t0), int(t0))) + + func = compile(utime_tuple, []) + func() + assert int(os.stat(path).st_atime) == int(t0) From santagada at codespeak.net Mon May 28 06:22:57 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Mon, 28 May 2007 06:22:57 +0200 (CEST) Subject: [pypy-svn] r43768 - in pypy/dist/pypy/lang/js: . test test/ecma/Array test/ecma/Boolean Message-ID: <20070528042257.335B380C6@code0.codespeak.net> Author: santagada Date: Mon May 28 06:22:55 2007 New Revision: 43768 Modified: pypy/dist/pypy/lang/js/interpreter.py pypy/dist/pypy/lang/js/jsgrammar.txt pypy/dist/pypy/lang/js/jsobj.py pypy/dist/pypy/lang/js/jsparser.py pypy/dist/pypy/lang/js/test/ecma/Array/15.4.1.2.js pypy/dist/pypy/lang/js/test/ecma/Array/15.4.4.3-1.js pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.3.1-2.js pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.3.1-3.js pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.3.1-4.js pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.3.1-5.js pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.3.js pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.4.2-1.js pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.4.2-2.js pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.4.2-3.js pypy/dist/pypy/lang/js/test/test_interp.py pypy/dist/pypy/lang/js/test/test_parser.py Log: fixed some semicolon problems on tests, implemented optional semicolons after a block and made Array behave more or less to the spec. Modified: pypy/dist/pypy/lang/js/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/js/interpreter.py (original) +++ pypy/dist/pypy/lang/js/interpreter.py Mon May 28 06:22:55 2007 @@ -106,7 +106,10 @@ return W_String('') def arrayjs(ctx, args, this): - return W_Array() + arr = W_Array() + for i in range(len(args)): + arr.Put(str(i), args[i]) + return arr def numberjs(ctx, args, this): @@ -151,6 +154,7 @@ w_math = W_Object(Class='Math') w_Global.Put('Math', w_math) w_math.Put('__proto__', w_ObjPrototype) + w_math.Put('prototype', w_ObjPrototype, dd=True, de=True, ro=True) w_math.Put('abs', W_Builtin(absjs, Class='function')) w_math.Put('floor', W_Builtin(floorjs, Class='function')) w_math.Put('pow', W_Builtin(powjs, Class='function')) @@ -158,11 +162,15 @@ w_math.Put('PI', W_Number(math.pi)) w_Global.Put('String', W_Builtin(stringjs, Class='String')) + + w_Array = W_Builtin(arrayjs, Class='Array') + w_Array.Put('__proto__', w_ObjPrototype) + w_Array.Put('prototype', w_ObjPrototype, dd=True, de=True, ro=True) #Global Properties w_Global.Put('Object', w_Object) w_Global.Put('Function', w_Function) - w_Global.Put('Array', W_Builtin(arrayjs, Class='Array')) + w_Global.Put('Array', w_Array) w_Global.Put('version', W_Builtin(versionjs)) #Number Modified: pypy/dist/pypy/lang/js/jsgrammar.txt ============================================================================== --- pypy/dist/pypy/lang/js/jsgrammar.txt (original) +++ pypy/dist/pypy/lang/js/jsgrammar.txt Mon May 28 06:22:55 2007 @@ -29,11 +29,28 @@ | ; +statementsemiopt : + | [";"]? + | + | [";"]? + | + | + | [";"]? + | [";"]? + | [";"]? + | + | + | [";"]? + | [";"]? + | + ; + + block : "{" >statementliststatementlist< - | statement + | statementsemiopt ; variablestatement : ["var"] variabledeclarationlist Modified: pypy/dist/pypy/lang/js/jsobj.py ============================================================================== --- pypy/dist/pypy/lang/js/jsobj.py (original) +++ pypy/dist/pypy/lang/js/jsobj.py Mon May 28 06:22:55 2007 @@ -503,7 +503,7 @@ return W_Reference(identifier) -def global_context(w_global=W_Object()): +def global_context(w_global): ctx = ExecutionContext([w_global], this = w_global, variable = w_global, @@ -525,6 +525,13 @@ jsproperty = Property('', w_Undefined)) return ctx +def empty_context(): + obj = W_Object() + ctx = ExecutionContext([obj], + this = obj, + variable = obj, + jsproperty = Property('', w_Undefined)) + return ctx class W_Reference(W_Root): """Reference Type""" Modified: pypy/dist/pypy/lang/js/jsparser.py ============================================================================== --- pypy/dist/pypy/lang/js/jsparser.py (original) +++ pypy/dist/pypy/lang/js/jsparser.py Mon May 28 06:22:55 2007 @@ -18,6 +18,7 @@ t = parsef(code) except ParseError: code += ';' + print code t = parsef(code) return ToAST().transform(t) Modified: pypy/dist/pypy/lang/js/test/ecma/Array/15.4.1.2.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/Array/15.4.1.2.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/Array/15.4.1.2.js Mon May 28 06:22:55 2007 @@ -148,9 +148,9 @@ if ( Math.abs( n ) == 0 || Math.abs( n ) == Number.POSITIVE_INFINITY) { return 0; } - n = sign * Math.floor( Math.abs(n) ) + n = sign * Math.floor( Math.abs(n) ); - n = n % Math.pow(2,32); + n = n % Math.pow(2,32); if ( n < 0 ){ n += Math.pow(2,32); Modified: pypy/dist/pypy/lang/js/test/ecma/Array/15.4.4.3-1.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/Array/15.4.4.3-1.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/Array/15.4.4.3-1.js Mon May 28 06:22:55 2007 @@ -116,7 +116,7 @@ eval("var TEST_ARRAY = new Array(true); TEST_ARRAY.join('\v')") ); -SEPARATOR = "\t" +SEPARATOR = "\t"; TEST_LENGTH = 100; TEST_STRING = ""; ARGUMENTS = ""; Modified: pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.3.1-2.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.3.1-2.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.3.1-2.js Mon May 28 06:22:55 2007 @@ -55,7 +55,7 @@ var SECTION = "15.6.3.1-2"; var VERSION = "ECMA_1"; startTest(); -var TITLE = "Boolean.prototype" +var TITLE = "Boolean.prototype"; writeHeaderToLog( SECTION + TITLE ); var array = new Array(); Modified: pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.3.1-3.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.3.1-3.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.3.1-3.js Mon May 28 06:22:55 2007 @@ -55,7 +55,7 @@ var SECTION = "15.6.3.1-3"; var VERSION = "ECMA_1"; startTest(); -var TITLE = "Boolean.prototype" +var TITLE = "Boolean.prototype"; writeHeaderToLog( SECTION + TITLE ); var array = new Array(); Modified: pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.3.1-4.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.3.1-4.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.3.1-4.js Mon May 28 06:22:55 2007 @@ -55,7 +55,7 @@ var SECTION = "15.6.3.1-4"; var VERSION = "ECMA_1"; startTest(); -var TITLE = "Boolean.prototype" +var TITLE = "Boolean.prototype"; writeHeaderToLog( SECTION + TITLE ); var BOOL_PROTO = Boolean.prototype; Modified: pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.3.1-5.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.3.1-5.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.3.1-5.js Mon May 28 06:22:55 2007 @@ -47,7 +47,7 @@ var VERSION = "ECMA_2"; startTest(); var SECTION = "15.6.3.1-5"; -var TITLE = "Boolean.prototype" +var TITLE = "Boolean.prototype"; writeHeaderToLog( SECTION + " " + TITLE ); Modified: pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.3.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.3.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.3.js Mon May 28 06:22:55 2007 @@ -53,7 +53,7 @@ var SECTION = "15.6.3"; var VERSION = "ECMA_2"; startTest(); -var TITLE = "Properties of the Boolean Constructor" +var TITLE = "Properties of the Boolean Constructor"; writeHeaderToLog( SECTION + TITLE ); Modified: pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.4.2-1.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.4.2-1.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.4.2-1.js Mon May 28 06:22:55 2007 @@ -55,7 +55,7 @@ var SECTION = "15.6.4.2-1"; var VERSION = "ECMA_1"; startTest(); -var TITLE = "Boolean.prototype.toString()" +var TITLE = "Boolean.prototype.toString()"; writeHeaderToLog( SECTION + TITLE ); Modified: pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.4.2-2.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.4.2-2.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.4.2-2.js Mon May 28 06:22:55 2007 @@ -53,7 +53,7 @@ var SECTION = "15.6.4.2-2"; var VERSION = "ECMA_1"; startTest(); -var TITLE = "Boolean.prototype.toString()" +var TITLE = "Boolean.prototype.toString()"; writeHeaderToLog( SECTION + TITLE ); new TestCase( SECTION, Modified: pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.4.2-3.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.4.2-3.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.4.2-3.js Mon May 28 06:22:55 2007 @@ -54,7 +54,7 @@ var SECTION = "15.6.4.2-3"; var VERSION = "ECMA_1"; startTest(); -var TITLE = "Boolean.prototype.toString()" +var TITLE = "Boolean.prototype.toString()"; writeHeaderToLog( SECTION + TITLE ); new TestCase( SECTION, "tostr=Boolean.prototype.toString; x=true; x.toString=tostr;x.toString()", "true", eval("tostr=Boolean.prototype.toString; x=true; x.toString=tostr;x.toString()") ); Modified: pypy/dist/pypy/lang/js/test/test_interp.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_interp.py (original) +++ pypy/dist/pypy/lang/js/test/test_interp.py Mon May 28 06:22:55 2007 @@ -1,9 +1,5 @@ -import sys -from StringIO import StringIO - import py - from pypy.lang.js import interpreter from pypy.lang.js.operations import AEC, Number, Position, Plus from pypy.lang.js.jsobj import W_Number, W_Object, \ Modified: pypy/dist/pypy/lang/js/test/test_parser.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_parser.py (original) +++ pypy/dist/pypy/lang/js/test/test_parser.py Mon May 28 06:22:55 2007 @@ -4,7 +4,7 @@ from pypy.rlib.parsing.ebnfparse import parse_ebnf, make_parse_function from pypy.rlib.parsing.parsing import ParseError, Rule from pypy.rlib.parsing.tree import RPythonVisitor -from pypy.lang.js.jsobj import global_context, ThrowException +from pypy.lang.js.jsobj import empty_context, ThrowException from pypy.lang.js.astbuilder import ASTBuilder from pypy import conftest import sys @@ -18,7 +18,7 @@ print e.nice_error_message(filename=str(GFILE),source=t) raise -parse = make_parse_function(regexs, rules, eof=True) +parse_function = make_parse_function(regexs, rules, eof=True) def setstartrule(rules, start): "takes the rule start and put it on the beginning of the rules" @@ -27,20 +27,17 @@ return newrules def get_defaultparse(): - global parse - if parse is None: - parse = make_parse_function(regexs, rules, eof=True) - return parse + return parse_function def parse_func(start=None): if start is not None: - parse = make_parse_function(regexs, setstartrule(rules, start), + parse_function = make_parse_function(regexs, setstartrule(rules, start), eof=True) else: - parse = get_defaultparse() + parse_function = get_defaultparse() def methodparse(self, text): - tree = parse(text) + tree = parse_function(text) if start is not None: assert tree.symbol == "hacked_first_symbol" tree = tree.children[0] @@ -288,7 +285,7 @@ def eval_expr(self, s): ast = self.to_ast(s) - return ast.eval(global_context()) + return ast.eval(empty_context()) def test_get_pos(self): from pypy.lang.js import operations @@ -327,4 +324,11 @@ assert w_num.ToNumber() == 4 w_str = self.eval_expr('"hello "+\'world\'') assert w_str.ToString() == 'hello world' + +from pypy.lang.js.jsparser import parse +def test_simplesemicolon(): + yield parse, 'x' + yield parse, 'if(1){x}' + yield parse, 'function x () {}' + yield parse,'if(1) {}' \ No newline at end of file From fijal at codespeak.net Mon May 28 12:35:27 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 28 May 2007 12:35:27 +0200 (CEST) Subject: [pypy-svn] r43773 - in pypy/branch/kill-ctypes/pypy/module/posix: . test Message-ID: <20070528103527.0C4D8809A@code0.codespeak.net> Author: fijal Date: Mon May 28 12:35:27 2007 New Revision: 43773 Modified: pypy/branch/kill-ctypes/pypy/module/posix/interp_posix.py pypy/branch/kill-ctypes/pypy/module/posix/test/test_posix2.py Log: Improve error reporting for execv and a bit hairy test. Modified: pypy/branch/kill-ctypes/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/kill-ctypes/pypy/module/posix/interp_posix.py Mon May 28 12:35:27 2007 @@ -5,9 +5,6 @@ import os -# Turned off for now. posix must support targets without ctypes -#from pypy.module.posix import ctypes_posix as _c - def wrap_oserror(space, e): assert isinstance(e, OSError) errno = e.errno @@ -433,7 +430,12 @@ def execv(space, command, w_args): try: os.execv(command, [space.str_w(i) for i in space.unpackiterable(w_args)]) - except OSError, e: + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise + msg = "execv() arg 2 must be an iterable of strings" + raise OperationError(space.w_TypeError, space.wrap(str(msg))) + except OSError, e: raise wrap_oserror(space, e) execv.unwrap_spec = [ObjSpace, str, W_Root] @@ -460,3 +462,6 @@ return space.newtuple([space.wrap(ob) for ob in result]) uname.unwrap_spec = [ObjSpace] +#def utime(space, path, w_tuple): +# +#utime.unwrap_spec = [ObjSpace, str, W_Root] Modified: pypy/branch/kill-ctypes/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/module/posix/test/test_posix2.py (original) +++ pypy/branch/kill-ctypes/pypy/module/posix/test/test_posix2.py Mon May 28 12:35:27 2007 @@ -141,6 +141,18 @@ os = self.posix raises(OSError, 'os.execv("saddsadsadsadsa", ["saddsadsasaddsa"])') + def test_execv_raising2(self): + os = self.posix + def t(n): + try: + os.execv("xxx", n) + except TypeError,t: + assert t.args[0] == "execv() arg 2 must be an iterable of strings" + else: + py.test.fail("didn't raise") + t(3) + t([3, "a"]) + def test_execve(self): skip("Not implemented") os = self.posix From fijal at codespeak.net Mon May 28 12:38:40 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 28 May 2007 12:38:40 +0200 (CEST) Subject: [pypy-svn] r43774 - pypy/branch/kill-ctypes/pypy/module/posix Message-ID: <20070528103840.3230B809A@code0.codespeak.net> Author: fijal Date: Mon May 28 12:38:39 2007 New Revision: 43774 Modified: pypy/branch/kill-ctypes/pypy/module/posix/interp_posix.py Log: Improve docstrings Modified: pypy/branch/kill-ctypes/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/kill-ctypes/pypy/module/posix/interp_posix.py Mon May 28 12:38:39 2007 @@ -428,6 +428,13 @@ geteuid.unwrap_spec = [ObjSpace] def execv(space, command, w_args): + """ execv(path, args) + +Execute an executable path with arguments, replacing current process. + + path: path of executable file + args: iterable of strings + """ try: os.execv(command, [space.str_w(i) for i in space.unpackiterable(w_args)]) except OperationError, e: @@ -440,6 +447,14 @@ execv.unwrap_spec = [ObjSpace, str, W_Root] def execve(space, command, w_args, w_env): + """ execve(path, args, env) + +Execute a path with arguments and environment, replacing current process. + + path: path of executable file + args: iterable of arguments + env: dictionary of strings mapping to strings + """ try: args = [space.str_w(i) for i in space.unpackiterable(w_args)] env = {} @@ -455,6 +470,10 @@ execve.unwrap_spec = [ObjSpace, str, W_Root, W_Root] def uname(space): + """ uname() -> (sysname, nodename, release, version, machine) + +Return a tuple identifying the current operating system. + """ try: result = _c.uname() except OSError, e: @@ -463,5 +482,10 @@ uname.unwrap_spec = [ObjSpace] #def utime(space, path, w_tuple): -# +# """ utime(path, (atime, mtime)) +#utime(path, None) +# +#Set the access and modified time of the file to the given values. If the +#second form is used, set the access and modified times to the current time. +# """ #utime.unwrap_spec = [ObjSpace, str, W_Root] From fijal at codespeak.net Mon May 28 12:53:24 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 28 May 2007 12:53:24 +0200 (CEST) Subject: [pypy-svn] r43775 - in pypy/branch/kill-ctypes/pypy/module/posix: . test Message-ID: <20070528105324.7FDAE8099@code0.codespeak.net> Author: fijal Date: Mon May 28 12:53:18 2007 New Revision: 43775 Modified: pypy/branch/kill-ctypes/pypy/module/posix/__init__.py pypy/branch/kill-ctypes/pypy/module/posix/interp_posix.py pypy/branch/kill-ctypes/pypy/module/posix/test/test_posix2.py Log: Add os.utime Modified: pypy/branch/kill-ctypes/pypy/module/posix/__init__.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/module/posix/__init__.py (original) +++ pypy/branch/kill-ctypes/pypy/module/posix/__init__.py Mon May 28 12:53:18 2007 @@ -51,6 +51,7 @@ '_exit' : 'interp_posix._exit', #'getuid' : 'interp_posix.getuid', #'geteuid' : 'interp_posix.geteuid', + 'utime' : 'interp_posix.utime', } if hasattr(os, 'ftruncate'): interpleveldefs['ftruncate'] = 'interp_posix.ftruncate' Modified: pypy/branch/kill-ctypes/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/kill-ctypes/pypy/module/posix/interp_posix.py Mon May 28 12:53:18 2007 @@ -481,11 +481,31 @@ return space.newtuple([space.wrap(ob) for ob in result]) uname.unwrap_spec = [ObjSpace] -#def utime(space, path, w_tuple): -# """ utime(path, (atime, mtime)) -#utime(path, None) -# -#Set the access and modified time of the file to the given values. If the -#second form is used, set the access and modified times to the current time. -# """ -#utime.unwrap_spec = [ObjSpace, str, W_Root] +def utime(space, path, w_tuple): + """ utime(path, (atime, mtime)) +utime(path, None) + +Set the access and modified time of the file to the given values. If the +second form is used, set the access and modified times to the current time. + """ + if space.is_w(w_tuple, space.w_None): + try: + ros.utime_null(path) + return + except OSError, e: + raise wrap_oserror(space, e) + try: + msg = "utime() arg 2 must be a tuple (atime, mtime) or None" + args_w = space.unpackiterable(w_tuple) + if len(args_w) != 2: + raise OperationError(space.w_TypeError, space.wrap(msg)) + actime = space.int_w(args_w[0]) + modtime = space.int_w(args_w[1]) + ros.utime_tuple(path, (actime, modtime)) + except OSError, e: + raise wrap_oserror(space, e) + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise + raise OperationError(space.w_TypeError, space.wrap(msg)) +utime.unwrap_spec = [ObjSpace, str, W_Root] Modified: pypy/branch/kill-ctypes/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/module/posix/test/test_posix2.py (original) +++ pypy/branch/kill-ctypes/pypy/module/posix/test/test_posix2.py Mon May 28 12:53:18 2007 @@ -171,6 +171,27 @@ stream = os.popen('echo 1') assert stream.read() == '1\n' + def test_utime(self): + os = self.posix + import os.path + # XXX utimes & float support + path = os.path.join(self.pdir, "test_utime.txt") + fh = open(path, "w") + fh.write("x") + fh.close() + from time import time, sleep + t0 = time() + sleep(1) + os.utime(path, None) + assert os.stat(path).st_atime > t0 + os.utime(path, (int(t0), int(t0))) + assert int(os.stat(path).st_atime) == int(t0) + + def test_utime_raises(self): + os = self.posix + raises(TypeError, "os.utime('xxx', 3)") + raises(OSError, "os.utime('somefilewhichihopewouldneverappearhere', None)") + class AppTestEnvironment(object): def setup_class(cls): cls.space = space From mwh at codespeak.net Mon May 28 12:57:15 2007 From: mwh at codespeak.net (mwh at codespeak.net) Date: Mon, 28 May 2007 12:57:15 +0200 (CEST) Subject: [pypy-svn] r43776 - pypy/branch/emptying-the-malloc-zoo Message-ID: <20070528105715.F08F18099@code0.codespeak.net> Author: mwh Date: Mon May 28 12:57:15 2007 New Revision: 43776 Added: pypy/branch/emptying-the-malloc-zoo/ - copied from r43775, pypy/dist/ Log: create a branch to try to clear up the malloc mess From mwh at codespeak.net Mon May 28 13:47:08 2007 From: mwh at codespeak.net (mwh at codespeak.net) Date: Mon, 28 May 2007 13:47:08 +0200 (CEST) Subject: [pypy-svn] r43778 - pypy/branch/emptying-the-malloc-zoo/pypy/rpython/lltypesystem Message-ID: <20070528114708.27D9E80A6@code0.codespeak.net> Author: mwh Date: Mon May 28 13:47:06 2007 New Revision: 43778 Modified: pypy/branch/emptying-the-malloc-zoo/pypy/rpython/lltypesystem/llmemory.py Log: small steps: rename all the various AddressOffset.raw_malloc methods to _raw_malloc Modified: pypy/branch/emptying-the-malloc-zoo/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/emptying-the-malloc-zoo/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/emptying-the-malloc-zoo/pypy/rpython/lltypesystem/llmemory.py Mon May 28 13:47:06 2007 @@ -22,8 +22,8 @@ return NotImplemented return CompositeOffset(self, other) - def raw_malloc(self, rest, zero): - raise NotImplementedError("raw_malloc(%r, %r)" % (self, rest)) + def _raw_malloc(self, rest, zero): + raise NotImplementedError("_raw_malloc(%r, %r)" % (self, rest)) def raw_memcopy(self, srcadr, dstsrc): raise NotImplementedError("raw_memcopy(%r)" % (self,)) @@ -67,7 +67,7 @@ else: raise TypeError('got %r, expected %r' % (A, self.TYPE)) - def raw_malloc(self, rest, zero): + def _raw_malloc(self, rest, zero): assert not rest if (isinstance(self.TYPE, lltype.ContainerType) and self.TYPE._gckind == 'gc'): @@ -118,12 +118,12 @@ else: return lltype.direct_fieldptr(struct, self.fldname) - def raw_malloc(self, rest, parenttype=None, zero=False): + def _raw_malloc(self, rest, parenttype=None, zero=False): if self.fldname != self.TYPE._arrayfld: # for the error msg - return AddressOffset.raw_malloc(self, rest, zero=zero) + return AddressOffset._raw_malloc(self, rest, zero=zero) assert rest - return rest[0].raw_malloc(rest[1:], parenttype=parenttype or self.TYPE, + return rest[0]._raw_malloc(rest[1:], parenttype=parenttype or self.TYPE, zero=zero) def raw_memcopy(self, srcadr, dstadr): @@ -171,8 +171,8 @@ ptr = item.ref(ptr) return ptr - def raw_malloc(self, rest, zero): - return self.offsets[0].raw_malloc(self.offsets[1:] + rest, zero=zero) + def _raw_malloc(self, rest, zero): + return self.offsets[0]._raw_malloc(self.offsets[1:] + rest, zero=zero) def raw_memcopy(self, srcadr, dstadr): for o in self.offsets[:-1]: @@ -198,7 +198,7 @@ else: return lltype.direct_arrayitems(arrayptr) - def raw_malloc(self, rest, parenttype=None, zero=False): + def _raw_malloc(self, rest, parenttype=None, zero=False): if rest: assert len(rest) == 1 assert isinstance(rest[0], ItemOffset) @@ -245,11 +245,11 @@ gcptr = self.gcheaderbuilder.object_from_header(headerptr) return gcptr - def raw_malloc(self, rest, zero): + def _raw_malloc(self, rest, zero): assert rest if isinstance(rest[0], GCHeaderAntiOffset): - return rest[1].raw_malloc(rest[2:], zero=zero) # just for fun - gcobjadr = rest[0].raw_malloc(rest[1:], zero=zero) + return rest[1]._raw_malloc(rest[2:], zero=zero) # just for fun + gcobjadr = rest[0]._raw_malloc(rest[1:], zero=zero) headerptr = self.gcheaderbuilder.new_header(gcobjadr.ptr) return cast_ptr_to_adr(headerptr) @@ -271,10 +271,10 @@ headerptr = self.gcheaderbuilder.header_of_object(gcptr) return headerptr - def raw_malloc(self, rest, zero): + def _raw_malloc(self, rest, zero): assert len(rest) >= 2 assert isinstance(rest[0], GCHeaderOffset) - return rest[1].raw_malloc(rest[2:], zero=zero) + return rest[1]._raw_malloc(rest[2:], zero=zero) # ____________________________________________________________ @@ -543,7 +543,7 @@ def raw_malloc(size): if not isinstance(size, AddressOffset): raise NotImplementedError(size) - return size.raw_malloc([], zero=False) + return size._raw_malloc([], zero=False) def raw_free(adr): # try to free the whole object if 'adr' is the address of the header @@ -568,7 +568,7 @@ if not isinstance(size, AddressOffset): raise NotImplementedError(size) assert lltype.typeOf(adr) == Address - zeroadr = size.raw_malloc([], zero=True) + zeroadr = size._raw_malloc([], zero=True) size.raw_memcopy(zeroadr, adr) def raw_memcopy(source, dest, size): @@ -615,7 +615,7 @@ self.unitsize = unitsize self.n = n - def raw_malloc(self, rest, zero=False): + def _raw_malloc(self, rest, zero=False): assert not rest arena = _arena(self, zero=zero) return arena.getitemaddr(0) From fijal at codespeak.net Mon May 28 15:01:41 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 28 May 2007 15:01:41 +0200 (CEST) Subject: [pypy-svn] r43786 - in pypy/branch/kill-ctypes/pypy: annotation rlib Message-ID: <20070528130141.BFB06809A@code0.codespeak.net> Author: fijal Date: Mon May 28 15:01:41 2007 New Revision: 43786 Modified: pypy/branch/kill-ctypes/pypy/annotation/bookkeeper.py pypy/branch/kill-ctypes/pypy/rlib/nonconst.py Log: Fix from the trunk Modified: pypy/branch/kill-ctypes/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/annotation/bookkeeper.py (original) +++ pypy/branch/kill-ctypes/pypy/annotation/bookkeeper.py Mon May 28 15:01:41 2007 @@ -344,7 +344,7 @@ else: listdef = ListDef(self, s_ImpossibleValue) for e in x: - listdef.generalize(self.annotation_from_example(e)) + listdef.generalize(self.immutablevalue(e, False)) result = SomeList(listdef) elif tp is dict or tp is r_dict: if need_const: @@ -385,8 +385,8 @@ dictdef.dictkey.update_rdict_annotations(s_eqfn, s_hashfn) for ek, ev in x.iteritems(): - dictdef.generalize_key(self.annotation_from_example(ek)) - dictdef.generalize_value(self.annotation_from_example(ev)) + dictdef.generalize_key(self.immutablevalue(ek, False)) + dictdef.generalize_value(self.immutablevalue(ev, False)) result = SomeDict(dictdef) elif ishashable(x) and x in BUILTIN_ANALYZERS: _module = getattr(x,"__module__","unknown") @@ -449,10 +449,6 @@ result.const = x return result - def annotation_from_example(self, x): - # XXX to kill at some point - return self.immutablevalue(x, False) - def getdesc(self, pyobj): # get the XxxDesc wrapper for the given Python object, which must be # one of: Modified: pypy/branch/kill-ctypes/pypy/rlib/nonconst.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rlib/nonconst.py (original) +++ pypy/branch/kill-ctypes/pypy/rlib/nonconst.py Mon May 28 15:01:41 2007 @@ -22,7 +22,7 @@ def compute_result_annotation(self, arg): if hasattr(arg, 'const'): - return self.bookkeeper.annotation_from_example(arg.const) + return self.bookkeeper.immutablevalue(arg.const, False) else: return arg From fijal at codespeak.net Mon May 28 15:02:11 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 28 May 2007 15:02:11 +0200 (CEST) Subject: [pypy-svn] r43787 - pypy/branch/kill-ctypes/pypy/rpython/lltypesystem Message-ID: <20070528130211.EAA12809A@code0.codespeak.net> Author: fijal Date: Mon May 28 15:02:11 2007 New Revision: 43787 Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rfficache.py pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/typecache.py Log: Add mode_t for open Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rfficache.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rfficache.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/rfficache.py Mon May 28 15:02:11 2007 @@ -4,12 +4,14 @@ """ import py +import os from pypy.translator.tool.cbuild import build_executable from subprocess import PIPE, Popen from pypy.tool.udir import udir def sizeof_c_type(c_typename, includes={}, compiler_exe=None): includes['stdio.h'] = True + includes['sys' + os.path.sep + 'types.h'] = True include_string = "\n".join(["#include <%s>" % i for i in includes.keys()]) c_source = py.code.Source(''' // includes @@ -43,9 +45,7 @@ for _name in 'char short int long'.split(): for name in (_name, 'unsigned ' + _name): TYPES.append(name) -TYPES.append('long long') -TYPES.append('unsigned long long') -TYPES.append('size_t') +TYPES += ['long long', 'unsigned long long', 'size_t', 'mode_t'] def get_type_sizes(filename, platform_key=machine_key(), types=TYPES, compiler_exe=None): @@ -55,7 +55,12 @@ except (ImportError, py.error.ENOENT): platforms = {} try: - return platforms[platform_key] + result = platforms[platform_key] + if sorted(result.keys()) != sorted(TYPES): + # invalidate file + platforms = {} + raise KeyError + return result except KeyError: value = dict([(i, sizeof_c_type(i, compiler_exe=compiler_exe)) for i in types]) Modified: pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/typecache.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/typecache.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/lltypesystem/typecache.py Mon May 28 15:02:11 2007 @@ -1,2 +1,2 @@ # this is automatically generated cache files for c types -platforms = {('', ('32bit', 'ELF'), 'Linux'): {'short': 16, 'int': 32, 'unsigned char': 8, 'long': 32, 'char': 8, 'unsigned short': 16, 'unsigned long': 32, 'long long': 64, 'unsigned long long': 64, 'size_t': 32, 'unsigned int': 32}} +platforms = {('', ('32bit', 'ELF'), 'Linux'): {'short': 16, 'int': 32, 'unsigned char': 8, 'long': 32, 'char': 8, 'unsigned short': 16, 'unsigned long': 32, 'long long': 64, 'mode_t': 32, 'unsigned long long': 64, 'size_t': 32, 'unsigned int': 32}} From fijal at codespeak.net Mon May 28 15:06:11 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 28 May 2007 15:06:11 +0200 (CEST) Subject: [pypy-svn] r43788 - pypy/branch/kill-ctypes/pypy/rpython Message-ID: <20070528130611.57275809A@code0.codespeak.net> Author: fijal Date: Mon May 28 15:06:10 2007 New Revision: 43788 Modified: pypy/branch/kill-ctypes/pypy/rpython/llinterp.py Log: Add a non-working method for tests to start complain. My guess is that this kind of operation should never be seen by llinterp, but I cannot judge now. Modified: pypy/branch/kill-ctypes/pypy/rpython/llinterp.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/llinterp.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/llinterp.py Mon May 28 15:06:10 2007 @@ -641,6 +641,12 @@ assert self.llinterpreter.gc is None return self.heap.malloc(obj, size, zero=True) + def op_flavored_malloc_varsize(self, flavor, obj, size): + XXX # probably should be never seen, and if so, should + # be tested + assert isinstance(flavor, str) + return self.heap.malloc(obj, flavor=flavor) + def op_flavored_malloc(self, flavor, obj): assert isinstance(flavor, str) if flavor == "stack": From fijal at codespeak.net Mon May 28 15:06:51 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 28 May 2007 15:06:51 +0200 (CEST) Subject: [pypy-svn] r43789 - in pypy/branch/kill-ctypes/pypy: rpython rpython/module rpython/module/test translator/c Message-ID: <20070528130651.28D49809A@code0.codespeak.net> Author: fijal Date: Mon May 28 15:06:50 2007 New Revision: 43789 Modified: pypy/branch/kill-ctypes/pypy/rpython/extfunctable.py pypy/branch/kill-ctypes/pypy/rpython/module/ll_os.py pypy/branch/kill-ctypes/pypy/rpython/module/test/test_ll_os.py pypy/branch/kill-ctypes/pypy/translator/c/extfunc.py Log: Untangling the mess part one - os.open Modified: pypy/branch/kill-ctypes/pypy/rpython/extfunctable.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/extfunctable.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/extfunctable.py Mon May 28 15:06:50 2007 @@ -179,7 +179,6 @@ # external function declarations posix = __import__(os.name) -declare(os.open , int , 'll_os/open') declare(os.read , str , 'll_os/read') declare(os.write , posannotation , 'll_os/write') declare(os.close , noneannotation, 'll_os/close') Modified: pypy/branch/kill-ctypes/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/module/ll_os.py Mon May 28 15:06:50 2007 @@ -104,14 +104,22 @@ register_external(ros.utime_tuple, [str, (int, int)], llimpl=utime_tuple_lltypeimpl) +os_open = rffi.llexternal('open', [rffi.CCHARP, lltype.Signed, rffi.MODE_T], + lltype.Signed) + +def os_open_lltypeimpl(path, flags, mode): + l_path = rffi.str2charp(path) + result = os_open(l_path, flags, mode) + lltype.free(l_path, flavor='raw') + if result == -1: + raise OSError(rffi.c_errno, "os_open failed") + return result +register_external(os.open, [str, int, int], int, export_name="open", + llimpl=os_open_lltypeimpl) class BaseOS: __metaclass__ = ClassMethods - def ll_os_open(cls, fname, flag, mode): - return os.open(cls.from_rstr(fname), flag, mode) - ll_os_open.suggested_primitive = True - def ll_os_write(cls, fd, astring): return os.write(fd, cls.from_rstr(astring)) ll_os_write.suggested_primitive = True Modified: pypy/branch/kill-ctypes/pypy/rpython/module/test/test_ll_os.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/module/test/test_ll_os.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/module/test/test_ll_os.py Mon May 28 15:06:50 2007 @@ -16,22 +16,6 @@ assert os.access(filename, mode) == impl.ll_os_access(rsfilename, mode) -def test_open_read_write_close(): - filename = str(udir.join('test_open_read_write_close.txt')) - rsfilename = impl.to_rstr(filename) - - fd = impl.ll_os_open(rsfilename, os.O_WRONLY | os.O_CREAT, 0777) - count = impl.ll_os_write(fd, impl.to_rstr("hello world\n")) - assert count == len("hello world\n") - impl.ll_os_close(fd) - - fd = impl.ll_os_open(rsfilename, os.O_RDONLY, 0777) - data = impl.ll_os_read(fd, 500) - assert impl.from_rstr(data) == "hello world\n" - impl.ll_os_close(fd) - - os.unlink(filename) - def test_getcwd(): data = impl.ll_os_getcwd() assert impl.from_rstr(data) == os.getcwd() Modified: pypy/branch/kill-ctypes/pypy/translator/c/extfunc.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/translator/c/extfunc.py (original) +++ pypy/branch/kill-ctypes/pypy/translator/c/extfunc.py Mon May 28 15:06:50 2007 @@ -23,7 +23,6 @@ # references to functions, so we cannot insert classmethods here. EXTERNALS = { - impl.ll_os_open.im_func: 'LL_os_open', impl.ll_read_into: 'LL_read_into', # it's a staticmethod impl.ll_os_write.im_func: 'LL_os_write', impl.ll_os_close.im_func: 'LL_os_close', From fijal at codespeak.net Mon May 28 15:07:54 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 28 May 2007 15:07:54 +0200 (CEST) Subject: [pypy-svn] r43790 - pypy/branch/kill-ctypes/pypy/translator/c/src Message-ID: <20070528130754.2FBBD809A@code0.codespeak.net> Author: fijal Date: Mon May 28 15:07:43 2007 New Revision: 43790 Modified: pypy/branch/kill-ctypes/pypy/translator/c/src/ll_os.h Log: This might go as well now Modified: pypy/branch/kill-ctypes/pypy/translator/c/src/ll_os.h ============================================================================== --- pypy/branch/kill-ctypes/pypy/translator/c/src/ll_os.h (original) +++ pypy/branch/kill-ctypes/pypy/translator/c/src/ll_os.h Mon May 28 15:07:43 2007 @@ -51,7 +51,6 @@ /* prototypes */ -int LL_os_open(RPyString *filename, int flag, int mode); long LL_read_into(int fd, RPyString *buffer); long LL_os_write(int fd, RPyString *buffer); void LL_os_close(int fd); @@ -103,15 +102,6 @@ #include "ll_osdefs.h" -int LL_os_open(RPyString *filename, int flag, int mode) -{ - /* XXX unicode_file_names */ - int fd = open(RPyString_AsString(filename), flag, mode); - if (fd < 0) - RPYTHON_RAISE_OSERROR(errno); - return fd; -} - long LL_read_into(int fd, RPyString *buffer) { long n = read(fd, RPyString_AsString(buffer), RPyString_Size(buffer)); From fijal at codespeak.net Mon May 28 15:21:38 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 28 May 2007 15:21:38 +0200 (CEST) Subject: [pypy-svn] r43792 - pypy/branch/kill-ctypes/pypy/rpython/module Message-ID: <20070528132138.87A32809C@code0.codespeak.net> Author: fijal Date: Mon May 28 15:21:38 2007 New Revision: 43792 Modified: pypy/branch/kill-ctypes/pypy/rpython/module/ll_os.py Log: hum, I think this might go away as well Modified: pypy/branch/kill-ctypes/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/module/ll_os.py Mon May 28 15:21:38 2007 @@ -139,14 +139,6 @@ os.close(fd) ll_os_close.suggested_primitive = True - def ll_os_dup(cls, fd): - return os.dup(fd) - ll_os_dup.suggested_primitive = True - - def ll_os_dup2(cls, old_fd, new_fd): - os.dup2(old_fd, new_fd) - ll_os_dup2.suggested_primitive = True - def ll_os_access(cls, path, mode): return os.access(cls.from_rstr(path), mode) ll_os_access.suggested_primitive = True From fijal at codespeak.net Mon May 28 15:45:37 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 28 May 2007 15:45:37 +0200 (CEST) Subject: [pypy-svn] r43795 - pypy/branch/kill-ctypes/pypy/rpython/module Message-ID: <20070528134537.CAF6A807E@code0.codespeak.net> Author: fijal Date: Mon May 28 15:45:37 2007 New Revision: 43795 Modified: pypy/branch/kill-ctypes/pypy/rpython/module/ll_os.py Log: They're not so dummy any more Modified: pypy/branch/kill-ctypes/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/module/ll_os.py Mon May 28 15:45:37 2007 @@ -1,5 +1,5 @@ """ -Dummy low-level implementations for the external functions of the 'os' module. +Low-level implementations for the external functions of the 'os' module. """ # Idea: each ll_os_xxx() function calls back the os.xxx() function that it From santagada at codespeak.net Mon May 28 16:06:27 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Mon, 28 May 2007 16:06:27 +0200 (CEST) Subject: [pypy-svn] r43796 - in pypy/dist/pypy/lang/js: . test test/ecma Message-ID: <20070528140627.8CB2A80A6@code0.codespeak.net> Author: santagada Date: Mon May 28 16:06:27 2007 New Revision: 43796 Modified: pypy/dist/pypy/lang/js/interpreter.py pypy/dist/pypy/lang/js/operations.py pypy/dist/pypy/lang/js/test/ecma/conftest.py pypy/dist/pypy/lang/js/test/ecma/shell.js pypy/dist/pypy/lang/js/test/test_interp.py pypy/dist/pypy/lang/js/test/test_parser.py Log: put position object on all ops, eval should raise an JS Exception and turned off debug, also some cosmetic changes. Modified: pypy/dist/pypy/lang/js/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/js/interpreter.py (original) +++ pypy/dist/pypy/lang/js/interpreter.py Mon May 28 16:06:27 2007 @@ -1,8 +1,9 @@ import math -from pypy.lang.js.jsparser import parse +from pypy.lang.js.jsparser import parse, ParseError from pypy.lang.js.astbuilder import ASTBuilder from pypy.lang.js.operations import * +from pypy.lang.js.jsobj import ThrowException from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.streamio import open_file_as_stream @@ -29,7 +30,12 @@ return args[0] else: code = W_String('') - return load_source(code.ToString()).execute(ctx) + try: + node = load_source(code.ToString()) + except ParseError, e: + raise ThrowException(W_String('SintaxError: '+str(e))) + + return node.execute(ctx) def functionjs(ctx, args, this): tam = len(args) Modified: pypy/dist/pypy/lang/js/operations.py ============================================================================== --- pypy/dist/pypy/lang/js/operations.py (original) +++ pypy/dist/pypy/lang/js/operations.py Mon May 28 16:06:27 2007 @@ -291,6 +291,7 @@ class If(Statement): def __init__(self, pos, condition, thenpart, elsepart=astundef): + self.pos = pos self.condition = condition self.thenPart = thenpart self.elsePart = elsepart @@ -303,8 +304,6 @@ return self.elsePart.execute(ctx) class Group(UnaryOp): - opcode = 'GROUP' - def eval(self, ctx): return self.expr.eval(ctx) @@ -395,24 +394,18 @@ ############################################################################## class Ursh(BinaryComparisonOp): - opcode = 'URSH' - def decision(self, ctx, op1, op2): a = op1.ToUInt32() b = op2.ToUInt32() return W_Number(a >> (b & 0x1F)) class Rsh(BinaryComparisonOp): - opcode = 'RSH' - def decision(self, ctx, op1, op2): a = op1.ToInt32() b = op2.ToUInt32() return W_Number(a >> intmask(b & 0x1F)) class Lsh(BinaryComparisonOp): - opcode = 'LSH' - def decision(self, ctx, op1, op2): a = op1.ToInt32() b = op2.ToUInt32() @@ -483,14 +476,10 @@ return r class Eq(BinaryComparisonOp): - opcode = 'EQ' - def decision(self, ctx, op1, op2): return W_Boolean(AEC(ctx, op1, op2)) class Ne(BinaryComparisonOp): - opcode = 'NE' - def decision(self, ctx, op1, op2): return W_Boolean(not AEC(ctx, op1, op2)) @@ -529,14 +518,10 @@ return x == y class StrictEq(BinaryComparisonOp): - opcode = 'STRICT_EQ' - def decision(self, ctx, op1, op2): return W_Boolean(SEC(op1, op2)) class StrictNe(BinaryComparisonOp): - opcode = 'STRICT_NE' - def decision(self, ctx, op1, op2): return W_Boolean(not SEC(op1, op2)) @@ -709,6 +694,7 @@ class Number(Expression): def __init__(self, pos, num): + self.pos = pos assert isinstance(num, float) self.num = num @@ -766,7 +752,8 @@ """ SourceElements nodes are found on each function declaration and in global code """ - def __init__(self, pos, var_decl, func_decl, nodes): + def __init__(self, pos, var_decl, func_decl, nodes): + self.pos = pos self.var_decl = var_decl self.func_decl = func_decl self.nodes = nodes @@ -1026,6 +1013,7 @@ class Boolean(Expression): def __init__(self, pos, boolval): + self.pos = pos self.bool = boolval def eval(self, ctx): Modified: pypy/dist/pypy/lang/js/test/ecma/conftest.py ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/conftest.py (original) +++ pypy/dist/pypy/lang/js/test/ecma/conftest.py Mon May 28 16:06:27 2007 @@ -55,8 +55,8 @@ t = load_file(str(self.fspath)) try: t.execute(self.interp.global_context) - except ParseError: - raise Failed(msg="Syntax Error",excinfo=py.code.ExceptionInfo()) + except ParseError, e: + raise Failed(msg=e.nice_error_message(filename=str(self.fspath)), excinfo=None) except JsBaseExcept: raise Failed(msg="Javascript Error", excinfo=py.code.ExceptionInfo()) except: Modified: pypy/dist/pypy/lang/js/test/ecma/shell.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/shell.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/shell.js Mon May 28 16:06:27 2007 @@ -50,7 +50,7 @@ var PASSED = " PASSED!"; var FAILED = " FAILED! expected: "; -var DEBUG = true; +var DEBUG = false; var DESCRIPTION; var EXPECTED; @@ -390,4 +390,5 @@ this.setOption('strict', this.orig.strict); this.setOption('werror', this.orig.werror); } -*/ \ No newline at end of file +*/ + Modified: pypy/dist/pypy/lang/js/test/test_interp.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_interp.py (original) +++ pypy/dist/pypy/lang/js/test/test_interp.py Mon May 28 16:06:27 2007 @@ -44,6 +44,11 @@ assert code_val.ToNumber() == value else: assert code_val.ToString() == value + +def asserte(code, value): + jsint = interpreter.Interpreter() + py.test.raises(value, 'jsint.run(interpreter.load_source(code))') + def test_interp_parse(): yield assertv, "1+1;", 2 @@ -296,11 +301,12 @@ """, ["0","1","2","3"]) def test_eval(): - assertp(""" + yield assertp, """ var x = 2; eval('x=x+1; print(x); z=2;'); print(z); - """, ["3","2"]) + """, ["3","2"] + yield asserte, "eval('var do =true;');", ThrowException def test_arrayobject(): assertv("""var x = new Array(); Modified: pypy/dist/pypy/lang/js/test/test_parser.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_parser.py (original) +++ pypy/dist/pypy/lang/js/test/test_parser.py Mon May 28 16:06:27 2007 @@ -331,4 +331,4 @@ yield parse, 'x' yield parse, 'if(1){x}' yield parse, 'function x () {}' - yield parse,'if(1) {}' \ No newline at end of file + yield parse,'if(1) {}' From fijal at codespeak.net Mon May 28 16:07:33 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 28 May 2007 16:07:33 +0200 (CEST) Subject: [pypy-svn] r43797 - pypy/branch/kill-ctypes/pypy/doc Message-ID: <20070528140733.97C65809C@code0.codespeak.net> Author: fijal Date: Mon May 28 16:07:33 2007 New Revision: 43797 Added: pypy/branch/kill-ctypes/pypy/doc/rffi.txt Log: Add some document about rffi, links are broken because it's on a branch. Added: pypy/branch/kill-ctypes/pypy/doc/rffi.txt ============================================================================== --- (empty file) +++ pypy/branch/kill-ctypes/pypy/doc/rffi.txt Mon May 28 16:07:33 2007 @@ -0,0 +1,60 @@ + +Foreign Function Interface for RPython +======================================= + +Purpose +------- + +This document describes an FFI for RPython language, concentrating +only (right now) on low-level backends like C or LLVM. It describes +how to declare and call low-level (C) functions from RPython level. + +Declaring low-level external function +------------------------------------- + +Declaring external C function in RPython is easy, but one need to +remember that low level functions eats low level types (like +lltype.Signed or lltype.Array) and memory management must be done +by hand. To declare a function, we write:: + + from pypy.rpython.lltypesystem import rffi + + external_function = rffi.llexternal(name, args, result, includes=[], sources=[]) + +where: + +* name - a C-level name of a function (how it would be rendered) +* args - low level types of args +* result - low level type of a result +* includes - additional C-level includes +* sources - additional C-level sources, needed to be compiled into + executable + +Types +------ + +In rffi_ there are various declared types for C-structures, like CCHARP +(char*), SIZE_T (size_t) and others. refer to file for details. +Instances of non-primitive types must be alloced by hand, with call +to lltype.malloc, and freed by lltype.free both with keyword argument +flavor='raw'. There are several helpers like string -> char* +converter, refer to the source for details. + +.. _rffi: http://codespeak.net/svn/pypy/dist/pypy/rpython/lltypesystem/rffi.py + +Registering function as external +--------------------------------- + +Once we provided low-level implementation of an external function, +would be nice to wrap call to some library function (like os.open) +with such a call. For this, there is a `register_external` routine, +located in `extfunc.py`_, which provides nice API for declaring such a +functions, passing llimpl as an argument and eventually llfakeimpl +as a fake low-level implementation for tests performed by an llinterp. + +.. _`extfunc.py`: http://codespeak.net/svn/pypy/dist/pypy/rpython/extfunc.py + +OO backends +----------- + +XXX to be written From fijal at codespeak.net Mon May 28 16:09:19 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 28 May 2007 16:09:19 +0200 (CEST) Subject: [pypy-svn] r43798 - pypy/branch/kill-ctypes/pypy/rpython/module Message-ID: <20070528140919.2F5D8809C@code0.codespeak.net> Author: fijal Date: Mon May 28 16:09:18 2007 New Revision: 43798 Modified: pypy/branch/kill-ctypes/pypy/rpython/module/ll_os.py Log: Improve a dicstring Modified: pypy/branch/kill-ctypes/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/module/ll_os.py Mon May 28 16:09:18 2007 @@ -2,6 +2,12 @@ Low-level implementations for the external functions of the 'os' module. """ +# actual idea might be found in doc/rffi.txt + +# ------------------------------------------ +# WARNING! old vision, don't use it WARNING! +# ------------------------------------------ + # Idea: each ll_os_xxx() function calls back the os.xxx() function that it # is supposed to implement, either directly or indirectly if there is some # argument decoding and buffering preparation that can be done here. From mwh at codespeak.net Mon May 28 17:25:01 2007 From: mwh at codespeak.net (mwh at codespeak.net) Date: Mon, 28 May 2007 17:25:01 +0200 (CEST) Subject: [pypy-svn] r43804 - in pypy/branch/emptying-the-malloc-zoo/pypy: rpython/lltypesystem rpython/memory rpython/memory/gctransform translator/c/src Message-ID: <20070528152501.E0F1D8099@code0.codespeak.net> Author: mwh Date: Mon May 28 17:25:01 2007 New Revision: 43804 Modified: pypy/branch/emptying-the-malloc-zoo/pypy/rpython/lltypesystem/lloperation.py pypy/branch/emptying-the-malloc-zoo/pypy/rpython/memory/gc.py pypy/branch/emptying-the-malloc-zoo/pypy/rpython/memory/gctransform/framework.py pypy/branch/emptying-the-malloc-zoo/pypy/rpython/memory/gctransform/refcounting.py pypy/branch/emptying-the-malloc-zoo/pypy/translator/c/src/mem.h Log: convert raw_malloc to a non-raising operation Modified: pypy/branch/emptying-the-malloc-zoo/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/emptying-the-malloc-zoo/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/emptying-the-malloc-zoo/pypy/rpython/lltypesystem/lloperation.py Mon May 28 17:25:01 2007 @@ -338,7 +338,7 @@ # __________ address operations __________ - 'raw_malloc': LLOp(canraise=(MemoryError,)), + 'raw_malloc': LLOp(), 'raw_malloc_usage': LLOp(sideeffects=False), 'raw_free': LLOp(), 'raw_memclear': LLOp(), Modified: pypy/branch/emptying-the-malloc-zoo/pypy/rpython/memory/gc.py ============================================================================== --- pypy/branch/emptying-the-malloc-zoo/pypy/rpython/memory/gc.py (original) +++ pypy/branch/emptying-the-malloc-zoo/pypy/rpython/memory/gc.py Mon May 28 17:25:01 2007 @@ -4,7 +4,7 @@ from pypy.rpython.memory.gcheader import GCHeaderBuilder from pypy.rpython.memory import lltypesimulation from pypy.rpython.lltypesystem import lltype, llmemory -from pypy.rlib.objectmodel import free_non_gc_object +from pypy.rlib.objectmodel import free_non_gc_object, debug_assert from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.rarithmetic import ovfcheck @@ -97,7 +97,10 @@ size = self.fixed_size(typeid) if self.is_varsize(typeid): size += length * self.varsize_item_sizes(typeid) - return raw_malloc(size) + result = raw_malloc(size) + if not result: + raise memoryError + return result def collect(self): self.get_roots() #this is there so that the annotator thinks get_roots is a function @@ -182,6 +185,8 @@ except OverflowError: raise memoryError result = raw_malloc(tot_size) + if not result: + raise memoryError hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR) hdr.typeid = typeid << 1 if has_finalizer: @@ -208,6 +213,8 @@ except OverflowError: raise memoryError result = raw_malloc(tot_size) + if not result: + raise memoryError raw_memclear(result, tot_size) hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR) hdr.typeid = typeid << 1 @@ -238,6 +245,8 @@ except OverflowError: raise memoryError result = raw_malloc(tot_size) + if not result: + raise memoryError (result + size_gc_header + offset_to_length).signed[0] = length hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR) hdr.typeid = typeid << 1 @@ -270,6 +279,8 @@ except OverflowError: raise memoryError result = raw_malloc(tot_size) + if not result: + raise memoryError raw_memclear(result, tot_size) (result + size_gc_header + offset_to_length).signed[0] = length hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR) @@ -942,8 +953,10 @@ def setup(self): self.tospace = raw_malloc(self.space_size) + debug_assert(bool(self.tospace), "couldn't allocate tospace") self.top_of_space = self.tospace + self.space_size self.fromspace = raw_malloc(self.space_size) + debug_assert(bool(self.fromspace), "couldn't allocate fromspace") self.free = self.tospace def free_memory(self): @@ -1125,6 +1138,8 @@ size_gc_header = self.size_gc_header() result = raw_malloc(size + size_gc_header) ## print "mallocing %s, size %s at %s" % (typeid, size, result) + if not result: + raise memoryError result.signed[0] = 0 # refcount result.signed[1] = typeid return result + size_gc_header Modified: pypy/branch/emptying-the-malloc-zoo/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/emptying-the-malloc-zoo/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/emptying-the-malloc-zoo/pypy/rpython/memory/gctransform/framework.py Mon May 28 17:25:01 2007 @@ -6,6 +6,7 @@ from pypy.rpython.memory import gc, lladdress from pypy.rpython.memory.gcheader import GCHeaderBuilder from pypy.rlib.rarithmetic import ovfcheck +from pypy.rlib.objectmodel import debug_assert from pypy.translator.backendopt import graphanalyze from pypy.annotation import model as annmodel from pypy.rpython import annlowlevel @@ -251,6 +252,7 @@ _alloc_flavor_ = 'raw' def setup_root_stack(): stackbase = lladdress.raw_malloc(rootstacksize) + debug_assert(bool(stackbase), "could not allocate root stack") lladdress.raw_memclear(stackbase, rootstacksize) gcdata.root_stack_top = stackbase gcdata.root_stack_base = stackbase Modified: pypy/branch/emptying-the-malloc-zoo/pypy/rpython/memory/gctransform/refcounting.py ============================================================================== --- pypy/branch/emptying-the-malloc-zoo/pypy/rpython/memory/gctransform/refcounting.py (original) +++ pypy/branch/emptying-the-malloc-zoo/pypy/rpython/memory/gctransform/refcounting.py Mon May 28 17:25:01 2007 @@ -45,7 +45,9 @@ # create incref, etc graph + memoryError = MemoryError() HDRPTR = lltype.Ptr(self.HDR) + def ll_incref(adr): if adr: gcheader = llmemory.cast_adr_to_ptr(adr - gc_header_offset, HDRPTR) @@ -71,6 +73,8 @@ def ll_malloc_fixedsize(size): size = gc_header_offset + size result = lladdress.raw_malloc(size) + if not result: + raise memoryError lladdress.raw_memclear(result, size) result += gc_header_offset return result @@ -80,8 +84,10 @@ varsize = ovfcheck(itemsize * length) tot_size = ovfcheck(fixsize + varsize) except OverflowError: - raise MemoryError + raise memoryError result = lladdress.raw_malloc(tot_size) + if not result: + raise memoryError lladdress.raw_memclear(result, tot_size) result += gc_header_offset return result Modified: pypy/branch/emptying-the-malloc-zoo/pypy/translator/c/src/mem.h ============================================================================== --- pypy/branch/emptying-the-malloc-zoo/pypy/translator/c/src/mem.h (original) +++ pypy/branch/emptying-the-malloc-zoo/pypy/translator/c/src/mem.h Mon May 28 17:25:01 2007 @@ -8,11 +8,7 @@ #define OP_RAW_MALLOC(size, r, restype) { \ r = (restype) PyObject_Malloc(size); \ - if (r == NULL) { \ - FAIL_EXCEPTION(PyExc_MemoryError, \ - "out of memory"); \ - } \ - else { \ + if (r != NULL) { \ memset((void*)r, 0, size); \ COUNT_MALLOC; \ } \ @@ -22,13 +18,9 @@ #define OP_RAW_MALLOC(size, r, restype) { \ r = (restype) PyObject_Malloc(size); \ - if (r == NULL) { \ - FAIL_EXCEPTION(PyExc_MemoryError, \ - "out of memory"); \ - } \ - else { \ + if (r != NULL) { \ COUNT_MALLOC; \ - } \ + } \ } #endif From fijal at codespeak.net Mon May 28 18:39:18 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 28 May 2007 18:39:18 +0200 (CEST) Subject: [pypy-svn] r43813 - pypy/dist/pypy/translator/c/test Message-ID: <20070528163918.B265B8097@code0.codespeak.net> Author: fijal Date: Mon May 28 18:39:13 2007 New Revision: 43813 Modified: pypy/dist/pypy/translator/c/test/test_genc.py Log: Failing test. seems that print is doing one bit too much allocation Modified: pypy/dist/pypy/translator/c/test/test_genc.py ============================================================================== --- pypy/dist/pypy/translator/c/test/test_genc.py (original) +++ pypy/dist/pypy/translator/c/test/test_genc.py Mon May 28 18:39:13 2007 @@ -365,3 +365,11 @@ s = t.buildannotator().build_types(f, []) rtyper = t.buildrtyper(type_system="lltype") rtyper.specialize() + +def test_x(): + py.test.skip("Failing test. Seems that print allocs one bit too much") + def f(): + print "xxx" + + fn = compile(f, []) + fn() From fijal at codespeak.net Mon May 28 20:07:16 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 28 May 2007 20:07:16 +0200 (CEST) Subject: [pypy-svn] r43819 - pypy/branch/kill-ctypes/pypy/rpython/module Message-ID: <20070528180716.8558B80A6@code0.codespeak.net> Author: fijal Date: Mon May 28 20:07:15 2007 New Revision: 43819 Modified: pypy/branch/kill-ctypes/pypy/rpython/module/ll_os.py Log: Improve a bit names Modified: pypy/branch/kill-ctypes/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/module/ll_os.py Mon May 28 20:07:15 2007 @@ -29,7 +29,6 @@ from pypy.rpython.extfunc import ExtFuncEntry, register_external from pypy.annotation.model import SomeString, SomeInteger, s_ImpossibleValue, \ s_None -from pypy.annotation.listdef import s_list_of_strings from pypy.rpython.lltypesystem import rffi from pypy.rpython.lltypesystem import lltype @@ -94,7 +93,8 @@ lltype.free(l_path, flavor='raw') if error == -1: raise OSError(rffi.c_errno, "utime_null failed") -register_external(ros.utime_null, [str], s_None, llimpl=utime_null_lltypeimpl) +register_external(ros.utime_null, [str], s_None, "ll_os.utime_null", + llimpl=utime_null_lltypeimpl) def utime_tuple_lltypeimpl(path, tp): # XXX right now they're all ints, might change in future @@ -107,7 +107,7 @@ lltype.free(l_utimebuf, flavor='raw') if error == -1: raise OSError(rffi.c_errno, "utime_tuple failed") -register_external(ros.utime_tuple, [str, (int, int)], +register_external(ros.utime_tuple, [str, (int, int)], s_None, "ll_os.utime_tuple", llimpl=utime_tuple_lltypeimpl) os_open = rffi.llexternal('open', [rffi.CCHARP, lltype.Signed, rffi.MODE_T], @@ -120,7 +120,7 @@ if result == -1: raise OSError(rffi.c_errno, "os_open failed") return result -register_external(os.open, [str, int, int], int, export_name="open", +register_external(os.open, [str, int, int], int, "ll_os.open", llimpl=os_open_lltypeimpl) class BaseOS: From fijal at codespeak.net Mon May 28 20:07:45 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 28 May 2007 20:07:45 +0200 (CEST) Subject: [pypy-svn] r43820 - in pypy/branch/kill-ctypes/pypy/rpython: . module module/test Message-ID: <20070528180745.A96B280AA@code0.codespeak.net> Author: fijal Date: Mon May 28 20:07:45 2007 New Revision: 43820 Added: pypy/branch/kill-ctypes/pypy/rpython/module/ll_termios.py pypy/branch/kill-ctypes/pypy/rpython/module/test/test_ll_termios.py Modified: pypy/branch/kill-ctypes/pypy/rpython/extfuncregistry.py Log: First function for termios, only on rpython level. Modified: pypy/branch/kill-ctypes/pypy/rpython/extfuncregistry.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/extfuncregistry.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/extfuncregistry.py Mon May 28 20:07:45 2007 @@ -10,6 +10,12 @@ from pypy.rpython.lltypesystem.module import ll_math from pypy.rpython.ootypesystem.module import ll_math as oo_math from pypy.rpython.module import ll_os +try: + import termios +except ImportError: + pass +else: + from pypy.rpython.module import ll_termios # the following functions all take one float, return one float # and are part of math.h Added: pypy/branch/kill-ctypes/pypy/rpython/module/ll_termios.py ============================================================================== --- (empty file) +++ pypy/branch/kill-ctypes/pypy/rpython/module/ll_termios.py Mon May 28 20:07:45 2007 @@ -0,0 +1,51 @@ + +""" +The low-level implementation of termios module +note that this module should only be imported when +termios module is there +""" + +import termios +from pypy.rpython.lltypesystem import rffi +from pypy.rpython.lltypesystem import lltype +from pypy.rpython.extfunc import register_external +from pypy.rlib.rarithmetic import intmask + +# XXX is this portable? well.. not at all, ideally +# I would like to have NCCS = CLaterConstant(NCCS) +TCFLAG_T = rffi.UINT +CC_T = rffi.UCHAR +NCCS = 32 +SPEED_T = rffi.UINT + +includes = ['termios.h', 'unistd.h'] + +TERMIOSP = rffi.CStruct('termios', ('c_iflag', TCFLAG_T), ('c_oflag', TCFLAG_T), + ('c_cflag', TCFLAG_T), ('c_lflag', TCFLAG_T), + ('c_cc', lltype.FixedSizeArray(CC_T, NCCS))) + +c_tcgetattr = rffi.llexternal('tcgetattr', [lltype.Signed, TERMIOSP], + lltype.Signed, includes=includes) +c_cfgetispeed = rffi.llexternal('cfgetispeed', [TERMIOSP], SPEED_T, + includes=includes) +c_cfgetospeed = rffi.llexternal('cfgetospeed', [TERMIOSP], SPEED_T, + includes=includes) + +def tcgetattr_llimpl(fd): + c_struct = lltype.malloc(TERMIOSP.TO, flavor='raw') + error = c_tcgetattr(fd, c_struct) + if error == -1: + lltype.free(c_struct, flavor='raw') + raise OSError(error, 'tcgetattr failed') + cc = [chr(c_struct.c_c_cc[i]) for i in range(NCCS)] + ispeed = c_cfgetispeed(c_struct) + ospeed = c_cfgetospeed(c_struct) + result = (intmask(c_struct.c_c_iflag), intmask(c_struct.c_c_oflag), + intmask(c_struct.c_c_cflag), intmask(c_struct.c_c_lflag), + intmask(ispeed), intmask(ospeed), cc) + lltype.free(c_struct, flavor='raw') + return result + +register_external(termios.tcgetattr, [int], (int, int, int, int, int, int, [str]), + llimpl=tcgetattr_llimpl, export_name='termios.tcgetattr') + Added: pypy/branch/kill-ctypes/pypy/rpython/module/test/test_ll_termios.py ============================================================================== --- (empty file) +++ pypy/branch/kill-ctypes/pypy/rpython/module/test/test_ll_termios.py Mon May 28 20:07:45 2007 @@ -0,0 +1,71 @@ + +import py +import sys +import os + +from pypy.tool.autopath import pypydir +from pypy.translator.c.test.test_genc import compile +from pypy.tool.udir import udir + +def setup_module(mod): + try: + import pexpect + mod.pexpect = pexpect + except ImportError: + py.test.skip("Pexpect not found") + try: + import termios + mod.termios = termios + except ImportError: + py.test.skip("termios not found") + py_py = py.path.local(pypydir).join('bin', 'py.py') + assert py_py.check() + mod.py_py = py_py + +class TestTermios(object): + def _spawn(self, *args, **kwds): + print 'SPAWN:', args, kwds + child = pexpect.spawn(*args, **kwds) + child.logfile = sys.stdout + return child + + def spawn(self, argv): + return self._spawn(sys.executable, argv) + + def test_getattr(self): + source = py.code.Source(""" + import sys + sys.path.insert(0, '%s') + from pypy.translator.c.test.test_genc import compile + import termios + def runs_tcgetattr(): + tpl = list(termios.tcgetattr(2)[:-1]) + print tpl + + fn = compile(runs_tcgetattr, [], backendopt=False, +) + print 'XXX' + fn(expected_extra_mallocs=1) + print str(termios.tcgetattr(2)[:-1]) + """ % os.path.dirname(pypydir)) + f = udir.join("test_tcgetattr.py") + f.write(source) + child = self.spawn([str(f)]) + child.expect("XXX") + child.expect('\[[^\]]*\]') + first = child.match.group(0) + child.expect('\[[^\]]*\]') + second = child.match.group(0) + assert first == second + + #def test_one(self): + # child = self.spawn() + # child.expect("Python ") + # child.expect('>>> ') + # child.sendline('import termios') + # child.expect('>>> ') + # child.sendline('termios.tcgetattr(0)') + # child.expect('\[.*?\[.*?\]\]') + # lst = eval(child.match.group(0)) + # assert len(lst) == 7 + # assert len(lst[-1]) == 32 # XXX is this portable??? From fijal at codespeak.net Mon May 28 22:19:12 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 28 May 2007 22:19:12 +0200 (CEST) Subject: [pypy-svn] r43822 - in pypy/branch/kill-ctypes/pypy: doc doc/config doc/discussion lang/js lib/distributed lib/distributed/demo lib/distributed/test Message-ID: <20070528201912.6EB528095@code0.codespeak.net> Author: fijal Date: Mon May 28 22:19:12 2007 New Revision: 43822 Modified: pypy/branch/kill-ctypes/pypy/doc/config/objspace.usemodules._socket.txt (props changed) pypy/branch/kill-ctypes/pypy/doc/discussion/compiled-swamp.txt (props changed) pypy/branch/kill-ctypes/pypy/doc/download.txt (props changed) pypy/branch/kill-ctypes/pypy/doc/rffi.txt (props changed) pypy/branch/kill-ctypes/pypy/lang/js/astbuilder.py (props changed) pypy/branch/kill-ctypes/pypy/lang/js/jsparser.py (props changed) pypy/branch/kill-ctypes/pypy/lang/js/newparser.py (props changed) pypy/branch/kill-ctypes/pypy/lib/distributed/demo/fileclient.py (props changed) pypy/branch/kill-ctypes/pypy/lib/distributed/demo/fileserver.py (props changed) pypy/branch/kill-ctypes/pypy/lib/distributed/faker.py (props changed) pypy/branch/kill-ctypes/pypy/lib/distributed/support.py (props changed) pypy/branch/kill-ctypes/pypy/lib/distributed/test/test_socklayer.py (props changed) Log: Fixeol From fijal at codespeak.net Mon May 28 22:27:27 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 28 May 2007 22:27:27 +0200 (CEST) Subject: [pypy-svn] r43823 - in pypy/branch/kill-ctypes/pypy: interpreter module/posix Message-ID: <20070528202727.6F7618095@code0.codespeak.net> Author: fijal Date: Mon May 28 22:27:27 2007 New Revision: 43823 Modified: pypy/branch/kill-ctypes/pypy/interpreter/error.py pypy/branch/kill-ctypes/pypy/module/posix/interp_posix.py Log: Move wrap_oserror to a more convinient place Modified: pypy/branch/kill-ctypes/pypy/interpreter/error.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/interpreter/error.py (original) +++ pypy/branch/kill-ctypes/pypy/interpreter/error.py Mon May 28 22:27:27 2007 @@ -222,6 +222,19 @@ # 31: ANSI color code "red" ansi_print(text, esc="31", file=file, newline=newline) +def wrap_oserror(space, e): + assert isinstance(e, OSError) + errno = e.errno + try: + msg = os.strerror(errno) + except ValueError: + msg = 'error %d' % errno + w_error = space.call_function(space.w_OSError, + space.wrap(errno), + space.wrap(msg)) + return OperationError(space.w_OSError, w_error) + + ### installing the excepthook for OperationErrors ##def operr_excepthook(exctype, value, traceback): ## if issubclass(exctype, OperationError): Modified: pypy/branch/kill-ctypes/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/kill-ctypes/pypy/module/posix/interp_posix.py Mon May 28 22:27:27 2007 @@ -1,21 +1,9 @@ from pypy.interpreter.baseobjspace import ObjSpace, W_Root from pypy.rlib.rarithmetic import intmask from pypy.rlib import ros -from pypy.interpreter.error import OperationError +from pypy.interpreter.error import OperationError, wrap_oserror import os - -def wrap_oserror(space, e): - assert isinstance(e, OSError) - errno = e.errno - try: - msg = os.strerror(errno) - except ValueError: - msg = 'error %d' % errno - w_error = space.call_function(space.w_OSError, - space.wrap(errno), - space.wrap(msg)) - return OperationError(space.w_OSError, w_error) def open(space, fname, flag, mode=0777): """Open a file (for low level IO). From fijal at codespeak.net Mon May 28 22:28:04 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 28 May 2007 22:28:04 +0200 (CEST) Subject: [pypy-svn] r43824 - pypy/branch/kill-ctypes/pypy/rpython/module Message-ID: <20070528202804.56E978095@code0.codespeak.net> Author: fijal Date: Mon May 28 22:28:03 2007 New Revision: 43824 Modified: pypy/branch/kill-ctypes/pypy/rpython/module/ll_termios.py Log: Strange hack to be able to translate this stuff Modified: pypy/branch/kill-ctypes/pypy/rpython/module/ll_termios.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/module/ll_termios.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/module/ll_termios.py Mon May 28 22:28:03 2007 @@ -20,6 +20,10 @@ includes = ['termios.h', 'unistd.h'] +# XXX all functions here raise OSError, because they cannot +# raise termios.error (lack of translation possibilities). hence +# I don't know how to solve this, the tests will probably don't work + TERMIOSP = rffi.CStruct('termios', ('c_iflag', TCFLAG_T), ('c_oflag', TCFLAG_T), ('c_cflag', TCFLAG_T), ('c_lflag', TCFLAG_T), ('c_cc', lltype.FixedSizeArray(CC_T, NCCS))) @@ -49,3 +53,4 @@ register_external(termios.tcgetattr, [int], (int, int, int, int, int, int, [str]), llimpl=tcgetattr_llimpl, export_name='termios.tcgetattr') +#def tcsetattr_llimpl(fd From fijal at codespeak.net Mon May 28 22:30:46 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 28 May 2007 22:30:46 +0200 (CEST) Subject: [pypy-svn] r43825 - in pypy/branch/kill-ctypes/pypy/module/termios: . test Message-ID: <20070528203046.D8BD58095@code0.codespeak.net> Author: fijal Date: Mon May 28 22:30:46 2007 New Revision: 43825 Added: pypy/branch/kill-ctypes/pypy/module/termios/ (props changed) pypy/branch/kill-ctypes/pypy/module/termios/__init__.py (contents, props changed) pypy/branch/kill-ctypes/pypy/module/termios/app_termios.py (contents, props changed) pypy/branch/kill-ctypes/pypy/module/termios/interp_termios.py (contents, props changed) pypy/branch/kill-ctypes/pypy/module/termios/test/ (props changed) pypy/branch/kill-ctypes/pypy/module/termios/test/test_termios.py (contents, props changed) Log: Add an app level version of termios (well, one function right now :) Added: pypy/branch/kill-ctypes/pypy/module/termios/__init__.py ============================================================================== --- (empty file) +++ pypy/branch/kill-ctypes/pypy/module/termios/__init__.py Mon May 28 22:30:46 2007 @@ -0,0 +1,37 @@ + +from pypy.interpreter.mixedmodule import MixedModule + +class Module(MixedModule): + "This module provides an interface to the Posix calls for tty I/O control.\n\ + For a complete description of these calls, see the Posix or Unix manual\n\ + pages. It is only available for those Unix versions that support Posix\n\ + termios style tty I/O control.\n\ + \n\ + All functions in this module take a file descriptor fd as their first\n\ + argument. This can be an integer file descriptor, such as returned by\n\ + sys.stdin.fileno(), or a file object, such as sys.stdin itself." + applevel_name = "termios" + + appleveldefs = { + 'error' : 'app_termios.error', + } + + interpleveldefs = { + 'tcdrain' : 'interp_termios.tcdrain', + 'tcflow' : 'interp_termios.tcflow', + 'tcflush' : 'interp_termios.tcflush', + 'tcgetattr' : 'interp_termios.tcgetattr', + 'tcsendbreak' : 'interp_termios.tcsendbreak', + 'tcsetattr' : 'interp_termios.tcsetattr', + } + +import termios +from pypy.module.termios import interp_termios + +# XXX this is extremaly not-portable, but how to prevent this? + +for i in dir(termios): + val = getattr(termios, i) + if i.isupper() and type(val) is int: + Module.interpleveldefs[i] = "space.wrap(%s)" % val + Added: pypy/branch/kill-ctypes/pypy/module/termios/app_termios.py ============================================================================== --- (empty file) +++ pypy/branch/kill-ctypes/pypy/module/termios/app_termios.py Mon May 28 22:30:46 2007 @@ -0,0 +1,3 @@ + +class error(Exception): + pass Added: pypy/branch/kill-ctypes/pypy/module/termios/interp_termios.py ============================================================================== --- (empty file) +++ pypy/branch/kill-ctypes/pypy/module/termios/interp_termios.py Mon May 28 22:30:46 2007 @@ -0,0 +1,68 @@ + +""" Termios module. I'm implementing it directly here, as I see +little use of termios module on RPython level by itself +""" + +from pypy.interpreter.baseobjspace import ObjSpace, W_Root +from pypy.interpreter.error import OperationError +import os +import termios +from pypy.rlib.objectmodel import we_are_translated + +# proper semantics are to have termios.error, but since it's not documented +# anyway, let's have it as OSError on interplevel. We need to have +# some details what is missing in RPython modules though + +def convert_error(space, error): + errno = error.errno + w_module = space.getbuiltinmodule('termios') + w_exception_class = space.getattr(w_module, space.wrap('error')) + try: + msg = os.strerror(errno) + except ValueError: + msg = 'error %d' % errno + w_exception = space.call_function(w_exception_class, space.wrap(errno), + space.wrap(msg)) + return OperationError(w_exception_class, w_exception) + +def tcsetattr(space, fd, when, w_attributes): + pass +tcsetattr.unwrap_spec = [ObjSpace, int, int, W_Root] + +def tcgetattr(space, fd): + # XXX Argh argh argh argh. ARGH! + if we_are_translated(): + try: + tup_w = termios.tcgetattr(fd) + except OSError, e: + raise convert_error(space, e) + else: + try: + tup_w = termios.tcgetattr(fd) + except termios.error, e: + e.errno = e.args[0] + raise convert_error(space, e) + l_w = [] + for w_item in tup_w[:-1]: + l_w.append(space.wrap(w_item)) + # last one need to be chosen carefully + w_cc = space.newlist([space.wrap(i) for i in tup_w[-1]]) + l_w.append(w_cc) + return space.newlist(l_w) +tcgetattr.unwrap_spec = [ObjSpace, int] + +def tcsendbreak(space, fd, duration): + pass +tcsendbreak.unwrap_spec = [ObjSpace, int, int] + +def tcdrain(space, fd, duration): + pass +tcdrain.unwrap_spec = [ObjSpace, int, int] + +def tcflush(space, fd, queue): + pass +tcflush.unwrap_spec = [ObjSpace, int, int] + +def tcflow(space, fd, action): + pass +tcflow.unwrap_spec = [ObjSpace, int, int] Added: pypy/branch/kill-ctypes/pypy/module/termios/test/test_termios.py ============================================================================== --- (empty file) +++ pypy/branch/kill-ctypes/pypy/module/termios/test/test_termios.py Mon May 28 22:30:46 2007 @@ -0,0 +1,67 @@ + +import py +import sys +from pypy.conftest import gettestobjspace +from pypy.tool.autopath import pypydir + +def setup_module(mod): + try: + import pexpect + mod.pexpect = pexpect + except ImportError: + py.test.skip("Pexpect not found") + try: + import termios + mod.termios = termios + except ImportError: + py.test.skip("termios not found") + py_py = py.path.local(pypydir).join('bin', 'py.py') + assert py_py.check() + mod.py_py = py_py + +class TestTermios(object): + def _spawn(self, *args, **kwds): + print 'SPAWN:', args, kwds + child = pexpect.spawn(*args, **kwds) + child.logfile = sys.stdout + return child + + def spawn(self, argv): + return self._spawn(sys.executable, [str(py_py)] + argv) + + def test_one(self): + child = self.spawn(['--withmod-termios']) + child.expect("Python ") + child.expect('>>> ') + child.sendline('import termios') + child.expect('>>> ') + child.sendline('termios.tcgetattr(0)') + child.expect('\[.*?\[.*?\]\]') + lst = eval(child.match.group(0)) + assert len(lst) == 7 + assert len(lst[-1]) == 32 # XXX is this portable??? + +class AppTestTermios(object): + def setup_class(cls): + cls.space = gettestobjspace(usemodules=['termios']) + d = {} + for name in dir(termios): + val = getattr(termios, name) + if name.isupper() and type(val) is int: + d[name] = val + cls.w_orig_module_dict = cls.space.appexec([], "(): return %r" % (d,)) + + def test_values(self): + skip("Not supported yet") + import termios + d = {} + for name in dir(termios): + val = getattr(termios, name) + if name.isupper() and type(val) is int: + d[name] = val + assert d == self.orig_module_dict + + def test_error(self): + # XXX not always true, but good assumption + import termios + raises(termios.error, "termios.tcgetattr(334)") From pdg at codespeak.net Tue May 29 04:15:41 2007 From: pdg at codespeak.net (pdg at codespeak.net) Date: Tue, 29 May 2007 04:15:41 +0200 (CEST) Subject: [pypy-svn] r43827 - pypy/dist/pypy/translator/jvm/test Message-ID: <20070529021541.E95638076@code0.codespeak.net> Author: pdg Date: Tue May 29 04:15:40 2007 New Revision: 43827 Added: pypy/dist/pypy/translator/jvm/test/test_contant.py pypy/dist/pypy/translator/jvm/test/test_exception.py pypy/dist/pypy/translator/jvm/test/test_float.py pypy/dist/pypy/translator/jvm/test/test_int.py Modified: pypy/dist/pypy/translator/jvm/test/runtest.py Log: translator/jvm/test - added test_constant test_exception test_float test_int and added _skip_powerpc to runtest Modified: pypy/dist/pypy/translator/jvm/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/runtest.py (original) +++ pypy/dist/pypy/translator/jvm/test/runtest.py Tue May 29 04:15:40 2007 @@ -99,6 +99,10 @@ def _skip_win(self, reason): if platform.system() == 'Windows': py.test.skip('Windows --> %s' % reason) + + def _skip_powerpc(self, reason): + if platform.processor() == 'powerpc': + py.test.skip('PowerPC --> %s' % reason) def interpret(self, fn, args, annotation=None): detect_missing_support_programs() Added: pypy/dist/pypy/translator/jvm/test/test_contant.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/jvm/test/test_contant.py Tue May 29 04:15:40 2007 @@ -0,0 +1,131 @@ +import py +from pypy.translator.jvm.test.runtest import JvmTest + +# used in tests below +class A: + pass + + +class TestConstant(JvmTest): + def test_char(self): + const = 'a' + def fn(): + return const + assert self.interpret(fn, []) == 'a' + + def test_void(self): + def fn(): + pass + assert self.interpret(fn, []) is None + + def test_tuple(self): + const = 1, 2 + def fn(): + return const + res = self.interpret(fn, []) + assert res.item0 == 1 + assert res.item1 == 2 + + def test_list(self): + const = [1, 2] + def fn(): + return const + res = self.ll_to_list(self.interpret(fn, [])) + assert res == [1, 2] + + def test_compound_const(self): + const = ([1, 2], [3, 4]) + def fn(): + return const + res = self.interpret(fn, []) + assert self.ll_to_list(res.item0) == [1, 2] + assert self.ll_to_list(res.item1) == [3, 4] + + def test_instance(self): + const = A() + def fn(): + return const + res = self.interpret(fn, []) + assert self.class_name(res) == 'A' + + def test_list_of_zeroes(self): + const = [0] * 10 + def fn(): + return const + res = self.ll_to_list(self.interpret(fn, [])) + assert res == const + + def test_list_of_instances(self): + const = [A()] + def fn(): + return const + res = self.ll_to_list(self.interpret(fn, [])) + assert self.class_name(res[0]) == 'A' + + def test_mix_string_and_char(self): + def fn(x): + if x < 0: + return 'a' + else: + return 'aa' + assert self.ll_to_string(self.interpret(fn, [-1])) == 'a' + assert self.ll_to_string(self.interpret(fn, [0])) == 'aa' + + def test_string_literal(self): + def fn(): + return 'hello "world"' + assert self.ll_to_string(self.interpret(fn, [])) == 'hello "world"' + + def test_string_literal2(self): + literals = ['\001\002\003', '\004\005\006'] + def fn(i): + s = literals[i] + return len(s), ord(s[0]) + ord(s[1]) + ord(s[2]) + res = self.interpret(fn, [0]) + assert res.item0 == 3 + assert res.item1 == 6 + res = self.interpret(fn, [1]) + assert res.item0 == 3 + assert res.item1 == 15 + + def test_float_special(self): + self._skip_win('inf & nan') + self._skip_powerpc('Suspected endian issue with '+ + 'representation of INF and NAN') + c = [float("inf"), float("nan")] + def fn(i): + return c[i]*2 == c[i] + def fn2(i): + return c[i] != c[i] + assert self.interpret(fn, [0]) == True + assert self.interpret(fn2, [1]) == True + + def test_customdict_circular(self): + from pypy.rlib.objectmodel import r_dict + def key_eq(a, b): + return a.x[0] == b.x[0] + def key_hash(a): + return ord(a.x[0]) + + class A: + def __init__(self, x): + self.x = x + a = A('foo') + a.dict = r_dict(key_eq, key_hash) + a.dict[a] = 42 + def fn(b): + if b: + s = A('foo') + else: + s = A('bar') + return a.dict[s] + assert self.interpret(fn, [True]) == 42 + + def test_multiple_step(self): + from pypy.translator.oosupport import constant + constant.MAX_CONST_PER_STEP = 2 + c1 = [1] + c2 = [2] + def fn(x, y): + return c1[x] + c2[y] + assert self.interpret(fn, [0, 0]) == 3 Added: pypy/dist/pypy/translator/jvm/test/test_exception.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/jvm/test/test_exception.py Tue May 29 04:15:40 2007 @@ -0,0 +1,57 @@ +import py +from pypy.translator.jvm.test.runtest import JvmTest +from pypy.rpython.test.test_exception import BaseTestException + +class TestJvmException(JvmTest, BaseTestException): + def test_nested_try(self): + def helper(x): + if x == 0: + raise ValueError + def dummy(): + pass + def fn(x): + try: + try: + helper(x) + finally: + dummy() + except ValueError, e: + raise + + self.interpret_raises(ValueError, fn, [0]) + + def test_exception_not_last(self): + def helper(x): + if x == 0: + raise ValueError + def fn(x): + helper(x) + try: + helper(1) + finally: + return -1 + return x + self.interpret_raises(ValueError, fn, [0]) + + def test_raise_and_catch_other(self): + pass + + def test_raise_prebuilt_and_catch_other(self): + pass + + def test_missing_return_block(self): + class Base: + def foo(self): + raise ValueError + + class Derived(Base): + def foo(self): + return 42 + + def fn(x): + if x: + obj = Base() + else: + obj = Derived() + return obj.foo() + assert self.interpret(fn, [0]) == 42 Added: pypy/dist/pypy/translator/jvm/test/test_float.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/jvm/test/test_float.py Tue May 29 04:15:40 2007 @@ -0,0 +1,18 @@ +import py +from pypy.translator.jvm.test.runtest import JvmTest +from pypy.rpython.test.test_rfloat import BaseTestRfloat + +class TestJvmFloat(JvmTest, BaseTestRfloat): + def test_parse_float(self): + ex = ['', ' ', '0', '1', '-1.5', '1.5E2', '2.5e-1', ' 0 ', '?'] + def fn(i): + s = ex[i] + try: + return float(s) + except ValueError: + return -999.0 + + for i in range(len(ex)): + expected = fn(i) + res = self.interpret(fn, [i]) + assert res == expected \ No newline at end of file Added: pypy/dist/pypy/translator/jvm/test/test_int.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/jvm/test/test_int.py Tue May 29 04:15:40 2007 @@ -0,0 +1,20 @@ +import py +from pypy.translator.jvm.test.runtest import JvmTest +from pypy.rpython.test.test_rint import BaseTestRint + +class TestJvmInt(JvmTest, BaseTestRint): + def test_char_constant(self): + def dummyfn(i): + return chr(i) + res = self.interpret(dummyfn, [ord(' ')]) + assert res == ' ' + # Is the following test supported by JVM? +## res = self.interpret(dummyfn, [0]) +## assert res == '\0' + res = self.interpret(dummyfn, [ord('a')]) + assert res == 'a' + + def test_rarithmetic(self): + pass # does this make more sense in jvm + + div_mod_iteration_count = 20 From arigo at codespeak.net Tue May 29 09:36:03 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 29 May 2007 09:36:03 +0200 (CEST) Subject: [pypy-svn] r43834 - pypy/extradoc/talk/dls2007 Message-ID: <20070529073603.57764808D@code0.codespeak.net> Author: arigo Date: Tue May 29 09:36:02 2007 New Revision: 43834 Added: pypy/extradoc/talk/dls2007/ pypy/extradoc/talk/dls2007/acm_proc_article-sp.cls - copied unchanged from r43833, pypy/extradoc/talk/dls2006/acm_proc_article-sp.cls pypy/extradoc/talk/dls2007/paper.tex Log: Partial LaTeX-ification of D08.2. That's 9 pages for now. Added: pypy/extradoc/talk/dls2007/paper.tex ============================================================================== --- (empty file) +++ pypy/extradoc/talk/dls2007/paper.tex Tue May 29 09:36:02 2007 @@ -0,0 +1,1015 @@ +\documentclass{acm_proc_article-sp} + +\begin{document} + +\title{Generating Just-In-Time Specializing Compilers} + +\numberofauthors{2} +\author{ +\alignauthor Armin Rigo\\ + \affaddr{Heinrich-Heine-Universit?t D?sseldorf}\\ + \affaddr{Institut f?r Informatik}\\ + \affaddr{Universit?tsstra{\ss}e 1}\\ + \affaddr{D-40225 D?sseldorf}\\ + \affaddr{Deutschland}\\ + \email{arigo at tunes.org} +\alignauthor Samuele Pedroni\\ + \affaddr{AB Strakt}\\ + \affaddr{Norra ?gatan 10A}\\ + \affaddr{416 64 G?teborg}\\ + \affaddr{Sweden}\\ + \email{pedronis at strakt.com} +} +\date{31 May 2007} +\maketitle + +%\category{D.3.4}{Programming Languages}{Processors}[code generation, +%interpreters, run-time environments] +%\category{F.3.2}{Logics and Meanings of Programs}{Semantics of Programming +%Languages}[program analysis] + +\begin{abstract} +PyPy's translation tool-chain -- from the interpreter written in RPython +to generated VMs for low-level platforms -- is now able to extend those +VMs with an automatically generated dynamic compiler, derived from the +interpreter. This is achieved by a pragmatic application of partial +evaluation techniques guided by a few hints added to the source of the +interpreter. Crucial for the effectiveness of dynamic compilation is +the use of run-time information to improve compilation results: in +our approach, a novel powerful primitive called "promotion" that "promotes" +run-time values to compile-time is used to that effect. In this report, +we describe it along with other novel techniques that allow the approach +to scale to something as large as PyPy's Python interpreter. +\end{abstract} + +\section{Introduction} + +Dynamic compilers are resource costly to write and hard to maintain, +but highly desirable for competitive performance. Straight-forward +bytecode interpreters are easier to write. Hybrid approaches have been +experimented with \cite{REJIT}, but this is clearly an area in need of +research and innovative approaches. + +One of the central goals of the PyPy project is to automatically +produce dynamic compilers from an interpreter, with as little +modifications of the interpreter code base itself as possible. + +The forest of flow graphs that the translation process \cite{VMCDLS} +generates and transforms constitutes a reasonable base for the +necessary analyses. That's a further reason why having a high-level +runnable and analyzable interpreter implementation was always a +central tenet of the project: in our approach, +the dynamic compiler is just another aspect +transparently introduced by and during the translation +process. + +Partial evaluation techniques should, at least theoretically, +allow such a derivation of a compiler from an interpreter [PE], but it +is not reasonable to expect the code produced for an input program by +a compiler derived using partial evaluation to be very good, +especially in the case of a dynamic language. Essentially, the input +program doesn't contain enough information to generate good code; for +example the input program contains mostly no kind of type +information in that case. + +What is really desired is not to produce a compiler doing static +ahead-of-time compilation, as classical partial evaluation would do, +but one capable of dynamic compilation, exploiting run-time +information in its result. Compilation should be able to suspend, let +the produced code run to collect run-time information (for example +language-level types), and then resume with this extra information. +This will allows the compiler to generate code optimized for the +effective run-time behaviour of the program. + +Inspired by Psyco \cite{PSYCO}, which is a hand-written dynamic compiler +based on partial evaluation for Python, we developed a technique - +*promotion* - for our dynamic compiler generator. Simply put, promotion +on a value stops compilation and waits until the run-time reaches this +point. When it does, the actual run-time value is promoted into a +compile-time constant, and compilation resumes with this extra +information. + +Promotion is an essential technique to be able to generate really +dynamic compilers that can exploit run-time information. +Besides promotion (section \ref{promotion}), +the novel techniques introduced by PyPy that allow +the approach to scale are virtualizable structures +(section \ref{virtualizable}) and need-oriented +binding time analysis (section \ref{bta}). + + +\subsection{Overview of partial evaluation} + +Partial evaluation is the process of evaluating a function, say ``f(x, +y)``, with only partial information about the values of its arguments, +say the value of the ``x`` argument only. This produces a *residual* +function ``g(y)``, which takes less arguments than the original -- only +the information not specified during the partial evaluation process needs +to be provided to the residual function, in this example the ``y`` +argument. + +Partial evaluation (PE) comes in two flavors: + +* *On-line* PE: a compiler-like algorithm takes the source code of the + function ``f(x, y)`` (or its intermediate representation, i.e. its + control flow graph in PyPy's terminology), and some partial + information, e.g. ``x=5``. From this, it produces the residual + function ``g(y)`` directly, by following in which operations the + knowledge ``x=5`` can be used, which loops can be unrolled, etc. + +* *Off-line* PE: in many cases, the goal of partial evaluation is to + improve performance in a specific application. Assume that we have a + single known function ``f(x, y)`` in which we think that the value of + ``x`` will change slowly during the execution of our program -- much + more slowly than the value of ``y``. An obvious example is a loop + that calls ``f(x, y)`` many times with always the same value ``x``. + We could then use an on-line partial evaluator to produce a ``g(y)`` + for each new value of ``x``. In practice, the overhead of the partial + evaluator might be too large for it to be executed at run-time. + However, if we know the function ``f`` in advance, and if we know + *which* arguments are the ones that we will want to partially evaluate + ``f`` with, then we do not need a full compiler-like analysis of ``f`` + every time the value of ``x`` changes. We can precompute once and for + all a specialized function ``f1(x)``, which when called produces the + residual function ``g(y)`` corresponding to ``x``. This is *off-line + partial evaluation;* the specialized function ``f1(x)`` is called a + *generating extension*. + +The PyPy JIT generation framework is based on off-line partial +evaluation. The function called ``f(x, y)`` above is typically the main +loop of some interpreter written in RPython. The size of the interpreter can range +from a three-liner used for testing purposes to the whole of PyPy's +Python interpreter. In all cases, ``x`` stands for the input program +(the bytecode to interpret) and ``y`` stands for the input data (like a +frame object with the binding of the input arguments and local +variables). Our framework is capable of automatically producing the +corresponding generating extension ``f1(x)``, which takes an input +program only and produces a residual function ``g(y)``. This ``f1(x)`` +is a compiler\footnote{ + What we get in PyPy is more precisely a \emph{just-in-time compiler:} + if promotion is used, compiling ahead of time is not possible. +} +for the very same language for which ``f(x, y)`` is +an interpreter. + +Off-line partial evaluation is based on *binding-time analysis,* which +is the process of determining among the variables used in a function (or +a set of functions) which ones are going to be known in advance and +which ones are not. In the example of ``f(x, y)``, such an analysis +would be able to infer that the constantness of the argument ``x`` +implies the constantness of many intermediate values used in the +function. The *binding time* of a variable determines how early the +value of the variable will be known. + +Once binding times have been determined, one possible approach to +producing the generating extension itself is by self-applying on-line +partial evaluators. This is known as the second Futamura projection +\cite{FU}. So far it is unclear if this approach can lead to optimal +results, or even if it scales well. In PyPy we selected a more direct +approach: the generating extension is produced by transformation of the +control flow graphs of the interpreter, guided by the binding times. We +call this process *timeshifting*. + + +\section{Architecture and Principles} + +PyPy contains a framework for generating just-in-time compilers using +off-line partial evaluation. As such, there are three distinct phases: + +* *Translation time:* during the normal translation of an RPython + program, say PyPy's Python interpreter, we perform binding-time + analysis and off-line specialization ("timeshifting") of the + interpreter. This produces a generating extension, which is linked + with the rest of the program. + +* *Compile time:* during the execution of the program, when a new + bytecode is about to be interpreted, the generating extension is + invoked instead. As the generating extension is a compiler, all the + computations it performs are called compile-time computations. Its + sole effect is to produce residual code. + +* *Run time:* the normal execution of the program (which includes the + time spent running the residual code created by the generating + extension). + +Translation time is a purely off-line phase; compile time and run time +are actually highly interleaved during the execution of the program. + + +\subsection{Binding Time Analysis} +\label{bta} + +At translation time, PyPy performs binding-time analysis of the source +RPython program after it has been turned to low-level graphs, i.e. at +the level at which operations manipulate pointer-and-structure-like +objects. + +The binding-time terminology that we are using in PyPy is based on the +colors that we use when displaying the control flow graphs: + +* *Green* variables contain values that are known at compile-time; +* *Red* variables contain values that are not known until run-time. + +The binding-time analyzer of our translation tool-chain is based on the +same type inference engine that is used on the source RPython program, +the annotator. In this mode, it is called the *hint-annotator*; it +operates over input graphs that are already low-level instead of +RPython-level, and propagates annotations that do not track types but +value dependencies and manually-provided binding time hints. + +The normal process of the hint-annotator is to propagate the binding +time (i.e. color) of the variables using the following kind of rules: + +* For a foldable operation (i.e. one without side effect and which + depends only on its argument values), if all arguments are green, + then the result can be green too. + +* Non-foldable operations always produce a red result. + +* At join points, where multiple possible values (depending on control + flow) are meeting into a fresh variable, if any incoming value comes + from a red variable, the result is red. Otherwise, the color of the + result might be green. We do not make it eagerly green, because of + the control flow dependency: the residual function is basically a + constant-folded copy of the source function, so it might retain some + of the same control flow. The value that needs to be stored in the + fresh join variable thus depends on which branches are taken in the + residual graph. + +\subsubsection*{Hints} + +Our goal in designing our approach to binding-time analysis was to +minimize the number of explicit hints that the user must provide in +the source of the RPython program. This minimalism was not pushed to +extremes, though, to keep the hint-annotator reasonably simple. + +The driving idea was that hints should be need-oriented. Indeed, in a +program like an interpreter, there are a small number of places where it +would be clearly beneficial for a given value to be known at +compile-time, i.e. green: this is where we require the hints to be +added. + +The hint-annotator assumes that all variables are red by default, and +then propagates annotations that record dependency information. +When encountering the user-provided hints, the dependency information +is used to make some variables green. All +hints are in the form of an operation ``hint(v1, someflag=True)`` +which semantically just returns its first argument unmodified. + +The crucial need-oriented hint is ``v2 = hint(v1, concrete=True)`` +which should be used in places where the programmer considers the +knowledge of the value to be essential. This hint is interpreted by +the hint-annotator as a request for both ``v1`` and ``v2`` to be green. It +has a *global* effect on the binding times: it means that not only +``v1`` but all the values that ``v1`` depends on -- recursively -- +are forced to be green. The hint-annotator complains if the +dependencies of ``v1`` include a value that cannot be green, like +a value read out of a field of a non-immutable structure. + +Such a need-oriented backward propagation has advantages over the +commonly used forward propagation, in which a variable is compile-time +if and only if all the variables it depends on are also compile-time. A +known issue with forward propagation is that it may mark as compile-time +either more variables than expected (which leads to over-specialization +of the residual code), or less variables than expected (preventing +specialization to occur where it would be the most useful). Our +need-oriented approach reduces the problem of over-specialization, and +it prevents under-specialization: an unsatisfiable ``hint(v1, +concrete=True)`` is reported as an error. + +In our context, though, such an error can be corrected. This is done by +promoting a well-chosen variable among the ones that ``v1`` depends on. + +Promotion is invoked with the use of a hint as well: +``v2 = hint(v1, promote=True)``. +This hint is a *local* request for ``v2`` to be green, without +requiring ``v1`` to be green. Note that this amounts to copying +a red value into a green one, which is not possible in classical +approaches to partial evaluation. See section \ref{promotion} for a +complete discussion of promotion. + +For examples and further discussion on how the hints are applied in practice +see `Make your own JIT compiler` \cite{D08.1}. + +\subsection{Timeshifting} + +Once binding times (colors) have been assigned to all variables in a +family of control flow graphs, the next step is to mutate the graphs\footnote{ + One should keep in mind that the program described as the "source RPython + program" in this document is typically an interpreter -- the canonical + example is that it is the whole PyPy Standard Interpreter. This + program is meant to execute at run-time, and directly compute the + intended result and side-effects. The translation process transforms + it into a forest of flow graphs. These are the flow graphs that + timeshifting processes (and not the application-level program, which typically + cannot be expressed as low-level flow graphs). +} +accordingly in order to produce a generating extension. We call +this process *timeshifting* because it changes the time at +which the graphs are meant to be run, from run-time to compile-time. + +Despite the execution time and side-effects shift to produce only +residual code, the timeshifted graphs have a shape (flow of control) +that is closely related to that of the original graphs. This is because +at compile-time the timeshifted graphs go over all the operations that +the original graphs would have performed at run-time, following the same +control flow; some of these operations and control flow constructs are +constant-folded at compile-time, and the rest is turned into equivalent +residual code. Another point of view is that as the timeshifted graphs +form a generating extension, they perform the equivalent of an abstract +interpretation of the original graphs over a domain containing +compile-time values and run-time value locations. + +The rest of this section describes this timeshifting process in more +detail. + +\subsubsection*{Red and Green Operations} + +The basic idea of timeshifting is to transform operations in a way that +depends on the color of their operands and result. Variables themselves +need to be represented based on their color: + +* The red (run-time) variables have abstract values at compile-time; + no actual value is available for them during compile-time. For them + we use a boxed representation that can carry either a run-time storage + location (a stack frame position or a register name) or an immediate + constant (for when the value is, after all, known at compile-time). + +* On the other hand, the green variables are the ones that can carry + their value already at compile-time, so they are left untouched during + timeshifting. + +The operations of the original graphs are then transformed as follows: + +* If an operation has no side effect nor any other run-time dependency, and + if it only involves green operands, then it can stay unmodified in the + graph. In this case, the operation that was run-time in the original + graph becomes a compile-time operation, and it will never be generated + in the residual code. (This is the case that makes the whole approach + worthwhile: some operations become purely compile-time.) + +* In all other cases, the operation might have to be generated in the + residual code. In the timeshifted graph it is replaced by a call + to a helper which will generate a residual operation manipulating + the input run-time values and return a new boxed representation + for the run-time result location. + +These helpers will constant-fold the operation if the inputs +are immediate constants and if the operation has no side-effects. Immediate constants can occur even though the +corresponding variable in the graph was red: a variable can be +dynamically found to contain a compile-time constant at a particular +point in (compile)-time, independently of the hint-annotator +proving that it is always the case. +In Partial Evaluation terminology, the timeshifted graphs are +performing some *on-line* partial evaluation in addition to the +off-line job enabled by the hint-annotator. + +\subsubsection*{Merges and Splits} + +The timeshifted code carries around an object that stores the +compilation-time state -- mostly the current bindings of the variables. +This state is used to shape the control flow of the generated residual +code, as follows. + +After a *split,* i.e. after a conditional branch that could not be +folded at compile-time, the compilation state is duplicated and both +branches are compiled independently. Conversely, after a *merge point,* +i.e. when two control flow paths meet each other, we try to join the two +paths in the residual code. This part is more difficult because the two +paths may need to be compiled with different variable bindings -- e.g. +different variables may be known to take different compile-time constant +values in the two branches. The two paths can either be kept separate +or merged; in the latter case, the merged compilation-time state needs +to be a generalization (*widening*) of the two already-seen states. +Deciding when to do each is a classical problem of partial evaluation, +as merging too eagerly may loose important precision and not merging +eagerly enough may create too many redundant residual code paths (to the +point of preventing termination of the compiler). + +So far, we did not investigate this problem in detail. We settled for a +simple widening heuristic: two different compile-time constants merge as +a run-time value, but we try to preserve the richer models of run-time +information that are enabled by the techniques described in the sequel +(promotion (\ref{promotion}), virtual structures (\ref{virtual})...). +This heuristic seems to work +for PyPy to some extent. + +\subsubsection*{Calls and inlining} + +For calls timeshifting can either produce code to generate a residual +call operation or recursively invoke the timeshifted version of the +callee. The residual operations generated by the timeshifted callee +will grow the compile-time produced residual function; this +effectively amounts to the compile-time inlining of the original callee into +its caller. This is the default behaviour for calls within the +user-controlled subset of original graphs of the interpreter that are +timeshifted. Inlining only stops at re-entrant calls to the +interpreter main loop; the net result is that at the level of the +interpreted language, each function (or method) gets compiled into +a single piece of residual code. + +\subsection{Promotion} +\label{promotion} + +In the sequel, we describe in more details one of the main new +techniques introduced in our approach, which we call *promotion*. In +short, it allows an arbitrary run-time value to be turned into a +compile-time value at any point in time. Each promotion point is +explicitly defined with a hint that must be put in the source code of +the interpreter. + +From a partial evaluation point of view, promotion is the converse of +the operation generally known as "lift". Lifting a value means +copying a variable whose binding time is compile-time into a variable +whose binding time is run-time -- it corresponds to the compiler +"forgetting" a particular value that it knew about. By contrast, +promotion is a way for the compiler to gain *more* information about +the run-time execution of a program. Clearly, this requires +fine-grained feedback from run-time to compile-time, thus a +dynamic setting. + +Promotion requires interleaving compile-time and run-time phases, +otherwise the compiler can only use information that is known ahead of +time. It is impossible in the "classical" approaches to partial +evaluation, in which the compiler always runs fully ahead of execution +This is a problem in many large use cases. For example, in an +interpreter for a dynamic language, there is mostly no information +that can be clearly and statically used by the compiler before any +code has run. + +A more theoretical way to see the issue is to consider that the +possible binding time for each variable in the interpreter is +constrained by the binding time of the other variables it depends on. +For some kind of interpreters this set of constraints may have no +interesting global solution -- if most variables can ultimately depend +on a value, even in just one corner case, which cannot be +compile-time, then in any solution most variables will be run-time. +In the presence of promotion, though, these constraints can be +occasionally violated: corner cases do not necessarily have to +influence the common case, and local solutions can be patched +together. + +A very different point of view on promotion is as a generalization of +techniques that already exist in dynamic compilers as found in modern +object-oriented language virtual machines. In this context feedback +techniques are crucial for good results. The main goal is to +optimize and reduce the overhead of dynamic dispatching and indirect +invocation. This is achieved with variations on the technique of +polymorphic inline caches \cite{PIC}: the dynamic lookups are cached and +the corresponding generated machine code contains chains of +compare-and-jump instructions which are modified at run-time. These +techniques also allow the gathering of information to direct inlining for even +better optimization results. + +In the presence of promotion, dispatch optimization can usually be +reframed as a partial evaluation task. Indeed, if the type of the +object being dispatched to is known at compile-time, the lookup can be +folded, and only a (possibly inlined) direct call remains in the +generated code. In the case where the type of the object is not known +at compile-time, it can first be read at run-time out of the object and +promoted to compile-time. As we will see in the sequel, this produces +very similar machine code.\footnote{ + This can also be seen as a generalization of a partial + evaluation transformation called "The Trick" (see e.g. \cite{PE}), + which again produces similar code but which is only + applicable for finite sets of values. +} + +The essential advantage is that it is no longer tied to the details of +the dispatch semantics of the language being interpreted, but applies in +more general situations. Promotion is thus the central enabling +primitive to make timeshifting a practical approach to language +independent dynamic compiler generation. + +\subsubsection*{Promotion in practice} + +The implementation of promotion requires a tight coupling between +compile-time and run-time: a *callback,* put in the generated code, +which can invoke the compiler again. When the callback is actually +reached at run-time, and only then, the compiler resumes and uses the +knowledge of the actual run-time value to generate more code. + +The new generated code is potentially different for each run-time value +seen. This implies that the generated code needs to contain some sort +of updatable switch, which can pick the right code path based on the +run-time value. + +While this describes the general idea, the details are open to slight +variations. Let us show more precisely the way the JIT compilers +produced by PyPy 1.0 work. Our first example is purely artificial: + +\begin{verbatim} + ... + b = a / 10 + c = hint(b, promote=True) + d = c + 5 + print d + ... +\end{verbatim} + +In this example, ``a`` and ``b`` are run-time variables and ``c`` and +``d`` are compile-time variables; ``b`` is copied into ``c`` via a +promotion. The division is a run-time operation while the addition is a +compile-time operation. + +The compiler derived from an interpreter containing the above code +generates the following machine code (in pseudo-assembler notation), +assuming that ``a`` comes from register ``r1``: + +\begin{verbatim} + ... + r2 = div r1, 10 + Label1: + jump Label2 + + + Label2: + call continue_compilation(r2, ) + jump Label1 +\end{verbatim} + +The first time this machine code runs, the ``continue\_compilation()`` +function resumes the compiler. The two arguments to the function are +the actual run-time value from the register ``r2``, which the compiler +will now consider as a compile-time constant, and an immediate pointer +to data that was generated along with the above code snippet and which +contains enough information for the compiler to know where and with +which state it should resume. + +Assuming that the first run-time value taken by ``r1`` is, say, 42, then +the compiler will see ``r2 == 4`` and update the above machine code as +follows: + +\begin{verbatim} + ... + r2 = div r1, 10 + Label1: + compare r2, 4 # patched + jump-if-equal Label3 # patched + jump Label2 # patched + + + Label2: + call continue_compilation(r2, ) + jump Label1 + + Label3: # new code + call print(9) # new code + ... +\end{verbatim} + +Notice how the addition is constant-folded by the compiler. (Of course, +in real examples, different promoted values typically make the compiler +constant-fold complex code path choices in different ways, and not just +simple operations.) Note also how the code following ``Label1`` is an +updatable switch which plays the role of a polymorphic inline cache. +The "polymorphic" terminology does not apply in our context, though, as +the switch does not necessarily have to be on the type of an object. + +After the update, the original call to ``continue\_compilation()`` +returns and execution loops back to the now-patched switch at +``Label1``. This run and all following runs in which ``r1`` is between +40 and 49 will thus directly go to ``Label3``. Obviously, if other +values show up, ``continue\_compilation()`` will be invoked again, so new +code will be generated and the code at ``Label1`` further patched to +check for more cases. + +If, over the course of the execution of a program, too many cases are +seen, the reserved space after ``Label1`` will eventually run out. +Currently, we simply reserve more space elsewhere and patch the final +jump accordingly. There could be better strategies which which we did +not implement so far, such as discarding old code and reusing their slots +in the switch, or sometimes giving up entirely and compiling a general +version of the code in which the value remains run-time. + + +\subsubsection*{Implementation notes} + +The *state data pointer* in the example above contains a snapshot of the +state of the compiler when it reached the promotion point. Its memory +impact is potentially large -- a complete continuation for each generated +switch, which can never be reclaimed because new run-time values may +always show up later during the execution of the program. + +To reduce the problem we compress the state into a so-called *path*. +The full state is only stored at a few specific points.\footnote{ + More precisely, at merge points that the user needs to mark + as "global". The control flow join point corresponding to the + looping of the interpreter main loop is a typical place to put + such a global merge point. +} +The compiler +records a trace of the multiple paths it followed from the last full +snapshot in a lightweight tree structure. The *state data pointer* is +then only a pointer to a node in the tree; the branch from that node to +the root describes a path that let the compiler quickly *replay* its +actions (without generating code again) from the latest full snapshot to +rebuild its internal state and get back to the original promotion point. + +For example, if the interpreter source code contains promotions inside a +run-time condition: + +\begin{verbatim} + if condition: + ... + hint(x, promote=True) + ... + else: + ... + hint(y, promote=True) + ... +\end{verbatim} + +then the tree will contain three nodes: a root node storing the +snapshot, a child with a "True case" marker, and another child with a +"False case" marker. Each promotion point generates a switch and a call +to ``continue\_compilation()`` pointing to the appropriate child node. +The compiler can re-reach the correct promotion point by following the +markers on the branch from the root to the child. + + +\subsection{Virtual structures} +\label{virtual} + +Interpreters for dynamic languages typically allocate a lot of small +objects, for example due to boxing. For this reason, we +implemented a way for the compiler to generate residual memory +allocations as lazily as possible. The idea is to try to keep new +run-time structures "exploded": instead of a single run-time pointer to +a heap-allocated data structure, the structure is "virtualized" as a set +of fresh variables, one per field. In the compiler, the variable that +would normally contain the pointer to the structure gets instead a +content that is neither a run-time value nor a compile-time constant, +but a special *virtual structure* -- a compile-time data structure that +recursively contains new variables, each of which can again store a +run-time, a compile-time, or a virtual structure value. + +This approach is based on the fact that the "run-time values" carried +around by the compiler really represent run-time locations -- the name of +a CPU register or a position in the machine stack frame. This is the +case for both regular variables and the fields of virtual structures. +It means that the compilation of a ``getfield`` or ``setfield`` +operation performed on a virtual structure simply loads or stores such a +location reference into the virtual structure; the actual value is not +copied around at run-time. + +It is not always possible to keep structures virtual. The main +situation in which it needs to be "forced" (i.e. actually allocated at +run-time) is when the pointer escapes to some non-virtual location like +a field of a real heap structure. + +Virtual structures still avoid the run-time allocation of most +short-lived objects, even in non-trivial situations. The following +example shows a typical case. Consider the Python expression ``a+b+c``. +Assume that ``a`` contains an integer. The PyPy Python interpreter +implements application-level integers as boxes -- instances of a +``W\_IntObject`` class with a single ``intval`` field. Here is the +addition of two integers: + +\begin{verbatim} + def add(w1, w2): # w1, w2 are W_IntObject instances + value1 = w1.intval + value2 = w2.intval + result = value1 + value2 + return W_IntObject(result) +\end{verbatim} + +When interpreting the bytecode for ``a+b+c``, two calls to ``add()`` are +issued; the intermediate ``W\_IntObject`` instance is built by the first +call and thrown away after the second call. By contrast, when the +interpreter is turned into a compiler, the construction of the +``W\_IntObject`` object leads to a virtual structure whose ``intval`` +field directly references the register in which the run-time addition +put its result. This location is read out of the virtual structure at +the beginning of the second ``add()``, and the second run-time addition +directly operates on the same register. + +An interesting effect of virtual structures is that they play nicely with +promotion. Indeed, before the interpreter can call the proper ``add()`` +function for integers, it must first determine that the two arguments +are indeed integer objects. In the corresponding dispatch logic, we +have added two hints to promote the type of each of the two arguments. +This produces a compiler that has the following behavior: in the general +case, the expression ``a+b`` will generate two consecutive run-time +switches followed by the residual code of the proper version of +``add()``. However, in ``a+b+c``, the virtual structure representing +the intermediate value will contain a compile-time constant as type. +Promoting a compile-time constant is trivial -- no run-time code is +generated. The whole expression ``a+b+c`` thus only requires three +switches instead of four. It is easy to see that even more switches can +be skipped in larger examples; typically, in a tight loop manipulating +only integers, all objects are virtual structures for the compiler and +the residual code is theoretically optimal -- all type propagation and +boxing/unboxing occurs at compile-time. + + +\subsection{Virtualizable structures} +\label{virtualizable} + +In the PyPy interpreter there are cases where structures cannot be +virtual -- because they escape, or are allocated outside the +JIT-generated code -- but where we would still like to keep the +"exploding" effect and carry the fields of the structure as local +variables in the generated code. + +It is likely that the same problem occurs more generally in many +interpreters: the typical example is that of frame objects, which stores +among other things the value of the local variables of each function +invocation. Ideally, the effect we would like to achieve is to keep the +frame object as a purely virtual structure, and the same for the array +or dictionary implementing the bindings of the locals. Then each local +variable of the interpreted language can be represented as a separate +run-time value in the generated code, or be itself further virtualized +(e.g. as a virtual ``W\_IntObject`` structure as seen above). + +The issue is that the frame object is sometimes built in advance by +non-JIT-generated code; even when it is not, it immediately escapes into +the global list of frames that is used to support the frame stack +introspection primitives that Python exposes. In other words, the frame +object cannot be purely virtual because a pointer to it must be stored +into a global data structure (even though in practice most of frame +objects are deallocated without ever having been introspected). + +To solve this problem, we introduced *virtualizable structures,* a mix +between regular run-time structures and virtual structures. A virtualizable structure is a +structure that exists at run-time in the heap, but that is +simultaneously treated as virtual by the compiler. Accesses to the +structure from the code generated by the JIT are virtualized away, +i.e. don't involve run-time copying. The trade-off is that in order +to keep both views synchronized, accesses to the run-time structure +from regular code not produced by the JIT needs to perform an extra +check. + +Because of this trade-off, a hint needs to be inserted manually to mark +the classes whose instances should be implemented in this way -- the +class of frame objects, in the case of PyPy. The hint is used by the +translation toolchain to add a hidden field to all frame objects, and to +translate all accesses to the object fields into low-level code that +first checks the hidden field. This is the only case so far in which +the presence of the JIT compiler imposes a global change to the rest of +the program during translation.\footnote{ + This is not a problem per se, as it is anyway just a small + extension to the translation framework, but it imposes a performance + overhead to all code manipulating frame objects. To mitigate this, we + added a way to declare during RPython type inference that the + indirection check is not needed in some parts of the code where we know + that the frame object cannot have a virtual counterpart. +} + +The hidden field is set when the frame structure enters JIT-generated +code, and cleared when it leaves. When a recursive call to +non-JIT-generated code finds a structure with the field set, it invokes +a JIT-generated callback to perform the reading or updating of the field +from the point of view of its virtual structure representation. The +actual fields in the heap structure are not used during this time. + +The effect that can be obtained in this way is that although frame +objects are still allocated in the heap, most of them will always remain +essentially empty. A pointer to these empty frames is pushed into and +popped off the global frame list, allowing the introspection mechanisms +to still work perfectly. + + +\subsection{Other implementation details} + +We quickly mention below a few other features and implementation details +of the implementation of the JIT generation framework. More information +can be found in the on-line documentation. + +* There are more user-specified hints available, like *deep-freezing,* + which marks an object as immutable in order to allow accesses to + its content to be constant-folded at compile-time. + +* The compiler representation of a run-time value for a non-virtual + structure may additionally remember that some fields are actually + compile-time constants. This occurs for example when a field is + read from the structure at run-time and then promoted to compile-time. + +* In addition to virtual structures, lists and dictionaries can also be + virtual. + +* Exception handling is achieved by inserting explicit operations into + the graphs before they are timeshifted. Most of these run-time + exception manipulations are then virtualized away, by treating the + exception state as virtual. + +* Timeshifting is performed in two phases: a first step transforms the + graphs by updating their control flow and inserting pseudo-operations + to drive the compiler; a second step (based on the RTyper \cite{D05.1}) + replaces all necessary operations by calls to support code. + +* The support code implements the generic behaviour of the compiler, + e.g. the merge logic. It is about 3500 lines of RPython code. The + rest of the hint-annotator and timeshifter is about 3800 lines of + Python code. + +* The machine code backends (two so far, Intel IA32 and PowerPC) are + about 3500 further lines of RPython code each. There is a + well-defined interface between the JIT compiler support code and the + backends, making writing new backends relatively easy. The unusual + part of the interface is the support for the run-time updatable + switches. + + +\subsection{Open issues} + +Here are what we think are the most important points that will need +attention in order to make the approach more robust: + +* The timeshifted graphs currently compile many branches eagerly. This + can easily result in residual code explosion. Depending on the source + interpreter this can also result in non-termination issues, where + compilation never completes. The opposite extreme would be to always + compile branches lazily, when they are about to be executed, as Psyco + does. While this neatly sidesteps termination issues, the best + solution is probably something in between these extremes. + +* As described in the Promotion section (\ref{promotion}), + we need fall-back solutions for when the + number of promoted run-time values seen at a particular point becomes + too large. + +* We need more flexible control about what to inline or not to inline in + the residual code. + +* The widening heuristics for merging needs to be refined. + +* The JIT generation framework needs to be made aware of some other + translation-time aspects \cite{D05.4} \cite{D07.1} in order to produce the + correct residual code (e.g. code calling the correct Garbage + Collection routines or supporting Stackless-style stack unwinding). + +* We did not work yet on profile-directed identification of program hot + spots. Currently, the interpreter must decide when to invoke the JIT + or not (which can itself be based on explicit requests from the interpreted + program). + +* The machine code backends can be improved. + +The latter point opens an interesting future research direction: can we +layer our kind of JIT compiler on top of a virtual machine that already +contains a lower-level JIT compiler? In other words, can we delegate +the difficult questions of machine code generation to a lower +independent layer, e.g. inlining, re-optimization of frequently executed +code, etc.? What changes would be required to an existing virtual +machine, e.g. a Java Virtual Machine, to support this? + + +\section{Results} + +The following test function is an example of purely arithmetic code +written in Python, which the PyPy JIT can run extremely fast: + +\begin{verbatim} + def f1(n): + "Arbitrary test function." + i = 0 + x = 1 + while i Author: arigo Date: Tue May 29 09:40:36 2007 New Revision: 43835 Added: pypy/extradoc/talk/dls2007/Makefile (contents, props changed) Modified: pypy/extradoc/talk/dls2007/paper.tex Log: More details. Added: pypy/extradoc/talk/dls2007/Makefile ============================================================================== --- (empty file) +++ pypy/extradoc/talk/dls2007/Makefile Tue May 29 09:40:36 2007 @@ -0,0 +1,9 @@ + +paper.pdf: paper.tex #paper.bib image/*.pdf + #pdflatex paper + #bibtex paper + pdflatex paper + pdflatex paper + +view: paper.pdf + xpdf $< & Modified: pypy/extradoc/talk/dls2007/paper.tex ============================================================================== --- pypy/extradoc/talk/dls2007/paper.tex (original) +++ pypy/extradoc/talk/dls2007/paper.tex Tue May 29 09:40:36 2007 @@ -2,7 +2,7 @@ \begin{document} -\title{Generating Just-In-Time Specializing Compilers} +\title{Get Your Own Just-In-Time Specializing Compiler For Free} \numberofauthors{2} \author{ @@ -40,6 +40,8 @@ run-time values to compile-time is used to that effect. In this report, we describe it along with other novel techniques that allow the approach to scale to something as large as PyPy's Python interpreter. +\footnote{This research was partially supported by the EU funded + project: IST 004779 PyPy (PyPy: Implementing Python in Python).} \end{abstract} \section{Introduction} @@ -929,17 +931,6 @@ an Intel-based Mac OS/X machine we have measured this ratio to be as low as 1.15x. -.. comment - - python 2.4: 0.82 - pypy-c-42412-allworking-tproxy: 1.90392203331 - pypy-c-41802-faassen: 1.60267777443 - pypy-c-41992-jit, turned off: 1.76742062569 - pypy-c-41992-jit: 0.00912307977676 <=== - psyco 1.5.2: 0.00621596813202 - gcc -O0: 0.0061992 - gcc -O2: 0.0021689 - \section{Conclusion} @@ -1012,4 +1003,9 @@ % conference on Object-oriented programming languages, systems, and % applications, pp. 944-953, ACM Press, 2006 +\bigskip + +\bibliographystyle{abbrv} +\bibliography{paper} + \end{document} From arigo at codespeak.net Tue May 29 15:31:42 2007 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 29 May 2007 15:31:42 +0200 (CEST) Subject: [pypy-svn] r43848 - pypy/extradoc/talk/dls2007 Message-ID: <20070529133142.25B098080@code0.codespeak.net> Author: arigo Date: Tue May 29 15:31:41 2007 New Revision: 43848 Added: pypy/extradoc/talk/dls2007/paper.bib - copied, changed from r43833, pypy/extradoc/talk/dyla2007/dyla.bib Modified: pypy/extradoc/talk/dls2007/Makefile pypy/extradoc/talk/dls2007/paper.tex Log: Finish LaTeXification, complete things a bit. Modified: pypy/extradoc/talk/dls2007/Makefile ============================================================================== --- pypy/extradoc/talk/dls2007/Makefile (original) +++ pypy/extradoc/talk/dls2007/Makefile Tue May 29 15:31:41 2007 @@ -1,7 +1,7 @@ -paper.pdf: paper.tex #paper.bib image/*.pdf - #pdflatex paper - #bibtex paper +paper.pdf: paper.tex paper.bib + pdflatex paper + bibtex paper pdflatex paper pdflatex paper Modified: pypy/extradoc/talk/dls2007/paper.tex ============================================================================== --- pypy/extradoc/talk/dls2007/paper.tex (original) +++ pypy/extradoc/talk/dls2007/paper.tex Tue May 29 15:31:41 2007 @@ -52,21 +52,36 @@ experimented with \cite{REJIT}, but this is clearly an area in need of research and innovative approaches. -One of the central goals of the PyPy project is to automatically +One of the central goals of the PyPy project \cite{PyPy} is to automatically produce dynamic compilers from an interpreter, with as little modifications of the interpreter code base itself as possible. -The forest of flow graphs that the translation process \cite{VMCDLS} -generates and transforms constitutes a reasonable base for the -necessary analyses. That's a further reason why having a high-level -runnable and analyzable interpreter implementation was always a -central tenet of the project: in our approach, -the dynamic compiler is just another aspect -transparently introduced by and during the translation -process. +PyPy contains a complete interpreter for the Python language, written in +a high-level language, RPython, which is a subset of Python amenable to +static analysis. It also contains a translation toolchain for compiling +this interpreter to either C (or C-like) environments, or to the higher +level environments provided by general-purpose virtual machines like +Java's and .NET. The translation toolchain can input any RPython +program, although our focus was on the translation of RPython programs +that are interpreters for dynamic languages.\footnote{We also have an +interpreter for Prolog and the beginning of one for JavaScript.} + +The translation framework uses control flow graphs in SSI format as its +intermediate representation (SSI is a stricter subset of SSA). The +details of this process are beyond the scope of the present paper, and +have been presented in \cite{pypyvmconstruction}. +The present paper describes a +special optional transformation that we integrated with this translation +framework: deriving a dynamic compiler from the interpreter. In other +words, our translation framework is able to input an interpreter for any +language (it works best for dynamic languages); as long as it is +written in RPython and contains a small number of extra hints, then it +can produce from it a complete virtual machine +\emph{that contains a just-in-time compiler for the dynamic language.} Partial evaluation techniques should, at least theoretically, -allow such a derivation of a compiler from an interpreter [PE], but it +allow such a derivation of a compiler from an interpreter +\cite{partial-evaluation}, but it is not reasonable to expect the code produced for an input program by a compiler derived using partial evaluation to be very good, especially in the case of a dynamic language. Essentially, the input @@ -83,9 +98,9 @@ This will allows the compiler to generate code optimized for the effective run-time behaviour of the program. -Inspired by Psyco \cite{PSYCO}, which is a hand-written dynamic compiler +Inspired by Psyco \cite{psyco-paper}, which is a hand-written dynamic compiler based on partial evaluation for Python, we developed a technique - -*promotion* - for our dynamic compiler generator. Simply put, promotion +\emph{promotion} - for our dynamic compiler generator. Simply put, promotion on a value stops compilation and waits until the run-time reaches this point. When it does, the actual run-time value is promoted into a compile-time constant, and compilation resumes with this extra @@ -102,98 +117,113 @@ \subsection{Overview of partial evaluation} -Partial evaluation is the process of evaluating a function, say ``f(x, -y)``, with only partial information about the values of its arguments, -say the value of the ``x`` argument only. This produces a *residual* -function ``g(y)``, which takes less arguments than the original -- only +\def\code#1{\texttt{#1}} + +Partial evaluation is the process of evaluating a function, say \code{f(x, +y)}, with only partial information about the values of its arguments, +say the value of the \code{x} argument only. This produces a \emph{residual} +function \code{g(y)}, which takes less arguments than the original -- only the information not specified during the partial evaluation process needs -to be provided to the residual function, in this example the ``y`` +to be provided to the residual function, in this example the \code{y} argument. Partial evaluation (PE) comes in two flavors: +% +\begin{enumerate} -* *On-line* PE: a compiler-like algorithm takes the source code of the - function ``f(x, y)`` (or its intermediate representation, i.e. its +\item\emph{On-line PE:} a compiler-like algorithm takes the source code of the + function \code{f(x, y)} (or its intermediate representation, i.e.\ its control flow graph in PyPy's terminology), and some partial - information, e.g. ``x=5``. From this, it produces the residual - function ``g(y)`` directly, by following in which operations the - knowledge ``x=5`` can be used, which loops can be unrolled, etc. + information, e.g.\ \code{x=5}. From this, it produces the residual + function \code{g(y)} directly, by following in which operations the + knowledge \code{x=5} can be used, which loops can be unrolled, etc. -* *Off-line* PE: in many cases, the goal of partial evaluation is to +\item\emph{Off-line PE:} in many cases, the goal of partial evaluation is to improve performance in a specific application. Assume that we have a - single known function ``f(x, y)`` in which we think that the value of - ``x`` will change slowly during the execution of our program -- much - more slowly than the value of ``y``. An obvious example is a loop - that calls ``f(x, y)`` many times with always the same value ``x``. - We could then use an on-line partial evaluator to produce a ``g(y)`` - for each new value of ``x``. In practice, the overhead of the partial + single known function \code{f(x, y)} in which we think that the value of + \code{x} will change slowly during the execution of our program -- much + more slowly than the value of \code{y}. An obvious example is a loop + that calls \code{f(x, y)} many times with always the same value \code{x}. + We could then use an on-line partial evaluator to produce a \code{g(y)} + for each new value of \code{x}. In practice, the overhead of the partial evaluator might be too large for it to be executed at run-time. - However, if we know the function ``f`` in advance, and if we know - *which* arguments are the ones that we will want to partially evaluate - ``f`` with, then we do not need a full compiler-like analysis of ``f`` - every time the value of ``x`` changes. We can precompute once and for - all a specialized function ``f1(x)``, which when called produces the - residual function ``g(y)`` corresponding to ``x``. This is *off-line - partial evaluation;* the specialized function ``f1(x)`` is called a - *generating extension*. + However, if we know the function \code{f} in advance, and if we know + \emph{which} arguments are the ones that we will want to partially evaluate + \code{f} with, then we do not need a full compiler-like analysis of \code{f} + every time the value of \code{x} changes. We can precompute once and for + all a specialized function \code{f1(x)}, which when called produces the + residual function \code{g(y)} corresponding to \code{x}. This is + \emph{off-line partial evaluation;} the specialized function \code{f1(x)} + is called a \emph{generating extension.} + +\end{enumerate} The PyPy JIT generation framework is based on off-line partial -evaluation. The function called ``f(x, y)`` above is typically the main +evaluation. The function called \code{f(x, y)} above is typically the main loop of some interpreter written in RPython. The size of the interpreter can range from a three-liner used for testing purposes to the whole of PyPy's -Python interpreter. In all cases, ``x`` stands for the input program -(the bytecode to interpret) and ``y`` stands for the input data (like a +Python interpreter. In all cases, \code{x} stands for the input program +(the bytecode to interpret) and \code{y} stands for the input data (like a frame object with the binding of the input arguments and local variables). Our framework is capable of automatically producing the -corresponding generating extension ``f1(x)``, which takes an input -program only and produces a residual function ``g(y)``. This ``f1(x)`` +corresponding generating extension \code{f1(x)}, which takes an input +program only and produces a residual function \code{g(y)}. This \code{f1(x)} is a compiler\footnote{ What we get in PyPy is more precisely a \emph{just-in-time compiler:} if promotion is used, compiling ahead of time is not possible. } -for the very same language for which ``f(x, y)`` is +for the very same language for which \code{f(x, y)} is an interpreter. -Off-line partial evaluation is based on *binding-time analysis,* which +Off-line partial evaluation is based on \emph{binding-time analysis,} which is the process of determining among the variables used in a function (or a set of functions) which ones are going to be known in advance and -which ones are not. In the example of ``f(x, y)``, such an analysis -would be able to infer that the constantness of the argument ``x`` +which ones are not. In the example of \code{f(x, y)}, such an analysis +would be able to infer that the constantness of the argument \code{x} implies the constantness of many intermediate values used in the -function. The *binding time* of a variable determines how early the +function. The \emph{binding time} of a variable determines how early the value of the variable will be known. Once binding times have been determined, one possible approach to producing the generating extension itself is by self-applying on-line partial evaluators. This is known as the second Futamura projection -\cite{FU}. So far it is unclear if this approach can lead to optimal +\cite{Futamura}. So far it is unclear if this approach can lead to optimal results, or even if it scales well. In PyPy we selected a more direct approach: the generating extension is produced by transformation of the control flow graphs of the interpreter, guided by the binding times. We -call this process *timeshifting*. +call this process \emph{timeshifting.} + + +\subsection{Related work} + +XXX PE; Psyco; REJIT; ? \section{Architecture and Principles} PyPy contains a framework for generating just-in-time compilers using off-line partial evaluation. As such, there are three distinct phases: +% +\begin{enumerate} -* *Translation time:* during the normal translation of an RPython +\item\emph{Translation time:} during the normal translation of an RPython program, say PyPy's Python interpreter, we perform binding-time analysis and off-line specialization ("timeshifting") of the interpreter. This produces a generating extension, which is linked with the rest of the program. -* *Compile time:* during the execution of the program, when a new +\item\emph{Compile time:} during the execution of the program, when a new bytecode is about to be interpreted, the generating extension is invoked instead. As the generating extension is a compiler, all the computations it performs are called compile-time computations. Its sole effect is to produce residual code. -* *Run time:* the normal execution of the program (which includes the +\item\emph{Run time:} the normal execution of the program (which includes the time spent running the residual code created by the generating extension). +\end{enumerate} + Translation time is a purely off-line phase; compile time and run time are actually highly interleaved during the execution of the program. @@ -202,33 +232,37 @@ \label{bta} At translation time, PyPy performs binding-time analysis of the source -RPython program after it has been turned to low-level graphs, i.e. at +RPython program after it has been turned to low-level graphs, i.e.\ at the level at which operations manipulate pointer-and-structure-like objects. The binding-time terminology that we are using in PyPy is based on the colors that we use when displaying the control flow graphs: - -* *Green* variables contain values that are known at compile-time; -* *Red* variables contain values that are not known until run-time. +% +\begin{itemize} +\item\emph{Green} variables contain values that are known at compile-time; +\item\emph{Red} variables contain values that are not known until run-time. +\end{itemize} The binding-time analyzer of our translation tool-chain is based on the same type inference engine that is used on the source RPython program, -the annotator. In this mode, it is called the *hint-annotator*; it +the annotator. In this mode, it is called the \emph{hint-annotator;} it operates over input graphs that are already low-level instead of RPython-level, and propagates annotations that do not track types but value dependencies and manually-provided binding time hints. The normal process of the hint-annotator is to propagate the binding -time (i.e. color) of the variables using the following kind of rules: +time (i.e.\ color) of the variables using the following kind of rules: +% +\begin{itemize} -* For a foldable operation (i.e. one without side effect and which +\item For a foldable operation (i.e.\ one without side effect and which depends only on its argument values), if all arguments are green, then the result can be green too. -* Non-foldable operations always produce a red result. +\item Non-foldable operations always produce a red result. -* At join points, where multiple possible values (depending on control +\item At join points, where multiple possible values (depending on control flow) are meeting into a fresh variable, if any incoming value comes from a red variable, the result is red. Otherwise, the color of the result might be green. We do not make it eagerly green, because of @@ -238,6 +272,8 @@ fresh join variable thus depends on which branches are taken in the residual graph. +\end{itemize} + \subsubsection*{Hints} Our goal in designing our approach to binding-time analysis was to @@ -248,24 +284,25 @@ The driving idea was that hints should be need-oriented. Indeed, in a program like an interpreter, there are a small number of places where it would be clearly beneficial for a given value to be known at -compile-time, i.e. green: this is where we require the hints to be +compile-time, i.e.\ green: this is where we require the hints to be added. The hint-annotator assumes that all variables are red by default, and then propagates annotations that record dependency information. When encountering the user-provided hints, the dependency information is used to make some variables green. All -hints are in the form of an operation ``hint(v1, someflag=True)`` +hints are in the form of an operation \code{hint(v1, someflag=True)} which semantically just returns its first argument unmodified. -The crucial need-oriented hint is ``v2 = hint(v1, concrete=True)`` +The crucial need-oriented hint is +$$\code{v2 = hint(v1, concrete=True)}$$ which should be used in places where the programmer considers the knowledge of the value to be essential. This hint is interpreted by -the hint-annotator as a request for both ``v1`` and ``v2`` to be green. It -has a *global* effect on the binding times: it means that not only -``v1`` but all the values that ``v1`` depends on -- recursively -- +the hint-annotator as a request for both \code{v1} and \code{v2} to be green. It +has a \emph{global} effect on the binding times: it means that not only +\code{v1} but all the values that \code{v1} depends on -- recursively -- are forced to be green. The hint-annotator complains if the -dependencies of ``v1`` include a value that cannot be green, like +dependencies of \code{v1} include a value that cannot be green, like a value read out of a field of a non-immutable structure. Such a need-oriented backward propagation has advantages over the @@ -276,22 +313,23 @@ of the residual code), or less variables than expected (preventing specialization to occur where it would be the most useful). Our need-oriented approach reduces the problem of over-specialization, and -it prevents under-specialization: an unsatisfiable ``hint(v1, -concrete=True)`` is reported as an error. +it prevents under-specialization: an unsatisfiable \code{hint(v1, +concrete=True)} is reported as an error. In our context, though, such an error can be corrected. This is done by -promoting a well-chosen variable among the ones that ``v1`` depends on. +promoting a well-chosen variable among the ones that \code{v1} depends on. Promotion is invoked with the use of a hint as well: -``v2 = hint(v1, promote=True)``. -This hint is a *local* request for ``v2`` to be green, without -requiring ``v1`` to be green. Note that this amounts to copying +\code{v2 = hint(v1, promote=True)}. +This hint is a \emph{local} request for \code{v2} to be green, without +requiring \code{v1} to be green. Note that this amounts to copying a red value into a green one, which is not possible in classical approaches to partial evaluation. See section \ref{promotion} for a complete discussion of promotion. For examples and further discussion on how the hints are applied in practice -see `Make your own JIT compiler` \cite{D08.1}. +see \emph{Make your own JIT compiler} at +\code{http://codespeak.net/pypy/dist/pypy/doc/jit.html}. % XXX check url \subsection{Timeshifting} @@ -307,7 +345,7 @@ cannot be expressed as low-level flow graphs). } accordingly in order to produce a generating extension. We call -this process *timeshifting* because it changes the time at +this process \emph{timeshifting} because it changes the time at which the graphs are meant to be run, from run-time to compile-time. Despite the execution time and side-effects shift to produce only @@ -330,32 +368,40 @@ The basic idea of timeshifting is to transform operations in a way that depends on the color of their operands and result. Variables themselves need to be represented based on their color: +% +\begin{itemize} -* The red (run-time) variables have abstract values at compile-time; +\item The red (run-time) variables have abstract values at compile-time; no actual value is available for them during compile-time. For them we use a boxed representation that can carry either a run-time storage location (a stack frame position or a register name) or an immediate constant (for when the value is, after all, known at compile-time). -* On the other hand, the green variables are the ones that can carry +\item On the other hand, the green variables are the ones that can carry their value already at compile-time, so they are left untouched during timeshifting. +\end{itemize} + The operations of the original graphs are then transformed as follows: +% +\begin{itemize} -* If an operation has no side effect nor any other run-time dependency, and +\item If an operation has no side effect nor any other run-time dependency, and if it only involves green operands, then it can stay unmodified in the graph. In this case, the operation that was run-time in the original graph becomes a compile-time operation, and it will never be generated in the residual code. (This is the case that makes the whole approach worthwhile: some operations become purely compile-time.) -* In all other cases, the operation might have to be generated in the +\item In all other cases, the operation might have to be generated in the residual code. In the timeshifted graph it is replaced by a call to a helper which will generate a residual operation manipulating the input run-time values and return a new boxed representation for the run-time result location. +\end{itemize} + These helpers will constant-fold the operation if the inputs are immediate constants and if the operation has no side-effects. Immediate constants can occur even though the corresponding variable in the graph was red: a variable can be @@ -363,7 +409,7 @@ point in (compile)-time, independently of the hint-annotator proving that it is always the case. In Partial Evaluation terminology, the timeshifted graphs are -performing some *on-line* partial evaluation in addition to the +performing some \emph{on-line} partial evaluation in addition to the off-line job enabled by the hint-annotator. \subsubsection*{Merges and Splits} @@ -373,16 +419,16 @@ This state is used to shape the control flow of the generated residual code, as follows. -After a *split,* i.e. after a conditional branch that could not be +After a \emph{split,} i.e.\ after a conditional branch that could not be folded at compile-time, the compilation state is duplicated and both -branches are compiled independently. Conversely, after a *merge point,* -i.e. when two control flow paths meet each other, we try to join the two +branches are compiled independently. Conversely, after a \emph{merge point,} +i.e.\ when two control flow paths meet each other, we try to join the two paths in the residual code. This part is more difficult because the two -paths may need to be compiled with different variable bindings -- e.g. -different variables may be known to take different compile-time constant +paths may need to be compiled with different variable bindings -- +e.g.\ different variables may be known to take different compile-time constant values in the two branches. The two paths can either be kept separate or merged; in the latter case, the merged compilation-time state needs -to be a generalization (*widening*) of the two already-seen states. +to be a generalization \emph{(widening)} of the two already-seen states. Deciding when to do each is a classical problem of partial evaluation, as merging too eagerly may loose important precision and not merging eagerly enough may create too many redundant residual code paths (to the @@ -414,7 +460,7 @@ \label{promotion} In the sequel, we describe in more details one of the main new -techniques introduced in our approach, which we call *promotion*. In +techniques introduced in our approach, which we call \emph{promotion.} In short, it allows an arbitrary run-time value to be turned into a compile-time value at any point in time. Each promotion point is explicitly defined with a hint that must be put in the source code of @@ -425,7 +471,7 @@ copying a variable whose binding time is compile-time into a variable whose binding time is run-time -- it corresponds to the compiler "forgetting" a particular value that it knew about. By contrast, -promotion is a way for the compiler to gain *more* information about +promotion is a way for the compiler to gain \emph{more} information about the run-time execution of a program. Clearly, this requires fine-grained feedback from run-time to compile-time, thus a dynamic setting. @@ -457,7 +503,8 @@ techniques are crucial for good results. The main goal is to optimize and reduce the overhead of dynamic dispatching and indirect invocation. This is achieved with variations on the technique of -polymorphic inline caches \cite{PIC}: the dynamic lookups are cached and +polymorphic inline caches \cite{polymorphic-inline-caches}: +the dynamic lookups are cached and the corresponding generated machine code contains chains of compare-and-jump instructions which are modified at run-time. These techniques also allow the gathering of information to direct inlining for even @@ -472,7 +519,7 @@ promoted to compile-time. As we will see in the sequel, this produces very similar machine code.\footnote{ This can also be seen as a generalization of a partial - evaluation transformation called "The Trick" (see e.g. \cite{PE}), + evaluation transformation called "The Trick" (see e.g.\ \cite{partial-evaluation}), which again produces similar code but which is only applicable for finite sets of values. } @@ -486,7 +533,7 @@ \subsubsection*{Promotion in practice} The implementation of promotion requires a tight coupling between -compile-time and run-time: a *callback,* put in the generated code, +compile-time and run-time: a \emph{callback,} put in the generated code, which can invoke the compiler again. When the callback is actually reached at run-time, and only then, the compiler resumes and uses the knowledge of the actual run-time value to generate more code. @@ -499,85 +546,86 @@ While this describes the general idea, the details are open to slight variations. Let us show more precisely the way the JIT compilers produced by PyPy 1.0 work. Our first example is purely artificial: - +% \begin{verbatim} - ... - b = a / 10 - c = hint(b, promote=True) - d = c + 5 - print d - ... + ... + b = a / 10 + c = hint(b, promote=True) + d = c + 5 + print d + ... \end{verbatim} -In this example, ``a`` and ``b`` are run-time variables and ``c`` and -``d`` are compile-time variables; ``b`` is copied into ``c`` via a +In this example, \code{a} and \code{b} are run-time variables and \code{c} and +\code{d} are compile-time variables; \code{b} is copied into \code{c} via a promotion. The division is a run-time operation while the addition is a compile-time operation. The compiler derived from an interpreter containing the above code generates the following machine code (in pseudo-assembler notation), -assuming that ``a`` comes from register ``r1``: - +assuming that \code{a} comes from register \code{r1}: +% \begin{verbatim} - ... - r2 = div r1, 10 - Label1: - jump Label2 - - - Label2: - call continue_compilation(r2, ) - jump Label1 + ... + r2 = div r1, 10 + Label1: + jump Label2 + + + Label2: + call continue_compilation(r2, ) + jump Label1 \end{verbatim} -The first time this machine code runs, the ``continue\_compilation()`` -function resumes the compiler. The two arguments to the function are -the actual run-time value from the register ``r2``, which the compiler +The first time this machine code runs, the function called +\code{continue\_compilation()} +resumes the compiler. The two arguments to the function are +the actual run-time value from the register \code{r2}, which the compiler will now consider as a compile-time constant, and an immediate pointer to data that was generated along with the above code snippet and which contains enough information for the compiler to know where and with which state it should resume. -Assuming that the first run-time value taken by ``r1`` is, say, 42, then -the compiler will see ``r2 == 4`` and update the above machine code as +Assuming that the first run-time value taken by \code{r1} is, say, 42, then +the compiler will see \code{r2 == 4} and update the above machine code as follows: - +% \begin{verbatim} - ... - r2 = div r1, 10 - Label1: - compare r2, 4 # patched - jump-if-equal Label3 # patched - jump Label2 # patched - - - Label2: - call continue_compilation(r2, ) - jump Label1 - - Label3: # new code - call print(9) # new code - ... + ... + r2 = div r1, 10 + Label1: + compare r2, 4 # patched + jump-if-equal Label3 # patched + jump Label2 # patched + + + Label2: + call continue_compilation(r2, ) + jump Label1 + + Label3: # new code + call print(9) # new code + ... \end{verbatim} Notice how the addition is constant-folded by the compiler. (Of course, in real examples, different promoted values typically make the compiler constant-fold complex code path choices in different ways, and not just -simple operations.) Note also how the code following ``Label1`` is an +simple operations.) Note also how the code following \code{Label1} is an updatable switch which plays the role of a polymorphic inline cache. The "polymorphic" terminology does not apply in our context, though, as the switch does not necessarily have to be on the type of an object. -After the update, the original call to ``continue\_compilation()`` +After the update, the original call to \code{continue\_compilation()} returns and execution loops back to the now-patched switch at -``Label1``. This run and all following runs in which ``r1`` is between -40 and 49 will thus directly go to ``Label3``. Obviously, if other -values show up, ``continue\_compilation()`` will be invoked again, so new -code will be generated and the code at ``Label1`` further patched to +\code{Label1}. This run and all following runs in which \code{r1} is between +40 and 49 will thus directly go to \code{Label3}. Obviously, if other +values show up, \code{continue\_compilation()} will be invoked again, so new +code will be generated and the code at \code{Label1} further patched to check for more cases. If, over the course of the execution of a program, too many cases are -seen, the reserved space after ``Label1`` will eventually run out. +seen, the reserved space after \code{Label1} will eventually run out. Currently, we simply reserve more space elsewhere and patch the final jump accordingly. There could be better strategies which which we did not implement so far, such as discarding old code and reusing their slots @@ -587,13 +635,13 @@ \subsubsection*{Implementation notes} -The *state data pointer* in the example above contains a snapshot of the +The state data pointer in the example above contains a snapshot of the state of the compiler when it reached the promotion point. Its memory impact is potentially large -- a complete continuation for each generated switch, which can never be reclaimed because new run-time values may always show up later during the execution of the program. -To reduce the problem we compress the state into a so-called *path*. +To reduce the problem we compress the state into a so-called \emph{path.} The full state is only stored at a few specific points.\footnote{ More precisely, at merge points that the user needs to mark as "global". The control flow join point corresponding to the @@ -602,15 +650,15 @@ } The compiler records a trace of the multiple paths it followed from the last full -snapshot in a lightweight tree structure. The *state data pointer* is +snapshot in a lightweight tree structure. The state data pointer is then only a pointer to a node in the tree; the branch from that node to -the root describes a path that let the compiler quickly *replay* its +the root describes a path that let the compiler quickly \emph{replay} its actions (without generating code again) from the latest full snapshot to rebuild its internal state and get back to the original promotion point. For example, if the interpreter source code contains promotions inside a run-time condition: - +% \begin{verbatim} if condition: ... @@ -625,7 +673,7 @@ then the tree will contain three nodes: a root node storing the snapshot, a child with a "True case" marker, and another child with a "False case" marker. Each promotion point generates a switch and a call -to ``continue\_compilation()`` pointing to the appropriate child node. +to \code{continue\_compilation()} pointing to the appropriate child node. The compiler can re-reach the correct promotion point by following the markers on the branch from the root to the child. @@ -642,7 +690,7 @@ of fresh variables, one per field. In the compiler, the variable that would normally contain the pointer to the structure gets instead a content that is neither a run-time value nor a compile-time constant, -but a special *virtual structure* -- a compile-time data structure that +but a special \emph{virtual structure} -- a compile-time data structure that recursively contains new variables, each of which can again store a run-time, a compile-time, or a virtual structure value. @@ -650,54 +698,54 @@ around by the compiler really represent run-time locations -- the name of a CPU register or a position in the machine stack frame. This is the case for both regular variables and the fields of virtual structures. -It means that the compilation of a ``getfield`` or ``setfield`` +It means that the compilation of a \code{getfield} or \code{setfield} operation performed on a virtual structure simply loads or stores such a location reference into the virtual structure; the actual value is not copied around at run-time. It is not always possible to keep structures virtual. The main -situation in which it needs to be "forced" (i.e. actually allocated at +situation in which it needs to be "forced" (i.e.\ actually allocated at run-time) is when the pointer escapes to some non-virtual location like a field of a real heap structure. Virtual structures still avoid the run-time allocation of most short-lived objects, even in non-trivial situations. The following -example shows a typical case. Consider the Python expression ``a+b+c``. -Assume that ``a`` contains an integer. The PyPy Python interpreter +example shows a typical case. Consider the Python expression \code{a+b+c}. +Assume that \code{a} contains an integer. The PyPy Python interpreter implements application-level integers as boxes -- instances of a -``W\_IntObject`` class with a single ``intval`` field. Here is the +\code{W\_IntObject} class with a single \code{intval} field. Here is the addition of two integers: - +% \begin{verbatim} - def add(w1, w2): # w1, w2 are W_IntObject instances - value1 = w1.intval - value2 = w2.intval - result = value1 + value2 - return W_IntObject(result) + def add(w1, w2): # w1, w2 are instances + value1 = w1.intval # of W_IntObject + value2 = w2.intval + result = value1 + value2 + return W_IntObject(result) \end{verbatim} -When interpreting the bytecode for ``a+b+c``, two calls to ``add()`` are -issued; the intermediate ``W\_IntObject`` instance is built by the first +When interpreting the bytecode for \code{a+b+c}, two calls to \code{add()} are +issued; the intermediate \code{W\_IntObject} instance is built by the first call and thrown away after the second call. By contrast, when the interpreter is turned into a compiler, the construction of the -``W\_IntObject`` object leads to a virtual structure whose ``intval`` +\code{W\_IntObject} object leads to a virtual structure whose \code{intval} field directly references the register in which the run-time addition put its result. This location is read out of the virtual structure at -the beginning of the second ``add()``, and the second run-time addition +the beginning of the second \code{add()}, and the second run-time addition directly operates on the same register. An interesting effect of virtual structures is that they play nicely with -promotion. Indeed, before the interpreter can call the proper ``add()`` +promotion. Indeed, before the interpreter can call the proper \code{add()} function for integers, it must first determine that the two arguments are indeed integer objects. In the corresponding dispatch logic, we have added two hints to promote the type of each of the two arguments. This produces a compiler that has the following behavior: in the general -case, the expression ``a+b`` will generate two consecutive run-time +case, the expression \code{a+b} will generate two consecutive run-time switches followed by the residual code of the proper version of -``add()``. However, in ``a+b+c``, the virtual structure representing +\code{add()}. However, in \code{a+b+c}, the virtual structure representing the intermediate value will contain a compile-time constant as type. Promoting a compile-time constant is trivial -- no run-time code is -generated. The whole expression ``a+b+c`` thus only requires three +generated. The whole expression \code{a+b+c} thus only requires three switches instead of four. It is easy to see that even more switches can be skipped in larger examples; typically, in a tight loop manipulating only integers, all objects are virtual structures for the compiler and @@ -722,7 +770,7 @@ or dictionary implementing the bindings of the locals. Then each local variable of the interpreted language can be represented as a separate run-time value in the generated code, or be itself further virtualized -(e.g. as a virtual ``W\_IntObject`` structure as seen above). +(e.g.\ as a virtual \code{W\_IntObject} structure as seen above). The issue is that the frame object is sometimes built in advance by non-JIT-generated code; even when it is not, it immediately escapes into @@ -732,12 +780,12 @@ into a global data structure (even though in practice most of frame objects are deallocated without ever having been introspected). -To solve this problem, we introduced *virtualizable structures,* a mix +To solve this problem, we introduced \emph{virtualizable structures,} a mix between regular run-time structures and virtual structures. A virtualizable structure is a structure that exists at run-time in the heap, but that is simultaneously treated as virtual by the compiler. Accesses to the structure from the code generated by the JIT are virtualized away, -i.e. don't involve run-time copying. The trade-off is that in order +i.e.\ don't involve run-time copying. The trade-off is that in order to keep both views synchronized, accesses to the run-time structure from regular code not produced by the JIT needs to perform an extra check. @@ -776,92 +824,52 @@ We quickly mention below a few other features and implementation details of the implementation of the JIT generation framework. More information -can be found in the on-line documentation. +can be found in the on-line documentation \cite{PyPy}. % => ref to web site +% +\begin{itemize} -* There are more user-specified hints available, like *deep-freezing,* +\item There are more user-specified hints available, like \emph{deep-freezing,} which marks an object as immutable in order to allow accesses to its content to be constant-folded at compile-time. -* The compiler representation of a run-time value for a non-virtual +\item The compiler representation of a run-time value for a non-virtual structure may additionally remember that some fields are actually compile-time constants. This occurs for example when a field is read from the structure at run-time and then promoted to compile-time. -* In addition to virtual structures, lists and dictionaries can also be +\item In addition to virtual structures, lists and dictionaries can also be virtual. -* Exception handling is achieved by inserting explicit operations into +\item Exception handling is achieved by inserting explicit operations into the graphs before they are timeshifted. Most of these run-time exception manipulations are then virtualized away, by treating the exception state as virtual. -* Timeshifting is performed in two phases: a first step transforms the +\item Timeshifting is performed in two phases: a first step transforms the graphs by updating their control flow and inserting pseudo-operations to drive the compiler; a second step (based on the RTyper \cite{D05.1}) replaces all necessary operations by calls to support code. -* The support code implements the generic behaviour of the compiler, - e.g. the merge logic. It is about 3500 lines of RPython code. The +\item The support code implements the generic behaviour of the compiler, + e.g.\ the merge logic. It is about 3500 lines of RPython code. The rest of the hint-annotator and timeshifter is about 3800 lines of Python code. -* The machine code backends (two so far, Intel IA32 and PowerPC) are +\item The machine code backends (two so far, Intel IA32 and PowerPC) are about 3500 further lines of RPython code each. There is a well-defined interface between the JIT compiler support code and the backends, making writing new backends relatively easy. The unusual part of the interface is the support for the run-time updatable switches. - -\subsection{Open issues} - -Here are what we think are the most important points that will need -attention in order to make the approach more robust: - -* The timeshifted graphs currently compile many branches eagerly. This - can easily result in residual code explosion. Depending on the source - interpreter this can also result in non-termination issues, where - compilation never completes. The opposite extreme would be to always - compile branches lazily, when they are about to be executed, as Psyco - does. While this neatly sidesteps termination issues, the best - solution is probably something in between these extremes. - -* As described in the Promotion section (\ref{promotion}), - we need fall-back solutions for when the - number of promoted run-time values seen at a particular point becomes - too large. - -* We need more flexible control about what to inline or not to inline in - the residual code. - -* The widening heuristics for merging needs to be refined. - -* The JIT generation framework needs to be made aware of some other - translation-time aspects \cite{D05.4} \cite{D07.1} in order to produce the - correct residual code (e.g. code calling the correct Garbage - Collection routines or supporting Stackless-style stack unwinding). - -* We did not work yet on profile-directed identification of program hot - spots. Currently, the interpreter must decide when to invoke the JIT - or not (which can itself be based on explicit requests from the interpreted - program). - -* The machine code backends can be improved. - -The latter point opens an interesting future research direction: can we -layer our kind of JIT compiler on top of a virtual machine that already -contains a lower-level JIT compiler? In other words, can we delegate -the difficult questions of machine code generation to a lower -independent layer, e.g. inlining, re-optimization of frequently executed -code, etc.? What changes would be required to an existing virtual -machine, e.g. a Java Virtual Machine, to support this? +\end{itemize} \section{Results} The following test function is an example of purely arithmetic code written in Python, which the PyPy JIT can run extremely fast: - +% \begin{verbatim} def f1(n): "Arbitrary test function." @@ -876,39 +884,40 @@ return x \end{verbatim} -We measured the time required to compute ``f1(2117)`` on the following +We measured the time required to compute \code{f1(2117)} on the following interpreters: +% +\begin{itemize} -* Python 2.4.4, the standard CPython implementation. +\item Python 2.4.4, the standard CPython implementation. -* A version of pypy-c including a generated JIT compiled. +\item A version of pypy-c (our Python interpreter translated to a stand-alone + executable via C) including a generated JIT compiled. -* gcc 4.1.1 compiling the above function rewritten in C (which, unlike +\item gcc 4.1.1 compiling the above function rewritten in C (which, unlike the other two, does not do any overflow checking on the arithmetic operations). +\end{itemize} + The relative results have been found to vary by 25\% depending on the machine. On our reference benchmark machine, a 4-cores Intel(R) Xeon(TM) CPU 3.20GHz with 5GB of RAM, we obtained the following results (the numbers in parenthesis are the slow-down ratio relative to the unoptimized gcc compilation): -+-----------------------------------------+------------------+ -| Interpreter | Seconds per call | -+=========================================+==================+ -| Python 2.4.4 | 0.82 (132x) | -+-----------------------------------------+------------------+ -| Python 2.4.4 with Psyco 1.5.2 | 0.0062 (1.00x) | -+-----------------------------------------+------------------+ -| pypy-c with the JIT turned off | 1.77 (285x) | -+-----------------------------------------+------------------+ -| pypy-c with the JIT turned on | 0.0091 (1.47x) | -+-----------------------------------------+------------------+ -| gcc | 0.0062 (1x) | -+-----------------------------------------+------------------+ -| gcc -O2 | 0.0022 (0.35x) | -+-----------------------------------------+------------------+ - +\begin{tabular}{|l|ll|} +\hline +Interpreter & \multicolumn{2}{|c|}{Seconds per call} \\ +\hline +Python 2.4.4 & 0.82 & (132x) \\ +Python 2.4.4 with Psyco 1.5.2 & 0.0062 & (1.00x) \\ +pypy-c with the JIT turned off & 1.77 & (285x) \\ +pypy-c with the JIT turned on & 0.0091 & (1.47x) \\ +gcc & 0.0062 & (1x) \\ +gcc -O2 & 0.0022 & (0.35x) \\ +\hline +\end{tabular} This table shows that the PyPy JIT is able to generate residual code that runs within the same order of magnitude as an unoptimizing gcc. It @@ -932,6 +941,54 @@ as 1.15x. +\section{Future work} + +Here are what we think are the most important points that will need +attention in order to make the approach more robust: +% +\begin{itemize} + +\item The timeshifted graphs currently compile many branches eagerly. This + can easily result in residual code explosion. Depending on the source + interpreter this can also result in non-termination issues, where + compilation never completes. The opposite extreme would be to always + compile branches lazily, when they are about to be executed, as Psyco + does. While this neatly sidesteps termination issues, the best + solution is probably something in between these extremes. + +\item As described in the Promotion section (\ref{promotion}), + we need fall-back solutions for when the + number of promoted run-time values seen at a particular point becomes + too large. + +\item We need more flexible control about what to inline or not to inline in + the residual code. + +\item The widening heuristics for merging needs to be refined. + +\item The JIT generation framework needs to be made aware of some other + translation-time aspects in order to produce the correct residual code + (e.g.\ code calling the correct Garbage Collection routines or + supporting Stackless-style stack unwinding \cite{D07.1}). + +\item We did not work yet on profile-directed identification of program hot + spots. Currently, the interpreter must decide when to invoke the JIT + or not (which can itself be based on explicit requests from the interpreted + program). + +\item The machine code backends can be improved. + +\end{itemize} + +The latter point opens an interesting future research direction: can we +layer our kind of JIT compiler on top of a virtual machine that already +contains a lower-level JIT compiler? In other words, can we delegate +the difficult questions of machine code generation to a lower +independent layer, e.g.\ inlining, re-optimization of frequently executed +code, etc.? What changes would be required to an existing virtual +machine, e.g.\ a Java Virtual Machine, to support this? + + \section{Conclusion} Producing the results described in the previous section requires the @@ -943,8 +1000,8 @@ boxing and to propagate them in the CPU stack and registers. Some slight reorganisation of the interpreter main loop without semantics -influence, marking the frames as virtualizable (\ref{virtualizable}), -and adding hints at +influence, marking the frames as virtualizable +(section \ref{virtualizable}), and adding hints at a few crucial points was all that was necessary for our Python interpreter. @@ -957,9 +1014,24 @@ compiler would be robust against language changes up to the need to maintain and possibly change the hints. -We consider this as a major breakthrough in term of the possibilities -it opens for language design and implementation; it was one of the -main goals of the research program within the PyPy project. +We consider this as a major breakthrough in term of the possibilities it +opens for language design and implementation; it was one of the main +goals of the research program within the PyPy project. Only groups with +very large amounts of resources can affort the high costs of writing +just-in-time compilers from scratch. Communities with limited available +resources for the implementation and maintenance of a language, such as, +generally, academic and open source projects, cannot afford such costs +-- and even when experimental just-in-time compilers exist, the mere +fact of having to maintain them in parallel with other implementations +is taxing for such communities, particularly when the languages in +question evolve quickly. In the PyPy approach, from a single simple +implementation for the language, we can generate stand-alone virtual +machines whose performance far excess that of traditional hand-written +virtual machines (like CPython, the reference C implementation of +Python); with the generation of a dynamic compiler, we achieve +state-of-the-art performance. + +% XXX balance columns %.. References (title not necessary, latex generates it) @@ -975,7 +1047,7 @@ %.. [D08.1] `Release a JIT Compiler for PyPy Including Processor Backends % for Intel and PowerPC`, PyPy EU-Report, 2007 % -%.. [FU] `Partial evaluation of compuation process -- an approach to a +%.. [FU] `Partial evaluation of computation process -- an approach to a % compiler-compiler`, Yoshihito Futamura, Higher-Order and % Symbolic Computation, 12(4):363-397, 1999. Reprinted from % Systems Computers Controls 2(5), 1971 @@ -1003,8 +1075,7 @@ % conference on Object-oriented programming languages, systems, and % applications, pp. 944-953, ACM Press, 2006 -\bigskip - +% ---- Bibliography ---- \bibliographystyle{abbrv} \bibliography{paper} From pdg at codespeak.net Tue May 29 17:03:37 2007 From: pdg at codespeak.net (pdg at codespeak.net) Date: Tue, 29 May 2007 17:03:37 +0200 (CEST) Subject: [pypy-svn] r43852 - pypy/dist/pypy/translator/jvm/test Message-ID: <20070529150337.1C0B6808F@code0.codespeak.net> Author: pdg Date: Tue May 29 17:03:36 2007 New Revision: 43852 Added: pypy/dist/pypy/translator/jvm/test/test_oo.py Log: translator/jvm/test - added test_oo Added: pypy/dist/pypy/translator/jvm/test/test_oo.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/jvm/test/test_oo.py Tue May 29 17:03:36 2007 @@ -0,0 +1,125 @@ +from pypy.translator.jvm.test.runtest import JvmTest + +class MyClass: + INCREMENT = 1 + + def __init__(self, x, y): + self.x = x + self.y = y + + def compute(self): + return self.x + self.y + + def compute_and_multiply(self, factor): + return self.compute() * factor + + def static_meth(x, y): + return x*y + static_meth = staticmethod(static_meth) + + def class_attribute(self): + return self.x + self.INCREMENT + +class MyDerivedClass(MyClass): + INCREMENT = 2 + + def __init__(self, x, y): + MyClass.__init__(self, x+12, y+34) + + def compute(self): + return self.x - self.y + + +# helper functions +def call_method(obj): + return obj.compute() + +def init_and_compute(cls, x, y): + return cls(x, y).compute() + +def nonnull_helper(lst): + if lst is None: + return 1 + else: + return 2 + + +class TestOO(JvmTest): + def test_indirect_call(self): + def f(): + return 1 + def g(): + return 2 + def fn(flag): + if flag: + x = f + else: + x = g + return x() + assert self.interpret(fn, [True]) == 1 + assert self.interpret(fn, [False]) == 2 + + def test_indirect_call_arguments(self): + def f(x): + return x+1 + def g(x): + return x+2 + def fn(flag, n): + if flag: + x = f + else: + x = g + return x(n) + assert self.interpret(fn, [True, 42]) == 43 + + + def test_compute(self): + def fn(x, y): + obj = MyClass(x, y) + return obj.compute() + assert self.interpret(fn, [42, 13]) == fn(42, 13) + + def test_compute_multiply(self): + def fn(x, y): + obj = MyClass(x, y) + return obj.compute_and_multiply(2) + assert self.interpret(fn, [42, 13]) == fn(42, 13) + + def test_inheritance(self): + def fn(x, y): + obj = MyDerivedClass(x, y) + return obj.compute_and_multiply(2) + assert self.interpret(fn, [42, 13]) == fn(42, 13) + + def test_liskov(self): + def fn(x, y): + base = MyClass(x, y) + derived = MyDerivedClass(x, y) + return call_method(base) + call_method(derived) + assert self.interpret(fn, [42, 13]) == fn(42, 13) + + def test_static_method(self): + def fn(x, y): + base = MyClass(x, y) + derived = MyDerivedClass(x, y) + return base.static_meth(x,y) + derived.static_meth(x, y)\ + + MyClass.static_meth(x, y) + MyDerivedClass.static_meth(x, y) + assert self.interpret(fn, [42, 13]) == fn(42, 13) + + def test_class_attribute(self): + def fn(x, y): + base = MyClass(x, y) + derived = MyDerivedClass(x, y) + return base.class_attribute() + derived.class_attribute() + assert self.interpret(fn, [42, 13]) == fn(42, 13) + + def test_runtimenew(self): + def fn(x, y): + return init_and_compute(MyClass, x, y) + init_and_compute(MyDerivedClass, x, y) + assert self.interpret(fn, [42, 13]) == fn(42, 13) + + def test_nonnull(self): + def fn(x, y): + return nonnull_helper([]) + nonnull_helper(None) + assert self.interpret(fn, [42, 13]) == fn(42, 13) + From santagada at codespeak.net Tue May 29 17:14:17 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Tue, 29 May 2007 17:14:17 +0200 (CEST) Subject: [pypy-svn] r43855 - in pypy/dist/pypy/lang/js: . test test/ecma/Array test/ecma/Boolean test/ecma/ExecutionContexts test/ecma/Expressions test/ecma/GlobalObject test/ecma/LexicalConventions test/ecma/Math test/ecma/String test/ecma/TypeConversion Message-ID: <20070529151417.20B8B808F@code0.codespeak.net> Author: santagada Date: Tue May 29 17:14:16 2007 New Revision: 43855 Modified: pypy/dist/pypy/lang/js/astbuilder.py pypy/dist/pypy/lang/js/interpreter.py pypy/dist/pypy/lang/js/jsgrammar.txt pypy/dist/pypy/lang/js/jsobj.py pypy/dist/pypy/lang/js/jsparser.py pypy/dist/pypy/lang/js/operations.py pypy/dist/pypy/lang/js/test/ecma/Array/15.4.2.1-2.js pypy/dist/pypy/lang/js/test/ecma/Array/15.4.2.1-3.js pypy/dist/pypy/lang/js/test/ecma/Array/15.4.2.2-1.js pypy/dist/pypy/lang/js/test/ecma/Array/15.4.5.1-2.js pypy/dist/pypy/lang/js/test/ecma/Array/15.4.5.2-2.js pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.4-1.js pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.4-2.js pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.4.1.js pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.4.3-4-n.js pypy/dist/pypy/lang/js/test/ecma/ExecutionContexts/10.2.3-1.js pypy/dist/pypy/lang/js/test/ecma/Expressions/11.10-1.js pypy/dist/pypy/lang/js/test/ecma/Expressions/11.10-2.js pypy/dist/pypy/lang/js/test/ecma/Expressions/11.10-3.js pypy/dist/pypy/lang/js/test/ecma/Expressions/11.4.8.js pypy/dist/pypy/lang/js/test/ecma/Expressions/11.7.1.js pypy/dist/pypy/lang/js/test/ecma/Expressions/11.7.2.js pypy/dist/pypy/lang/js/test/ecma/Expressions/11.7.3.js pypy/dist/pypy/lang/js/test/ecma/GlobalObject/15.1.2.2-2.js pypy/dist/pypy/lang/js/test/ecma/LexicalConventions/7.2-2-n.js pypy/dist/pypy/lang/js/test/ecma/LexicalConventions/7.2-3-n.js pypy/dist/pypy/lang/js/test/ecma/LexicalConventions/7.3-1.js pypy/dist/pypy/lang/js/test/ecma/LexicalConventions/7.4.3-4-n.js pypy/dist/pypy/lang/js/test/ecma/Math/15.8.1.1-2.js pypy/dist/pypy/lang/js/test/ecma/Math/15.8.1.js pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.11-1.js pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.11-2.js pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.11-3.js pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.11-5.js pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.11-6.js pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.12-1.js pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.12-2.js pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.12-3.js pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.12-4.js pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.12-5.js pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.7-1.js pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.7-2.js pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.7-3.js pypy/dist/pypy/lang/js/test/ecma/TypeConversion/9.6.js pypy/dist/pypy/lang/js/test/test_interp.py Log: made a more broader way to make the last ; on a block of code be optional, implemented shiftops and -= (sub in place). also did a major reworking on how the Function creator work. lot's of ; missing on ecma tests are fixed now. Modified: pypy/dist/pypy/lang/js/astbuilder.py ============================================================================== --- pypy/dist/pypy/lang/js/astbuilder.py (original) +++ pypy/dist/pypy/lang/js/astbuilder.py Tue May 29 17:14:16 2007 @@ -5,7 +5,7 @@ class ASTBuilder(RPythonVisitor): BINOP_TO_CLS = { '+': operations.Plus, - '-': operations.Minus, + '-': operations.Sub, '*': operations.Mult, '/': operations.Division, '%': operations.Mod, @@ -22,6 +22,9 @@ '>=': operations.Ge, '<': operations.Lt, '<=': operations.Le, + '>>': operations.Rsh, + '>>>': operations.Ursh, + '<<': operations.Lsh, '.': operations.MemberDot, '[': operations.Member, ',': operations.Comma, @@ -97,6 +100,7 @@ visit_logicalorexpression = binaryop visit_logicalandexpression = binaryop visit_relationalexpression = binaryop + visit_shiftexpression = binaryop visit_expression = binaryop def visit_memberexpression(self, node): @@ -295,6 +299,8 @@ i = 1 setup, i = self.get_next_expr(node, i) condition, i = self.get_next_expr(node, i) + if isinstance(condition, operations.Undefined): + condition = operations.Boolean(pos, True) update, i = self.get_next_expr(node, i) body, i = self.get_next_expr(node, i) return operations.For(pos, setup, condition, update, body) Modified: pypy/dist/pypy/lang/js/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/js/interpreter.py (original) +++ pypy/dist/pypy/lang/js/interpreter.py Tue May 29 17:14:16 2007 @@ -7,13 +7,14 @@ from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.streamio import open_file_as_stream +ASTBUILDER = ASTBuilder() + def writer(x): print x def load_source(script_source): - astb = ASTBuilder() temp_tree = parse(script_source) - return astb.dispatch(temp_tree) + return ASTBUILDER.dispatch(temp_tree) def load_file(filename): f = open_file_as_stream(filename) @@ -45,10 +46,12 @@ for i in range(tam-1): argslist.append(args[i].GetValue().ToString()) fargs = ','.join(argslist) - functioncode = "__anon__ = function (%s) {%s}"%(fargs, fbody) + functioncode = "function (%s) {%s}"%(fargs, fbody) else: - functioncode = "__anon__ = function () {}" - return evaljs(ctx, [W_String(functioncode),], this) + functioncode = "function () {}" + #remove program and sourcelements node + funcnode = parse(functioncode).children[0].children[0] + return ASTBUILDER.dispatch(funcnode).execute(ctx) def parseIntjs(ctx, args, this): if len(args) < 1: Modified: pypy/dist/pypy/lang/js/jsgrammar.txt ============================================================================== --- pypy/dist/pypy/lang/js/jsgrammar.txt (original) +++ pypy/dist/pypy/lang/js/jsgrammar.txt Tue May 29 17:14:16 2007 @@ -6,13 +6,18 @@ ; sourceelements : sourceelement >sourceelements< - | sourceelement + | sourceelementsemiopt ; sourceelement : | ; +sourceelementsemiopt : + | + ; + + statement : | [";"] | Modified: pypy/dist/pypy/lang/js/jsobj.py ============================================================================== --- pypy/dist/pypy/lang/js/jsobj.py (original) +++ pypy/dist/pypy/lang/js/jsobj.py Tue May 29 17:14:16 2007 @@ -70,6 +70,7 @@ return r_uint(0) def Get(self, P): + print P raise NotImplementedError def Put(self, P, V, dd=False, Modified: pypy/dist/pypy/lang/js/jsparser.py ============================================================================== --- pypy/dist/pypy/lang/js/jsparser.py (original) +++ pypy/dist/pypy/lang/js/jsparser.py Tue May 29 17:14:16 2007 @@ -14,11 +14,5 @@ parsef = make_parse_function(regexs, rules, eof=True) def parse(code): - try: - t = parsef(code) - except ParseError: - code += ';' - print code - t = parsef(code) - + t = parsef(code) return ToAST().transform(t) Modified: pypy/dist/pypy/lang/js/operations.py ============================================================================== --- pypy/dist/pypy/lang/js/operations.py (original) +++ pypy/dist/pypy/lang/js/operations.py Tue May 29 17:14:16 2007 @@ -67,12 +67,15 @@ class UnaryOp(Expression): def __init__(self, pos, expr, postfix=False): self.pos = pos + assert isinstance(expr, Node) self.expr = expr self.postfix = postfix class BinaryOp(Expression): def __init__(self, pos, left, right): self.pos = pos + assert isinstance(left, Node) + assert isinstance(right, Node) self.left = left self.right = right @@ -130,6 +133,8 @@ val = mult(ctx, v1.GetValue(), v3) elif op == "+=": val = plus(ctx, v1.GetValue(), v3) + elif op == "-=": + val = sub(ctx, v1.GetValue(), v3) elif op == "/=": val = division(ctx, v1.GetValue(), v3) elif op == "%=": @@ -638,7 +643,7 @@ fright = nright.ToNumber() return W_Number(fleft / fright) -def minus(ctx, nleft, nright): +def sub(ctx, nleft, nright): fleft = nleft.ToNumber() fright = nright.ToNumber() return W_Number(fleft - fright) @@ -660,8 +665,8 @@ mathop = staticmethod(division) -class Minus(BinaryNumberOp): - mathop = staticmethod(minus) +class Sub(BinaryNumberOp): + mathop = staticmethod(sub) class Null(Expression): @@ -970,6 +975,7 @@ class ForIn(Statement): def __init__(self, pos, iterator, lobject, body): self.pos = pos + assert isinstance(iterator, Node) self.iterator = iterator self.object = lobject self.body = body @@ -992,7 +998,7 @@ class For(Statement): def __init__(self, pos, setup, condition, update, body): - self.pos = pos + self.pos = pos self.setup = setup self.condition = condition self.update = update Modified: pypy/dist/pypy/lang/js/test/ecma/Array/15.4.2.1-2.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/Array/15.4.2.1-2.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/Array/15.4.2.1-2.js Tue May 29 17:14:16 2007 @@ -72,7 +72,7 @@ var TEST_STRING = "new Array("; -var ARGUMENTS = "" +var ARGUMENTS = ""; var TEST_LENGTH = Math.pow(2,10); //Math.pow(2,32); for ( var index = 0; index < TEST_LENGTH; index++ ) { Modified: pypy/dist/pypy/lang/js/test/ecma/Array/15.4.2.1-3.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/Array/15.4.2.1-3.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/Array/15.4.2.1-3.js Tue May 29 17:14:16 2007 @@ -78,7 +78,7 @@ writeHeaderToLog( SECTION + " "+ TITLE); var TEST_STRING = "new Array("; -var ARGUMENTS = "" +var ARGUMENTS = ""; var TEST_LENGTH = Math.pow(2,10); //Math.pow(2,32); for ( var index = 0; index < TEST_LENGTH; index++ ) { Modified: pypy/dist/pypy/lang/js/test/ecma/Array/15.4.2.2-1.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/Array/15.4.2.2-1.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/Array/15.4.2.2-1.js Tue May 29 17:14:16 2007 @@ -169,9 +169,8 @@ if ( Math.abs( n ) == 0 || Math.abs( n ) == Number.POSITIVE_INFINITY) { return 0; } - n = sign * Math.floor( Math.abs(n) ) - - n = n % Math.pow(2,32); + n = sign * Math.floor( Math.abs(n) ); + n = n % Math.pow(2,32); if ( n < 0 ){ n += Math.pow(2,32); Modified: pypy/dist/pypy/lang/js/test/ecma/Array/15.4.5.1-2.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/Array/15.4.5.1-2.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/Array/15.4.5.1-2.js Tue May 29 17:14:16 2007 @@ -89,7 +89,7 @@ AddCase( "3.00", "three" ); AddCase( "00010", "eight" ); AddCase( "37xyz", "thirty-five" ); -AddCase("5000000000", 5) +AddCase("5000000000", 5); AddCase( "-2", -3 ); new TestCase( SECTION, Modified: pypy/dist/pypy/lang/js/test/ecma/Array/15.4.5.2-2.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/Array/15.4.5.2-2.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/Array/15.4.5.2-2.js Tue May 29 17:14:16 2007 @@ -66,7 +66,7 @@ addCase( new Array(Math.pow(2,12)), Math.pow(2,12), 0, 0 ); addCase( new Array(Math.pow(2,13)), Math.pow(2,13), Math.pow(2,12), Math.pow(2,12) ); addCase( new Array(Math.pow(2,12)), Math.pow(2,12), Math.pow(2,12), Math.pow(2,12) ); -addCase( new Array(Math.pow(2,14)), Math.pow(2,14), Math.pow(2,12), Math.pow(2,12) ) +addCase( new Array(Math.pow(2,14)), Math.pow(2,14), Math.pow(2,12), Math.pow(2,12) ); // some tests where array is not empty // array is populated with strings Modified: pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.4-1.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.4-1.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.4-1.js Tue May 29 17:14:16 2007 @@ -53,7 +53,7 @@ */ -var VERSION = "ECMA_1" +var VERSION = "ECMA_1"; startTest(); var SECTION = "15.6.4-1"; Modified: pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.4-2.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.4-2.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.4-2.js Tue May 29 17:14:16 2007 @@ -53,7 +53,7 @@ */ -var VERSION = "ECMA_2" +var VERSION = "ECMA_2"; startTest(); var SECTION = "15.6.4-2"; Modified: pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.4.1.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.4.1.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.4.1.js Tue May 29 17:14:16 2007 @@ -50,7 +50,7 @@ var SECTION = "15.6.4.1"; var VERSION = "ECMA_1"; startTest(); -var TITLE = "Boolean.prototype.constructor" +var TITLE = "Boolean.prototype.constructor"; writeHeaderToLog( SECTION + TITLE ); new TestCase( SECTION, Modified: pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.4.3-4-n.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.4.3-4-n.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/Boolean/15.6.4.3-4-n.js Tue May 29 17:14:16 2007 @@ -56,7 +56,7 @@ writeHeaderToLog( SECTION + " "+ TITLE); -DESCRIPTION = "valof=Boolean.prototype.valueOf; x=new String( 'hello' ); x.valueOf=valof;x.valueOf()" +DESCRIPTION = "valof=Boolean.prototype.valueOf; x=new String( 'hello' ); x.valueOf=valof;x.valueOf()"; EXPECTED = "error"; new TestCase( SECTION, Modified: pypy/dist/pypy/lang/js/test/ecma/ExecutionContexts/10.2.3-1.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/ExecutionContexts/10.2.3-1.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/ExecutionContexts/10.2.3-1.js Tue May 29 17:14:16 2007 @@ -60,7 +60,7 @@ writeHeaderToLog( SECTION + " "+ TITLE); -var o = new MyObject("hello") +var o = new MyObject("hello"); new TestCase( SECTION, "var o = new MyObject('hello'); o.THIS == x", Modified: pypy/dist/pypy/lang/js/test/ecma/Expressions/11.10-1.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/Expressions/11.10-1.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/Expressions/11.10-1.js Tue May 29 17:14:16 2007 @@ -94,6 +94,7 @@ } return ( sign * Math.floor(Math.abs(n)) ); } + function ToInt32( n ) { n = Number( n ); var sign = ( n < 0 ) ? -1 : 1; @@ -107,6 +108,7 @@ return ( n ); } + function ToUint32( n ) { n = Number( n ); var sign = ( n < 0 ) ? -1 : 1; @@ -114,9 +116,8 @@ if ( Math.abs( n ) == 0 || Math.abs( n ) == Number.POSITIVE_INFINITY) { return 0; } - n = sign * Math.floor( Math.abs(n) ) - - n = n % Math.pow(2,32); + n = sign * Math.floor( Math.abs(n) ); + n = n % Math.pow(2,32); if ( n < 0 ){ n += Math.pow(2,32); Modified: pypy/dist/pypy/lang/js/test/ecma/Expressions/11.10-2.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/Expressions/11.10-2.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/Expressions/11.10-2.js Tue May 29 17:14:16 2007 @@ -113,7 +113,7 @@ if ( Math.abs( n ) == 0 || Math.abs( n ) == Number.POSITIVE_INFINITY) { return 0; } - n = sign * Math.floor( Math.abs(n) ) + n = sign * Math.floor( Math.abs(n) ); n = n % Math.pow(2,32); Modified: pypy/dist/pypy/lang/js/test/ecma/Expressions/11.10-3.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/Expressions/11.10-3.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/Expressions/11.10-3.js Tue May 29 17:14:16 2007 @@ -113,7 +113,7 @@ if ( Math.abs( n ) == 0 || Math.abs( n ) == Number.POSITIVE_INFINITY) { return 0; } - n = sign * Math.floor( Math.abs(n) ) + n = sign * Math.floor( Math.abs(n) ); n = n % Math.pow(2,32); Modified: pypy/dist/pypy/lang/js/test/ecma/Expressions/11.4.8.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/Expressions/11.4.8.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/Expressions/11.4.8.js Tue May 29 17:14:16 2007 @@ -105,7 +105,7 @@ if ( Math.abs( n ) == 0 || Math.abs( n ) == Number.POSITIVE_INFINITY) { return 0; } - n = sign * Math.floor( Math.abs(n) ) + n = sign * Math.floor( Math.abs(n) ); n = n % Math.pow(2,32); Modified: pypy/dist/pypy/lang/js/test/ecma/Expressions/11.7.1.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/Expressions/11.7.1.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/Expressions/11.7.1.js Tue May 29 17:14:16 2007 @@ -112,7 +112,7 @@ if ( Math.abs( n ) == 0 || Math.abs( n ) == Number.POSITIVE_INFINITY) { return 0; } - n = sign * Math.floor( Math.abs(n) ) + n = sign * Math.floor( Math.abs(n) ); n = n % Math.pow(2,32); Modified: pypy/dist/pypy/lang/js/test/ecma/Expressions/11.7.2.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/Expressions/11.7.2.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/Expressions/11.7.2.js Tue May 29 17:14:16 2007 @@ -127,7 +127,7 @@ if ( Math.abs( n ) == 0 || Math.abs( n ) == Number.POSITIVE_INFINITY) { return 0; } - n = sign * Math.floor( Math.abs(n) ) + n = sign * Math.floor( Math.abs(n) ); n = n % Math.pow(2,32); Modified: pypy/dist/pypy/lang/js/test/ecma/Expressions/11.7.3.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/Expressions/11.7.3.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/Expressions/11.7.3.js Tue May 29 17:14:16 2007 @@ -118,7 +118,7 @@ if ( Math.abs( n ) == 0 || Math.abs( n ) == Number.POSITIVE_INFINITY) { return 0; } - n = sign * Math.floor( Math.abs(n) ) + n = sign * Math.floor( Math.abs(n) ); n = n % Math.pow(2,32); Modified: pypy/dist/pypy/lang/js/test/ecma/GlobalObject/15.1.2.2-2.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/GlobalObject/15.1.2.2-2.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/GlobalObject/15.1.2.2-2.js Tue May 29 17:14:16 2007 @@ -173,7 +173,7 @@ parseInt("0x1000000000000081") ); s = -"0xFFFFFFFFFFFFF80000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" +"0xFFFFFFFFFFFFF80000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"; s += "0000000000000000000000000000000000000"; Modified: pypy/dist/pypy/lang/js/test/ecma/LexicalConventions/7.2-2-n.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/LexicalConventions/7.2-2-n.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/LexicalConventions/7.2-2-n.js Tue May 29 17:14:16 2007 @@ -64,7 +64,7 @@ writeHeaderToLog( SECTION + " "+ TITLE); DESCRIPTION = "\r\r\r\nb"; -EXPECTED = "error" +EXPECTED = "error"; new TestCase( SECTION, DESCRIPTION, "error", eval("\r\r\r\nb")); Modified: pypy/dist/pypy/lang/js/test/ecma/LexicalConventions/7.2-3-n.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/LexicalConventions/7.2-3-n.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/LexicalConventions/7.2-3-n.js Tue May 29 17:14:16 2007 @@ -63,7 +63,7 @@ DESCRIPTION = "\r\nb"; -EXPECTED = "error" +EXPECTED = "error"; new TestCase( SECTION, "a", "error", eval("\r\nb")); Modified: pypy/dist/pypy/lang/js/test/ecma/LexicalConventions/7.3-1.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/LexicalConventions/7.3-1.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/LexicalConventions/7.3-1.js Tue May 29 17:14:16 2007 @@ -68,7 +68,7 @@ "pass", "" ); -var x = "// test \n testcase.actual = 'pass'" +var x = "// test \n testcase.actual = 'pass'"; testcase.actual = eval(x); Modified: pypy/dist/pypy/lang/js/test/ecma/LexicalConventions/7.4.3-4-n.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/LexicalConventions/7.4.3-4-n.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/LexicalConventions/7.4.3-4-n.js Tue May 29 17:14:16 2007 @@ -79,7 +79,7 @@ jsOptions.reset(); -DESCRIPTION = "var super = true" +DESCRIPTION = "var super = true"; EXPECTED = "error"; // force exception since this is a negative test Modified: pypy/dist/pypy/lang/js/test/ecma/Math/15.8.1.1-2.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/Math/15.8.1.1-2.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/Math/15.8.1.1-2.js Tue May 29 17:14:16 2007 @@ -54,7 +54,7 @@ writeHeaderToLog( SECTION + " "+ TITLE); -var MATH_E = 2.7182818284590452354 +var MATH_E = 2.7182818284590452354; new TestCase( SECTION, "delete(Math.E)", false, Modified: pypy/dist/pypy/lang/js/test/ecma/Math/15.8.1.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/Math/15.8.1.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/Math/15.8.1.js Tue May 29 17:14:16 2007 @@ -52,7 +52,7 @@ Date: 7 july 1997 */ -var SECTION = "15.8.1" +var SECTION = "15.8.1"; var VERSION = "ECMA_1"; startTest(); var TITLE = "Value Properties of the Math Object"; Modified: pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.11-1.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.11-1.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.11-1.js Tue May 29 17:14:16 2007 @@ -93,7 +93,7 @@ function Unicode( c ) { u = GetUnicodeValues( c ); this.upper = u[0]; - this.lower = u[1] + this.lower = u[1]; return this; } function GetUnicodeValues( c ) { Modified: pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.11-2.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.11-2.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.11-2.js Tue May 29 17:14:16 2007 @@ -90,7 +90,7 @@ function Unicode( c ) { u = GetUnicodeValues( c ); this.upper = u[0]; - this.lower = u[1] + this.lower = u[1]; return this; } function GetUnicodeValues( c ) { Modified: pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.11-3.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.11-3.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.11-3.js Tue May 29 17:14:16 2007 @@ -89,7 +89,7 @@ function Unicode( c ) { u = GetUnicodeValues( c ); this.upper = u[0]; - this.lower = u[1] + this.lower = u[1]; return this; } function GetUnicodeValues( c ) { Modified: pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.11-5.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.11-5.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.11-5.js Tue May 29 17:14:16 2007 @@ -95,7 +95,7 @@ function Unicode( c ) { u = GetUnicodeValues( c ); this.upper = u[0]; - this.lower = u[1] + this.lower = u[1]; return this; } function GetUnicodeValues( c ) { Modified: pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.11-6.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.11-6.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.11-6.js Tue May 29 17:14:16 2007 @@ -91,7 +91,7 @@ function Unicode( c ) { u = GetUnicodeValues( c ); this.upper = u[0]; - this.lower = u[1] + this.lower = u[1]; return this; } function GetUnicodeValues( c ) { Modified: pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.12-1.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.12-1.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.12-1.js Tue May 29 17:14:16 2007 @@ -95,7 +95,7 @@ function Unicode( c ) { u = GetUnicodeValues( c ); this.upper = u[0]; - this.lower = u[1] + this.lower = u[1]; return this; } function GetUnicodeValues( c ) { Modified: pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.12-2.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.12-2.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.12-2.js Tue May 29 17:14:16 2007 @@ -92,7 +92,7 @@ function Unicode( c ) { u = GetUnicodeValues( c ); this.upper = u[0]; - this.lower = u[1] + this.lower = u[1]; return this; } function GetUnicodeValues( c ) { Modified: pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.12-3.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.12-3.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.12-3.js Tue May 29 17:14:16 2007 @@ -134,7 +134,7 @@ function Unicode( c ) { u = GetUnicodeValues( c ); this.upper = u[0]; - this.lower = u[1] + this.lower = u[1]; return this; } function GetUnicodeValues( c ) { Modified: pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.12-4.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.12-4.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.12-4.js Tue May 29 17:14:16 2007 @@ -90,7 +90,7 @@ function Unicode( c ) { u = GetUnicodeValues( c ); this.upper = u[0]; - this.lower = u[1] + this.lower = u[1]; return this; } function GetUnicodeValues( c ) { Modified: pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.12-5.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.12-5.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.12-5.js Tue May 29 17:14:16 2007 @@ -90,7 +90,7 @@ function Unicode( c ) { u = GetUnicodeValues( c ); this.upper = u[0]; - this.lower = u[1] + this.lower = u[1]; return this; } function GetUnicodeValues( c ) { Modified: pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.7-1.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.7-1.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.7-1.js Tue May 29 17:14:16 2007 @@ -164,10 +164,10 @@ test(); function LastIndexOf( string, search, position ) { - string = String( string ); - search = String( search ); + string = String( string ); + search = String( search ); - position = Number( position ) + position = Number( position ); if ( isNaN( position ) ) { position = Infinity; Modified: pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.7-2.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.7-2.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.7-2.js Tue May 29 17:14:16 2007 @@ -165,7 +165,7 @@ string = String( string ); search = String( search ); - position = Number( position ) + position = Number( position ); if ( isNaN( position ) ) { position = Infinity; Modified: pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.7-3.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.7-3.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/String/15.5.4.7-3.js Tue May 29 17:14:16 2007 @@ -109,7 +109,7 @@ string = String( string ); search = String( search ); - position = Number( position ) + position = Number( position ); if ( isNaN( position ) ) { position = Infinity; Modified: pypy/dist/pypy/lang/js/test/ecma/TypeConversion/9.6.js ============================================================================== --- pypy/dist/pypy/lang/js/test/ecma/TypeConversion/9.6.js (original) +++ pypy/dist/pypy/lang/js/test/ecma/TypeConversion/9.6.js Tue May 29 17:14:16 2007 @@ -125,7 +125,7 @@ if ( Math.abs( n ) == 0 || Math.abs( n ) == Number.POSITIVE_INFINITY) { return 0; } - n = sign * Math.floor( Math.abs(n) ) + n = sign * Math.floor( Math.abs(n) ); n = n % Math.pow(2,32); Modified: pypy/dist/pypy/lang/js/test/test_interp.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_interp.py (original) +++ pypy/dist/pypy/lang/js/test/test_interp.py Tue May 29 17:14:16 2007 @@ -495,6 +495,10 @@ yield assertv, "2 ^ 2;", 0 yield assertv, "2 & 3;", 2 yield assertv, "2 | 3;", 3 + yield assertv, "2 << 2;", 8 + yield assertv, "4 >> 2;", 1 + yield assertv, "-2 >> 31", -1 + yield assertv, "-2 >>> 31;", 1 def test_for_vararg(): assertp(""" @@ -531,6 +535,7 @@ def test_inplace_assign(): yield assertv, "x=1; x+=1; x;", 2 + yield assertv, "x=1; x-=1; x;", 0 yield assertv, "x=2; x*=2; x;", 4 yield assertv, "x=2; x/=2; x;", 1 yield assertv, "x=4; x%=2; x;", 0 @@ -550,3 +555,6 @@ def test_semicolon(): assertv("1", 1) + +def test_functionjs(): + assertv("x = Function('return 1'); x()", 1) \ No newline at end of file From pdg at codespeak.net Tue May 29 17:54:58 2007 From: pdg at codespeak.net (pdg at codespeak.net) Date: Tue, 29 May 2007 17:54:58 +0200 (CEST) Subject: [pypy-svn] r43857 - pypy/dist/pypy/translator/jvm/test Message-ID: <20070529155458.245DD80A1@code0.codespeak.net> Author: pdg Date: Tue May 29 17:54:58 2007 New Revision: 43857 Added: pypy/dist/pypy/translator/jvm/test/test_op.py pypy/dist/pypy/translator/jvm/test/test_overflow.py pypy/dist/pypy/translator/jvm/test/test_primitive.py Modified: pypy/dist/pypy/translator/jvm/test/runtest.py Log: translator/jvm/test - added test_overflow test_primitive test_op, updated runtest-still needs work Modified: pypy/dist/pypy/translator/jvm/test/runtest.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/runtest.py (original) +++ pypy/dist/pypy/translator/jvm/test/runtest.py Tue May 29 17:54:58 2007 @@ -15,6 +15,28 @@ FLOAT_PRECISION = 8 +# CLI duplicate. Needs JVMtest support -Paul +# check function depends on this function +#def compile_function(func, annotation=[], graph=None, backendopt=True): +# olddefs = patch() +# gen = _build_gen(func, annotation, graph, backendopt) +# gen.generate_source() +# exe_name = gen.build_exe() +# unpatch(*olddefs) # restore original values +# return CliFunctionWrapper(exe_name) + +# CLI duplicate. Needs compile_function support (see above) -Paul +# check is used in test_overflow +#def check(func, annotation, args): +# mono = compile_function(func, annotation) +# res1 = func(*args) +# res2 = mono(*args) +# +# if type(res1) is float: +# assert round(res1, FLOAT_PRECISION) == round(res2, FLOAT_PRECISION) +# else: +# assert res1 == res2 + class StructTuple(tuple): def __getattr__(self, name): if name.startswith('item'): Added: pypy/dist/pypy/translator/jvm/test/test_op.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/jvm/test/test_op.py Tue May 29 17:54:58 2007 @@ -0,0 +1,7 @@ +from pypy.translator.jvm.test.runtest import JvmTest +from pypy.translator.oosupport.test_template.operations import BaseTestOperations + +# ====> ../../oosupport/test_template/operations.py + +class TestOperations(JvmTest, BaseTestOperations): + pass Added: pypy/dist/pypy/translator/jvm/test/test_overflow.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/jvm/test/test_overflow.py Tue May 29 17:54:58 2007 @@ -0,0 +1,47 @@ +#from pypy.translator.jvm.test.runtest import check +import py +from pypy.translator.jvm.test.runtest import JvmTest +from pypy.rlib.rarithmetic import ovfcheck + +import sys + +class TestJvmOperation(JvmTest): + def test_recursive(self): + py.test.skip("JVM runtest lacks support to run these tests") + +#def op_add(x, y): +# try: +# return ovfcheck(x+y) +# except OverflowError: +# return 42 +# +#def op_sub(x, y): +# try: +# return ovfcheck(x-y) +# except OverflowError: +# return 42 +# +#def op_mul(x, y): +# try: +# return ovfcheck(x*y) +# except OverflowError: +# return 42 +# +#def op_lshift(x, y): +# try: +# return ovfcheck(x< Author: santagada Date: Tue May 29 18:12:33 2007 New Revision: 43858 Modified: pypy/dist/pypy/lang/js/astbuilder.py pypy/dist/pypy/lang/js/jsgrammar.txt pypy/dist/pypy/lang/js/test/test_astbuilder.py pypy/dist/pypy/lang/js/test/test_interp.py Log: hex and octal literals Modified: pypy/dist/pypy/lang/js/astbuilder.py ============================================================================== --- pypy/dist/pypy/lang/js/astbuilder.py (original) +++ pypy/dist/pypy/lang/js/astbuilder.py Tue May 29 18:12:33 2007 @@ -75,6 +75,16 @@ pos = self.get_pos(node) number = operations.Number(pos, float(node.additional_info)) return number + + def visit_HEXINTEGERLITERAL(self, node): + pos = self.get_pos(node) + number = operations.Number(pos, float(int(node.additional_info, 16))) + return number + + def visit_OCTALLITERAL(self, node): + pos = self.get_pos(node) + number = operations.Number(pos, float(int(node.additional_info, 8))) + return number def string(self,node): pos = self.get_pos(node) Modified: pypy/dist/pypy/lang/js/jsgrammar.txt ============================================================================== --- pypy/dist/pypy/lang/js/jsgrammar.txt (original) +++ pypy/dist/pypy/lang/js/jsgrammar.txt Tue May 29 18:12:33 2007 @@ -175,7 +175,8 @@ numericliteral : | - ; + | + ; DECIMALLITERAL : "(((0|[1-9][0-9]*)(\.[0-9]*)?)|(\.[0-9]+))([eE][\+\-]?[0-9]*)?" ; @@ -183,6 +184,9 @@ HEXINTEGERLITERAL : "0[xX][0-9a-fA-F]+" ; +OCTALLITERAL : "0[0-7]+" + ; + stringliteral : | ; Modified: pypy/dist/pypy/lang/js/test/test_astbuilder.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_astbuilder.py (original) +++ pypy/dist/pypy/lang/js/test/test_astbuilder.py Tue May 29 18:12:33 2007 @@ -19,7 +19,8 @@ yield to_ast, "null;" yield to_ast, "f();" yield to_ast, "new f();" - #yield to_ast, "" + yield to_ast, "01;" + yield to_ast, "0xFF;" def test_funcvarfinder(): pos = operations.Position() Modified: pypy/dist/pypy/lang/js/test/test_interp.py ============================================================================== --- pypy/dist/pypy/lang/js/test/test_interp.py (original) +++ pypy/dist/pypy/lang/js/test/test_interp.py Tue May 29 18:12:33 2007 @@ -557,4 +557,8 @@ assertv("1", 1) def test_functionjs(): - assertv("x = Function('return 1'); x()", 1) \ No newline at end of file + assertv("x = Function('return 1'); x()", 1) + +def test_octal_and_hex(): + yield assertv, "010;", 8 + yield assertv, "0xF", 15 From pedronis at codespeak.net Tue May 29 18:14:22 2007 From: pedronis at codespeak.net (pedronis at codespeak.net) Date: Tue, 29 May 2007 18:14:22 +0200 (CEST) Subject: [pypy-svn] r43859 - pypy/extradoc/talk/dls2007 Message-ID: <20070529161422.9455F8092@code0.codespeak.net> Author: pedronis Date: Tue May 29 18:14:22 2007 New Revision: 43859 Modified: pypy/extradoc/talk/dls2007/paper.tex Log: tweaks Modified: pypy/extradoc/talk/dls2007/paper.tex ============================================================================== --- pypy/extradoc/talk/dls2007/paper.tex (original) +++ pypy/extradoc/talk/dls2007/paper.tex Tue May 29 18:14:22 2007 @@ -14,7 +14,7 @@ \affaddr{Deutschland}\\ \email{arigo at tunes.org} \alignauthor Samuele Pedroni\\ - \affaddr{AB Strakt}\\ + \affaddr{Open End AB}\\ \affaddr{Norra ?gatan 10A}\\ \affaddr{416 64 G?teborg}\\ \affaddr{Sweden}\\ @@ -37,7 +37,7 @@ interpreter. Crucial for the effectiveness of dynamic compilation is the use of run-time information to improve compilation results: in our approach, a novel powerful primitive called "promotion" that "promotes" -run-time values to compile-time is used to that effect. In this report, +run-time values to compile-time is used to that effect. In this paper, we describe it along with other novel techniques that allow the approach to scale to something as large as PyPy's Python interpreter. \footnote{This research was partially supported by the EU funded From ale at codespeak.net Tue May 29 18:16:02 2007 From: ale at codespeak.net (ale at codespeak.net) Date: Tue, 29 May 2007 18:16:02 +0200 (CEST) Subject: [pypy-svn] r43860 - in pypy/dist/pypy/lib/pyontology: . test Message-ID: <20070529161602.8D2578092@code0.codespeak.net> Author: ale Date: Tue May 29 18:16:02 2007 New Revision: 43860 Modified: pypy/dist/pypy/lib/pyontology/pyontology.py pypy/dist/pypy/lib/pyontology/sparql_grammar.py pypy/dist/pypy/lib/pyontology/test/test_sparql.py Log: Evaluate the FILTER constraint Modified: pypy/dist/pypy/lib/pyontology/pyontology.py ============================================================================== --- pypy/dist/pypy/lib/pyontology/pyontology.py (original) +++ pypy/dist/pypy/lib/pyontology/pyontology.py Tue May 29 18:16:02 2007 @@ -441,7 +441,8 @@ if isinstance(item, rdflib_literal): return item.datatype is None or item.datatype == self.Type else: - return XMLTypes[self.Type.split("#")[-1]] == type(item) + self_type = XMLTypes[self.Type.split("#")[-1]][1] + return self_type == type(item) datatype = Type datatype.Type = typ @@ -470,9 +471,11 @@ getUriref('rdf', 'Literal') : Literal, # getUriref('rdf', 'type') : Property, } -#XMLTypes = ['string', 'float', 'integer', 'date'] -XMLTypes = {'string': str, 'float': float, 'integer': int, - 'date': lambda x: datetime.date(*[int(v) for v in x.split('-')])} + +XMLTypes = {'string': (str, str), 'float': (float, float), + 'integer': (int, int), + 'date': (lambda x: datetime.date(*[int(v) for v in x.split('-')]), + datetime.date)} for typ in XMLTypes: uri = getUriref('xmlschema', typ) @@ -519,7 +522,11 @@ def attach_fd(self): for (s, p, o) in (self.graph.triples((None,)*3)): - self.consider_triple((s, p, o)) + try: + self.consider_triple((s, p, o)) + except Exception, exc: + print "Exception: ", exc + print s , p, o def finish(self): cons = [(c.cost,c) for c in self.constraints if hasattr(c, 'cost')] @@ -536,10 +543,10 @@ # In case of an inconsistent ontology remove the comments to get more # than one error -# try: - constraint.narrow(self.variables) -# except ConsistencyFailure, e: -# print "FAilure", e + try: + constraint.narrow(self.variables) + except ConsistencyFailure, e: + print "Failure", e things = list(self.variables['owl_Thing'].getValues()) things += list(self.variables['owl_Literal'].getValues()) self.variables['owl_Thing'].setValues(things) @@ -565,8 +572,9 @@ for item in trip_: if isinstance(item[0], rdflib_literal): o = item[0] - if o.datatype in builtin_voc: - o = XMLTypes[o.datatype.split('#')[-1]](o) + dt = o.datatype + if dt in builtin_voc: + o = XMLTypes[dt.split('#')[-1]][0](o) self.variables['owl_Literal'].addValue(o) newtrip.append(o) elif item[0].NCNAME_PREFIX: @@ -588,10 +596,13 @@ constrain = where.GroupGraphPattern[0].Constraint if constrain: constrain = constrain[0] - expr = constrain.pop(-1) + expr = constrain.pop(-1)[0] + vars_mangled = [] for x in constrain: - expr = expr.replace(x, URIRef('query_'+x)) - constrain = Expression([URIRef('query_'+x) for x in constrain], expr) + name = self.mangle_name(URIRef('query_'+x)) + vars_mangled.append(name) + expr = expr.replace(x, name) + constrain = Expression([x for x in vars_mangled], expr) return new, prefixes, resvars, constrain, vars # There are 8 ways of having the triples in the query, if predicate is not a builtin owl predicate @@ -612,6 +623,8 @@ new, prefixes, resvars, constrain, vars = self._sparql(query) query_dom = {} query_constr = [] + if constrain : + query_constr.append(constrain) for trip in new: case = trip.pop(-1) if case == 0: @@ -624,22 +637,16 @@ elif case == 1: # Add a HasValue constraint ns,pred = trip[1].split("#") - if 0: #ns in namespaces.values(): - import pdb - pdb.set_trace() - self.consider_triple(trip) + var = self.make_var(Restriction, URIRef(trip[0])) + prop = URIRef(trip[1]) + prop_name = self.mangle_name(prop) + self.onProperty(var, prop) + query_dom[prop_name] = self.variables[prop_name] + if trip[2] in self.variables['owl_Literal'] : + val = trip[2] else: - var = self.make_var(Restriction, URIRef(trip[0])) - prop = URIRef(trip[1]) - prop_name = self.mangle_name(prop) - self.onProperty(var, prop) - query_dom[prop_name] = self.variables[prop_name] - if trip[2] in self.variables['owl_Literal'] : - val = trip[2] - else: - val = self.variables[self.mangle_name(trip[2])] - query_constr.append(HasvalueConstraint(var, prop_name, val)) - #self.hasValue(var, trip[2]) + val = self.variables[self.mangle_name(trip[2])] + query_constr.append(HasvalueConstraint(var, prop_name, val)) elif case == 2: # for all p's return p if p[0]==s and p[1]==o @@ -723,8 +730,6 @@ dom = dict([(v, self.variables[v]) for v in solve_vars]) dom.update(query_dom) -# import pdb -# pdb.set_trace() # solve the repository and return the solution rep = Repository(solve_vars, dom, query_constr) res_s = Solver(MyDistributor(solve_vars)).solve(rep, verbose=0) @@ -776,7 +781,7 @@ elif type(o) == rdflib_literal: # self.variables.setdefault('owl_Literal', ClassDomain('owl_Literal',u'')) if o.datatype in builtin_voc: - o = XMLTypes[o.datatype.split('#')[-1]](o) + o = XMLTypes[o.datatype.split('#')[-1]][0](o) self.variables['owl_Literal'].addValue(o) val = o else: @@ -1081,13 +1086,21 @@ def allValuesFrom(self, s, var): sub = self.make_var(Restriction, s) obj = self.make_var(ClassDomain, var) - cons = AllValueConstraint(sub, obj) - self.constraints.append(cons) + try: + prop = self.variables[sub].property + cons = AllValueConstraint(sub, prop, obj) + self.constraints.append(cons) + except KeyError: + self.variables[sub].un_constraint.append(AllValueConstraint) def someValuesFrom(self, s, var): sub = self.make_var(Restriction, s) obj = self.make_var(ClassDomain, var) - cons = SomeValueConstraint(sub, obj) + try: + prop = self.variables[sub].property + except KeyError: + prop = None + cons = SomeValueConstraint(sub, prop, obj) self.constraints.append(cons) # ----------------- ---------------- Modified: pypy/dist/pypy/lib/pyontology/sparql_grammar.py ============================================================================== --- pypy/dist/pypy/lib/pyontology/sparql_grammar.py (original) +++ pypy/dist/pypy/lib/pyontology/sparql_grammar.py Tue May 29 18:16:02 2007 @@ -44,11 +44,11 @@ return ret def construct_constraint(s, loc, toks): - varlist = [] + varlist = [] for x in toks.Var: varlist.append(x[0]) - varlist.append(toks[0]) - return [varlist] + varlist.append(toks) + return [varlist, toks[0]] class SPARQLGrammar(object): Modified: pypy/dist/pypy/lib/pyontology/test/test_sparql.py ============================================================================== --- pypy/dist/pypy/lib/pyontology/test/test_sparql.py (original) +++ pypy/dist/pypy/lib/pyontology/test/test_sparql.py Tue May 29 18:16:02 2007 @@ -69,7 +69,6 @@ assert len(res[0]) == len(res[1]) == 4 assert res[0][1] in ['http://example.org/ns#p', 'http://example.org/ns#q'] assert res[1][1] in ['http://example.org/ns#p', 'http://example.org/ns#q'] - # assert result[3].formula == 'query_x<2' assert (['query_x<2'], {}).formula == 'query_x<2' # There are 8 ways of having the triples in the query, if predicate is not a builtin owl predicate @@ -224,6 +223,27 @@ assert len(res) == 8 assert res[0]['x'] == u'http://www.lt-world.org/ltw.owl#obj_61128' +# Generating simply a list of people. + +query0 = """ + PREFIX ltw : + PREFIX rdf : + SELECT ?person + WHERE { + ?person_obj rdf:type ltw:Active_Person . + ?person_obj ltw:personName ?person + } + """ + +def test_query0(): + O = Ontology() + O.add_file(datapath("testont2.rdf"), 'xml') + O.attach_fd() + + res = O.sparql(query0) + assert len(res) == 1 + assert res[0]['person'] == '\nKlara Vicsi' + query_y = """ PREFIX ltw : PREFIX owl : @@ -327,20 +347,6 @@ ?person_obj ltw:personName ?person . } ORDER BY ?person""" -#how many projects have been funded by BMBF in 2006 -query2 = """ - PREFIX ltw : - PREFIX owl : - PREFIX rdf : - SELECT ?project ?date_begin ?x - WHERE { - ?project ltw:supportedBy ?x . - ?x rdf:type ltw:Active_FundingOrganisation . - ?x ltw:organisationNameAbbreviation 'BMBF' . - ?project ltw:dateStart ?date_begin . - ?project ltw:dateEnd ?date_end . - FILTER ( ?date_begin < 2007 && ?date_end >= 2006) . - }""" #which project is funded in a technological area (i.e. Semantic web), query_3 = """ PREFIX ltw : @@ -410,6 +416,22 @@ assert res[0]['person'] == u'\nKlara Vicsi' +#how many projects have been funded by BMBF in 2006 +query2 = """ + PREFIX ltw : + PREFIX owl : + PREFIX rdf : + PREFIX xsd : + SELECT ?project ?date_begin ?x + WHERE { + ?project ltw:supportedBy ?x . + ?x rdf:type ltw:Active_FundingOrganisation . + ?x ltw:organisationNameAbbreviation 'BMBF' . + ?project ltw:dateStart ?date_begin . + ?project ltw:dateEnd ?date_end . + FILTER ( ?date_begin < "2007-01-01"^^ && ?date_end >= "2006-01-01"^^) . + }""" + def test_query_2(): # py.test.skip("Needs special care for dateTime type") O = Ontology() @@ -430,6 +452,7 @@ res = O.sparql(query_3) assert len(res) == 1 assert res[0]['project'] == u'http://www.lt-world.org/ltw.owl#obj_59773' + assert res[0]['x'] == u'http://www.lt-world.org/ltw.owl#obj_59773' import xmlrpclib, socket, os, signal From pdg at codespeak.net Tue May 29 18:16:57 2007 From: pdg at codespeak.net (pdg at codespeak.net) Date: Tue, 29 May 2007 18:16:57 +0200 (CEST) Subject: [pypy-svn] r43861 - pypy/dist/pypy/translator/jvm/test Message-ID: <20070529161657.94C7E8092@code0.codespeak.net> Author: pdg Date: Tue May 29 18:16:57 2007 New Revision: 43861 Added: pypy/dist/pypy/translator/jvm/test/test_range.py pypy/dist/pypy/translator/jvm/test/test_runtest.py Modified: pypy/dist/pypy/translator/jvm/test/test_snippet.py Log: translator/jvm/test - added test_range test_runtest, updated test_snippet Added: pypy/dist/pypy/translator/jvm/test/test_range.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/jvm/test/test_range.py Tue May 29 18:16:57 2007 @@ -0,0 +1,7 @@ +import py +from pypy.translator.jvm.test.runtest import JvmTest +from pypy.rpython.test.test_rrange import BaseTestRrange + +class TestJvmRange(JvmTest, BaseTestRrange): + def test_rlist_range(self): + pass # it doesn't make sense here Added: pypy/dist/pypy/translator/jvm/test/test_runtest.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/jvm/test/test_runtest.py Tue May 29 18:16:57 2007 @@ -0,0 +1,66 @@ +from pypy.translator.jvm.test.runtest import JvmTest +from pypy.translator.jvm.test.runtest import FLOAT_PRECISION +from pypy.annotation.listdef import s_list_of_strings + +def ident(x): + return x + +class TestRunTest(JvmTest): + + def test_patch_os(self): + import os + from pypy.translator.cli.support import patch, unpatch, NT_OS + original_O_CREAT = os.O_CREAT + olddefs = patch() + assert os.O_CREAT == NT_OS['O_CREAT'] + unpatch(*olddefs) + assert os.O_CREAT == original_O_CREAT + + def test_int(self): + assert self.interpret(ident, [42]) == 42 + + def test_bool(self): + assert self.interpret(ident, [True]) == True + assert self.interpret(ident, [False]) == False + + def test_float(self): + x = 10/3.0 + res = self.interpret(ident, [x]) + assert self.float_eq(x, res) + + def test_char(self): + assert self.interpret(ident, ['a']) == 'a' + + def test_list(self): + def fn(): + return [1, 2, 3] + assert self.interpret(fn, []) == [1, 2, 3] + + def test_tuple(self): + def fn(): + return 1, 2 + assert self.interpret(fn, []) == (1, 2) + + def test_string(self): + def fn(): + return 'foo' + res = self.interpret(fn, []) + assert self.ll_to_string(res) == 'foo' + + def test_exception(self): + def fn(): + raise ValueError + self.interpret_raises(ValueError, fn, []) + + def test_exception_subclass(self): + def fn(): + raise IndexError + self.interpret_raises(LookupError, fn, []) + + def test_object_or_none(self): + def fn(flag): + if flag: + return "hello"; + else: + return None + assert self.interpret(fn, [False]) is None Modified: pypy/dist/pypy/translator/jvm/test/test_snippet.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_snippet.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_snippet.py Tue May 29 18:16:57 2007 @@ -23,4 +23,15 @@ return res assert self.interpret(base_func, [True]) == 110 assert self.interpret(base_func, [False]) == 001 + + def test_link_SSA(self): + def fn(): + lst = [42, 43, 44] + for i in range(len(lst)): + item = lst[i] + if i < 10: + lst[i] = item+10 + return lst + res = self.ll_to_list(self.interpret(fn, [])) + assert res == [52, 53, 54] From fijal at codespeak.net Tue May 29 22:46:06 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 29 May 2007 22:46:06 +0200 (CEST) Subject: [pypy-svn] r43886 - in pypy/branch/kill-ctypes/pypy: module/termios module/termios/test rpython/module rpython/module/test Message-ID: <20070529204606.257C88082@code0.codespeak.net> Author: fijal Date: Tue May 29 22:46:05 2007 New Revision: 43886 Modified: pypy/branch/kill-ctypes/pypy/module/termios/__init__.py pypy/branch/kill-ctypes/pypy/module/termios/interp_termios.py pypy/branch/kill-ctypes/pypy/module/termios/test/test_termios.py pypy/branch/kill-ctypes/pypy/rpython/module/ll_termios.py pypy/branch/kill-ctypes/pypy/rpython/module/test/test_ll_termios.py Log: Add some obscure annotator hack and make it translatable and testable Modified: pypy/branch/kill-ctypes/pypy/module/termios/__init__.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/module/termios/__init__.py (original) +++ pypy/branch/kill-ctypes/pypy/module/termios/__init__.py Tue May 29 22:46:05 2007 @@ -1,5 +1,8 @@ from pypy.interpreter.mixedmodule import MixedModule +import termios +from pypy.rpython.module.ll_termios import termios_error +from pypy.rlib.nonconst import NonConstant class Module(MixedModule): "This module provides an interface to the Posix calls for tty I/O control.\n\ @@ -25,6 +28,13 @@ 'tcsetattr' : 'interp_termios.tcsetattr', } + def startup(self, space): + # XXX nasty annotation trick + try: + raise termios_error(NonConstant(-3), NonConstant("xxx")) + except termios.error, e: + pass + import termios from pypy.module.termios import interp_termios Modified: pypy/branch/kill-ctypes/pypy/module/termios/interp_termios.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/module/termios/interp_termios.py (original) +++ pypy/branch/kill-ctypes/pypy/module/termios/interp_termios.py Tue May 29 22:46:05 2007 @@ -5,9 +5,9 @@ from pypy.interpreter.baseobjspace import ObjSpace, W_Root from pypy.interpreter.error import OperationError +from pypy.rpython.module import ll_termios import os import termios -from pypy.rlib.objectmodel import we_are_translated # proper semantics are to have termios.error, but since it's not documented # anyway, let's have it as OSError on interplevel. We need to have @@ -30,23 +30,15 @@ tcsetattr.unwrap_spec = [ObjSpace, int, int, W_Root] def tcgetattr(space, fd): - # XXX Argh argh argh argh. ARGH! - if we_are_translated(): - try: - tup_w = termios.tcgetattr(fd) - except OSError, e: - raise convert_error(space, e) - else: - try: - tup_w = termios.tcgetattr(fd) - except termios.error, e: - e.errno = e.args[0] - raise convert_error(space, e) - l_w = [] - for w_item in tup_w[:-1]: - l_w.append(space.wrap(w_item)) + try: + tup = termios.tcgetattr(fd) + except termios.error, e: + e.errno = e.args[0] + raise convert_error(space, e) + iflag, oflag, cflag, lflag, ispeed, ospeed, cc = tup + l_w = [space.wrap(i) for i in [iflag, oflag, cflag, lflag, ispeed, ospeed]] # last one need to be chosen carefully - w_cc = space.newlist([space.wrap(i) for i in tup_w[-1]]) + w_cc = space.newlist([space.wrap(i) for i in cc]) l_w.append(w_cc) return space.newlist(l_w) tcgetattr.unwrap_spec = [ObjSpace, int] Modified: pypy/branch/kill-ctypes/pypy/module/termios/test/test_termios.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/module/termios/test/test_termios.py (original) +++ pypy/branch/kill-ctypes/pypy/module/termios/test/test_termios.py Tue May 29 22:46:05 2007 @@ -52,7 +52,6 @@ cls.w_orig_module_dict = cls.space.appexec([], "(): return %r" % (d,)) def test_values(self): - skip("Not supported yet") import termios d = {} for name in dir(termios): Modified: pypy/branch/kill-ctypes/pypy/rpython/module/ll_termios.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/module/ll_termios.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/module/ll_termios.py Tue May 29 22:46:05 2007 @@ -10,6 +10,9 @@ from pypy.rpython.lltypesystem import lltype from pypy.rpython.extfunc import register_external from pypy.rlib.rarithmetic import intmask +from pypy.rpython.extregistry import ExtRegistryEntry +from pypy.annotation import model as annmodel +from pypy.rpython import rclass # XXX is this portable? well.. not at all, ideally # I would like to have NCCS = CLaterConstant(NCCS) @@ -20,10 +23,6 @@ includes = ['termios.h', 'unistd.h'] -# XXX all functions here raise OSError, because they cannot -# raise termios.error (lack of translation possibilities). hence -# I don't know how to solve this, the tests will probably don't work - TERMIOSP = rffi.CStruct('termios', ('c_iflag', TCFLAG_T), ('c_oflag', TCFLAG_T), ('c_cflag', TCFLAG_T), ('c_lflag', TCFLAG_T), ('c_cc', lltype.FixedSizeArray(CC_T, NCCS))) @@ -35,12 +34,16 @@ c_cfgetospeed = rffi.llexternal('cfgetospeed', [TERMIOSP], SPEED_T, includes=includes) +class termios_error(termios.error): + def __init__(self, num, msg): + self.args = (num, msg) + def tcgetattr_llimpl(fd): c_struct = lltype.malloc(TERMIOSP.TO, flavor='raw') error = c_tcgetattr(fd, c_struct) if error == -1: lltype.free(c_struct, flavor='raw') - raise OSError(error, 'tcgetattr failed') + raise termios_error(error, 'tcgetattr failed') cc = [chr(c_struct.c_c_cc[i]) for i in range(NCCS)] ispeed = c_cfgetispeed(c_struct) ospeed = c_cfgetospeed(c_struct) Modified: pypy/branch/kill-ctypes/pypy/rpython/module/test/test_ll_termios.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/module/test/test_ll_termios.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/module/test/test_ll_termios.py Tue May 29 22:46:05 2007 @@ -32,7 +32,7 @@ def spawn(self, argv): return self._spawn(sys.executable, argv) - def test_getattr(self): + def test_tcgetattr(self): source = py.code.Source(""" import sys sys.path.insert(0, '%s') @@ -58,14 +58,29 @@ second = child.match.group(0) assert first == second - #def test_one(self): - # child = self.spawn() - # child.expect("Python ") - # child.expect('>>> ') - # child.sendline('import termios') - # child.expect('>>> ') - # child.sendline('termios.tcgetattr(0)') - # child.expect('\[.*?\[.*?\]\]') - # lst = eval(child.match.group(0)) - # assert len(lst) == 7 - # assert len(lst[-1]) == 32 # XXX is this portable??? + def test_tcgetattr2(self): + source = py.code.Source(""" + import sys + sys.path.insert(0, '%s') + from pypy.translator.c.test.test_genc import compile + from pypy.rpython.module import ll_termios + import termios + def runs_tcgetattr(): + try: + termios.tcgetattr(338) + except termios.error, e: + return 2 + return 3 + + fn = compile(runs_tcgetattr, [], backendopt=False) + res = fn() + if res == 2: + print 'OK!' + else: + print 'fail!' + """ % os.path.dirname(pypydir)) + f = udir.join("test_tcgetattr.py") + f.write(source) + child = self.spawn([str(f)]) + child.expect("OK!") + From santagada at codespeak.net Wed May 30 04:43:37 2007 From: santagada at codespeak.net (santagada at codespeak.net) Date: Wed, 30 May 2007 04:43:37 +0200 (CEST) Subject: [pypy-svn] r43892 - pypy/dist/pypy/lang/js Message-ID: <20070530024337.75FF78082@code0.codespeak.net> Author: santagada Date: Wed May 30 04:43:35 2007 New Revision: 43892 Modified: pypy/dist/pypy/lang/js/interpreter.py pypy/dist/pypy/lang/js/operations.py Log: fixed the bugs discovered in translation and the addition of sqrt to the Math object. Modified: pypy/dist/pypy/lang/js/interpreter.py ============================================================================== --- pypy/dist/pypy/lang/js/interpreter.py (original) +++ pypy/dist/pypy/lang/js/interpreter.py Wed May 30 04:43:35 2007 @@ -134,7 +134,10 @@ def powjs(ctx, args, this): return W_Number(math.pow(args[0].ToNumber(), args[1].ToNumber())) - + +def sqrtjs(ctx, args, this): + return W_Number(math.sqrt(args[0].ToNumber())) + def versionjs(ctx, args, this): return w_Undefined @@ -167,6 +170,7 @@ w_math.Put('abs', W_Builtin(absjs, Class='function')) w_math.Put('floor', W_Builtin(floorjs, Class='function')) w_math.Put('pow', W_Builtin(powjs, Class='function')) + w_math.Put('sqrt', W_Builtin(sqrtjs, Class='function')) w_math.Put('E', W_Number(math.e)) w_math.Put('PI', W_Number(math.pi)) Modified: pypy/dist/pypy/lang/js/operations.py ============================================================================== --- pypy/dist/pypy/lang/js/operations.py (original) +++ pypy/dist/pypy/lang/js/operations.py Wed May 30 04:43:35 2007 @@ -720,6 +720,7 @@ def string_unquote(self, string): temp = [] stop = len(string)-1 + assert stop >= 0 last = "" #removing the begining quotes (" or \') @@ -866,7 +867,7 @@ ctx.variable.Put(name, w_Undefined) else: ctx.variable.Put(name, self.expr.eval(ctx).GetValue()) - return self.identifier + return self.identifier.eval(ctx) class VariableDeclList(Expression): @@ -956,12 +957,12 @@ self.body = body def execute(self, ctx): - identifier = self.vardecl.eval(ctx) + self.vardecl.eval(ctx) obj = self.object.eval(ctx).GetValue().ToObject() for prop in obj.propdict.values(): if prop.de: continue - iterator = identifier.eval(ctx) + iterator = self.vardecl.eval(ctx) iterator.PutValue(prop.value, ctx) try: result = self.body.execute(ctx) From fijal at codespeak.net Wed May 30 17:26:07 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 30 May 2007 17:26:07 +0200 (CEST) Subject: [pypy-svn] r43896 - pypy/branch/kill-ctypes/pypy/rpython Message-ID: <20070530152607.89FCF807E@code0.codespeak.net> Author: fijal Date: Wed May 30 17:26:06 2007 New Revision: 43896 Modified: pypy/branch/kill-ctypes/pypy/rpython/extfunc.py Log: Dangerous hack and no test :-( I need some consultancy regarding these (as mwh said "it's armin's country") Modified: pypy/branch/kill-ctypes/pypy/rpython/extfunc.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/extfunc.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/extfunc.py Wed May 30 17:26:06 2007 @@ -2,6 +2,7 @@ from pypy.rpython.lltypesystem.lltype import typeOf from pypy.objspace.flow.model import Constant from pypy.annotation.model import unionof +from pypy.annotation.listdef import TooLateForChange from pypy.annotation.signature import annotation import py @@ -26,15 +27,26 @@ class ExtFuncEntry(ExtRegistryEntry): def compute_result_annotation(self, *args_s): + bookkeeper = self.bookkeeper + if self.signature_args is None: + signature_args = None + else: + signature_args = [annotation(arg, bookkeeper) + for arg in self.signature_args] + signature_result = annotation(self.signature_result, bookkeeper) if hasattr(self, 'ann_hook'): self.ann_hook() - if self.signature_args is not None: - assert len(args_s) == len(self.signature_args),\ + if signature_args is not None: + assert len(args_s) == len(signature_args),\ "Argument number mismatch" - for arg, expected in zip(args_s, self.signature_args): + for arg, expected in zip(args_s, signature_args): arg = unionof(arg, expected) assert expected.contains(arg) - return self.signature_result + # XXX: obviously very very dangerous hack. I need + # to reconsifder it a bit, invalidate and so on + self.__class__.signature_args = signature_args + self.__class__.signature_result = signature_result + return signature_result def specialize_call(self, hop): rtyper = hop.rtyper @@ -53,7 +65,8 @@ fakeimpl = getattr(self, fake_method_name, self.instance) if impl: obj = rtyper.getannmixlevel().delayedfunction( - impl, self.signature_args, self.signature_result) + impl, self.signature_args, + self.signature_result) else: obj = rtyper.type_system.getexternalcallable(args_ll, ll_result, name, _external_name=self.name, _callable=fakeimpl) @@ -77,11 +90,13 @@ class FunEntry(ExtFuncEntry): _about_ = function - if args is None: - signature_args = None - else: - signature_args = [annotation(arg, None) for arg in args] - signature_result = annotation(result, None) + #if args is None: + # signature_args = None + #else: + # signature_args = [annotation(arg, None) for arg in args] + #signature_result = annotation(result, None) + signature_args = args + signature_result = result name=export_name if llimpl: lltypeimpl = staticmethod(llimpl) From fijal at codespeak.net Wed May 30 17:26:31 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 30 May 2007 17:26:31 +0200 (CEST) Subject: [pypy-svn] r43897 - in pypy/branch/kill-ctypes/pypy/rpython/module: . test Message-ID: <20070530152631.3000D807E@code0.codespeak.net> Author: fijal Date: Wed May 30 17:26:31 2007 New Revision: 43897 Modified: pypy/branch/kill-ctypes/pypy/rpython/module/ll_termios.py pypy/branch/kill-ctypes/pypy/rpython/module/test/test_ll_termios.py Log: termios.tcsetattr, RPython level Modified: pypy/branch/kill-ctypes/pypy/rpython/module/ll_termios.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/module/ll_termios.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/module/ll_termios.py Wed May 30 17:26:31 2007 @@ -29,10 +29,17 @@ c_tcgetattr = rffi.llexternal('tcgetattr', [lltype.Signed, TERMIOSP], lltype.Signed, includes=includes) +c_tcsetattr = rffi.llexternal('tcsetattr', [lltype.Signed, lltype.Signed, + TERMIOSP], lltype.Signed, includes=includes) c_cfgetispeed = rffi.llexternal('cfgetispeed', [TERMIOSP], SPEED_T, includes=includes) c_cfgetospeed = rffi.llexternal('cfgetospeed', [TERMIOSP], SPEED_T, includes=includes) +c_cfsetispeed = rffi.llexternal('cfsetispeed', [TERMIOSP, SPEED_T], + lltype.Signed, includes=includes) +c_cfsetospeed = rffi.llexternal('cfsetospeed', [TERMIOSP, SPEED_T], + lltype.Signed, includes=includes) + class termios_error(termios.error): def __init__(self, num, msg): @@ -56,4 +63,29 @@ register_external(termios.tcgetattr, [int], (int, int, int, int, int, int, [str]), llimpl=tcgetattr_llimpl, export_name='termios.tcgetattr') -#def tcsetattr_llimpl(fd +def tcsetattr_llimpl(fd, when, attributes): + c_struct = lltype.malloc(TERMIOSP.TO, flavor='raw') + c_struct.c_c_iflag, c_struct.c_c_oflag, c_struct.c_c_cflag, \ + c_struct.c_c_lflag, ispeed, ospeed, cc = attributes + for i in range(NCCS): + c_struct.c_c_cc[i] = rffi.r_uchar(ord(cc[i])) + error = c_cfsetispeed(c_struct, ispeed) + if error == -1: + lltype.free(c_struct, flavor='raw') + raise termios_error(error, 'tcsetattr failed') + error = c_cfsetospeed(c_struct, ospeed) + if error == -1: + lltype.free(c_struct, flavor='raw') + raise termios_error(error, 'tcsetattr failed') + error = c_tcsetattr(fd, when, c_struct) + if error == -1: + lltype.free(c_struct, flavor='raw') + raise termios_error(error, 'tcsetattr failed') + lltype.free(c_struct, flavor='raw') + +r_uint = rffi.r_uint +register_external(termios.tcsetattr, [int, int, (r_uint, r_uint, r_uint, + r_uint, r_uint, r_uint, [str])], llimpl=tcsetattr_llimpl, + export_name='termios.tcsetattr') + + Modified: pypy/branch/kill-ctypes/pypy/rpython/module/test/test_ll_termios.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/module/test/test_ll_termios.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/module/test/test_ll_termios.py Wed May 30 17:26:31 2007 @@ -84,3 +84,30 @@ child = self.spawn([str(f)]) child.expect("OK!") + def test_tcsetattr(self): + # a test, which doesn't even check anything. + # I've got no idea how to test it to be honest :-( + source = py.code.Source(""" + import sys + sys.path.insert(0, '%s') + from pypy.translator.c.test.test_genc import compile + from pypy.rpython.module import ll_termios + import termios, time + def runs_tcsetattr(): + tp = termios.tcgetattr(2) + a, b, c, d, e, f, g = tp + termios.tcsetattr(2, termios.TCSANOW, (a, b, c, d, e, f, g)) + time.sleep(1) + tp = termios.tcgetattr(2) + assert tp[5] == f + + fn = compile(runs_tcsetattr, [], backendopt=False) + fn() + print 'OK!' + """ % os.path.dirname(pypydir)) + f = udir.join("test_tcsetattr.py") + f.write(source) + child = self.spawn([str(f)]) + child.expect("OK!") + + From pdg at codespeak.net Wed May 30 18:39:49 2007 From: pdg at codespeak.net (pdg at codespeak.net) Date: Wed, 30 May 2007 18:39:49 +0200 (CEST) Subject: [pypy-svn] r43906 - pypy/dist/pypy/translator/jvm/test Message-ID: <20070530163949.58BB3807E@code0.codespeak.net> Author: pdg Date: Wed May 30 18:39:48 2007 New Revision: 43906 Modified: pypy/dist/pypy/translator/jvm/test/test_builtin.py pypy/dist/pypy/translator/jvm/test/test_contant.py Log: translator/jvm/test - updated test_builtin and test_constant Modified: pypy/dist/pypy/translator/jvm/test/test_builtin.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_builtin.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_builtin.py Wed May 30 18:39:48 2007 @@ -6,4 +6,25 @@ class TestJavaBuiltin(JvmTest, BaseTestRbuiltin): def test_os(self): py.test.skip("Jvm os support uncertain") - + + def test_os_open(self): + py.test.skip("ll_os_open is not currently implemented in the Jvm backed") + + def test_os_getcwd(self): + py.test.skip("ll_os_getcwd is not currently implemented in the Jvm backed") + + def test_os_write(self): + py.test.skip("ll_os_open is not currently implemented in the Jvm backed") + + def test_os_write_single_char(self): + py.test.skip("ll_os_open is not currently implemented in the Jvm backed") + + def test_os_read(self): + py.test.skip("ll_os_open is not currently implemented in the Jvm backed") + + def test_os_path_exists(self): + py.test.skip("ll_os_stat is not currently implemented in the Jvm backed") + + def test_os_isdir(self): + py.test.skip("ll_os_stat is not currently implemented in the Jvm backed") + \ No newline at end of file Modified: pypy/dist/pypy/translator/jvm/test/test_contant.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_contant.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_contant.py Wed May 30 18:39:48 2007 @@ -101,25 +101,26 @@ assert self.interpret(fn2, [1]) == True def test_customdict_circular(self): - from pypy.rlib.objectmodel import r_dict - def key_eq(a, b): - return a.x[0] == b.x[0] - def key_hash(a): - return ord(a.x[0]) - - class A: - def __init__(self, x): - self.x = x - a = A('foo') - a.dict = r_dict(key_eq, key_hash) - a.dict[a] = 42 - def fn(b): - if b: - s = A('foo') - else: - s = A('bar') - return a.dict[s] - assert self.interpret(fn, [True]) == 42 + py.test.skip("Circular dicts are not supported in JVM") +# from pypy.rlib.objectmodel import r_dict +# def key_eq(a, b): +# return a.x[0] == b.x[0] +# def key_hash(a): +# return ord(a.x[0]) +# +# class A: +# def __init__(self, x): +# self.x = x +# a = A('foo') +# a.dict = r_dict(key_eq, key_hash) +# a.dict[a] = 42 +# def fn(b): +# if b: +# s = A('foo') +# else: +# s = A('bar') +# return a.dict[s] +# assert self.interpret(fn, [True]) == 42 def test_multiple_step(self): from pypy.translator.oosupport import constant From pdg at codespeak.net Wed May 30 18:42:24 2007 From: pdg at codespeak.net (pdg at codespeak.net) Date: Wed, 30 May 2007 18:42:24 +0200 (CEST) Subject: [pypy-svn] r43907 - pypy/dist/pypy/translator/jvm/test Message-ID: <20070530164224.2B648807F@code0.codespeak.net> Author: pdg Date: Wed May 30 18:42:23 2007 New Revision: 43907 Added: pypy/dist/pypy/translator/jvm/test/test_constant.py - copied unchanged from r43906, pypy/dist/pypy/translator/jvm/test/test_contant.py Removed: pypy/dist/pypy/translator/jvm/test/test_contant.py Log: translator/jvm/test - renamed constant.py to constant.py From pdg at codespeak.net Wed May 30 22:28:34 2007 From: pdg at codespeak.net (pdg at codespeak.net) Date: Wed, 30 May 2007 22:28:34 +0200 (CEST) Subject: [pypy-svn] r43910 - pypy/dist/pypy/translator/jvm/test Message-ID: <20070530202834.B95BB8081@code0.codespeak.net> Author: pdg Date: Wed May 30 22:28:33 2007 New Revision: 43910 Modified: pypy/dist/pypy/translator/jvm/test/test_builtin.py pypy/dist/pypy/translator/jvm/test/test_float.py pypy/dist/pypy/translator/jvm/test/test_int.py pypy/dist/pypy/translator/jvm/test/test_op.py pypy/dist/pypy/translator/jvm/test/test_primitive.py Log: translator/jvm/test - updated test_int test_float test_builtin test_primitive test_op Modified: pypy/dist/pypy/translator/jvm/test/test_builtin.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_builtin.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_builtin.py Wed May 30 22:28:33 2007 @@ -27,4 +27,16 @@ def test_os_isdir(self): py.test.skip("ll_os_stat is not currently implemented in the Jvm backed") - \ No newline at end of file + + def test_builtin_math_floor(self): + py.test.skip("metavm.py needs to be updated to handle this math op") + + def test_builtin_math_fmod(self): + py.test.skip("metavm.py needs to be updated to handle this math op") + + def test_builtin_math_frexp(self): + py.test.skip("metavm.py needs to be updated to handle this math op") + + def test_builtin_math_modf(self): + py.test.skip("metavm.py needs to be updated to handle this math op") + \ No newline at end of file Modified: pypy/dist/pypy/translator/jvm/test/test_float.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_float.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_float.py Wed May 30 22:28:33 2007 @@ -4,15 +4,32 @@ class TestJvmFloat(JvmTest, BaseTestRfloat): def test_parse_float(self): - ex = ['', ' ', '0', '1', '-1.5', '1.5E2', '2.5e-1', ' 0 ', '?'] - def fn(i): - s = ex[i] - try: - return float(s) - except ValueError: - return -999.0 + py.test.skip("JVM backend unknown opcode ooparse_float with ll_str_0") +# ex = ['', ' ', '0', '1', '-1.5', '1.5E2', '2.5e-1', ' 0 ', '?'] +# def fn(i): +# s = ex[i] +# try: +# return float(s) +# except ValueError: +# return -999.0 +# +# for i in range(len(ex)): +# expected = fn(i) +# res = self.interpret(fn, [i]) +# assert res == expected + + def test_longlong_conversion(self): + py.test.skip("JVM backend unknown opcode cast_float_to_longlong") - for i in range(len(ex)): - expected = fn(i) - res = self.interpret(fn, [i]) - assert res == expected \ No newline at end of file + def test_pow(self): + py.test.skip("JVM backend unknown opcode float_pow") + + def test_float_constant_conversions(self): + py.test.skip("JVM backend lacks appropriate percision") + + def test_from_r_uint(self): + py.test.skip("JVM backend lacks appropriate percision") + + def test_to_r_uint(self): + py.test.skip("JVM backend lacks appropriate percision") + \ No newline at end of file Modified: pypy/dist/pypy/translator/jvm/test/test_int.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_int.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_int.py Wed May 30 22:28:33 2007 @@ -16,5 +16,20 @@ def test_rarithmetic(self): pass # does this make more sense in jvm + + def test_specializing_int_functions(self): + py.test.skip("Error with longlong precision results in 2 == 1") + + def test_float_conversion(self): + py.test.skip("Unknown opcode cast_longlong_to_float") + + def test_float_conversion_implicit(self): + py.test.skip("Unknown opcode cast_longlong_to_float") + + def test_neg_abs_ovf(self): + py.test.skip("emit doesn't get str or opcode, but None") + + def test_protected_div_mod(self): + py.test.skip("fails because of unusual exception propagation") div_mod_iteration_count = 20 Modified: pypy/dist/pypy/translator/jvm/test/test_op.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_op.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_op.py Wed May 30 22:28:33 2007 @@ -1,7 +1,49 @@ +import py from pypy.translator.jvm.test.runtest import JvmTest from pypy.translator.oosupport.test_template.operations import BaseTestOperations # ====> ../../oosupport/test_template/operations.py class TestOperations(JvmTest, BaseTestOperations): - pass + def test_two_overflows(self): + py.test.skip("Backend lacks appropriate precision") + + def test_ignore_resume_point(self): + py.test.skip("Unknown opcode resume_point") + + def test_rshift(self): + py.test.skip("VerifyError happens. Expecting int on the stack") + + def test_uint_neg(self): + py.test.skip("Backend lacks appropriate precision") + + def test_eq(self): + py.test.skip("Str to long is not implemented, needed for test") + + def test_ne(self): + py.test.skip("Str to long is not implemented, needed for test") + + def test_ge(self): + py.test.skip("Str to long is not implemented, needed for test") + + def test_le(self): + py.test.skip("Str to long is not implemented, needed for test") + + def test_and_not(self): + py.test.skip("VerifyError happens. Accessing uninit reg") + + def test_modulo(self): + py.test.skip("Backend lacks appropriate precision") + + def test_operations(self): + py.test.skip("Backend lacks appropriate precision") + + def test_abs(self): + py.test.skip("Backend lacks appropriate precision") + + def test_is_true(self): + py.test.skip("VerifyError happens. Accessing uninit reg") + + def test_is_early_constant(self): + py.test.skip("Unknown opcode is_early_constant") + \ No newline at end of file Modified: pypy/dist/pypy/translator/jvm/test/test_primitive.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_primitive.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_primitive.py Wed May 30 22:28:33 2007 @@ -1,13 +1,15 @@ import os import time +import py from pypy.translator.jvm.test.runtest import JvmTest class TestPrimitive(JvmTest): def test_time_time(self): - def fn(): - return time.time() - t1 = self.interpret(fn, []) - t2 = self.interpret(fn, []) - assert t1 <= t2 + py.test.skip("ll_time_time is not implemented in jvm backend") +# def fn(): +# return time.time() +# t1 = self.interpret(fn, []) +# t2 = self.interpret(fn, []) +# assert t1 <= t2 From fijal at codespeak.net Thu May 31 17:02:11 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 31 May 2007 17:02:11 +0200 (CEST) Subject: [pypy-svn] r43942 - in pypy/branch/kill-ctypes/pypy/rpython/module: . test Message-ID: <20070531150211.0C28080A6@code0.codespeak.net> Author: fijal Date: Thu May 31 17:02:11 2007 New Revision: 43942 Modified: pypy/branch/kill-ctypes/pypy/rpython/module/ll_termios.py pypy/branch/kill-ctypes/pypy/rpython/module/test/test_ll_termios.py Log: Finish ll termios module Modified: pypy/branch/kill-ctypes/pypy/rpython/module/ll_termios.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/module/ll_termios.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/module/ll_termios.py Thu May 31 17:02:11 2007 @@ -20,6 +20,7 @@ CC_T = rffi.UCHAR NCCS = 32 SPEED_T = rffi.UINT +INT = rffi.INT includes = ['termios.h', 'unistd.h'] @@ -27,19 +28,19 @@ ('c_cflag', TCFLAG_T), ('c_lflag', TCFLAG_T), ('c_cc', lltype.FixedSizeArray(CC_T, NCCS))) -c_tcgetattr = rffi.llexternal('tcgetattr', [lltype.Signed, TERMIOSP], - lltype.Signed, includes=includes) -c_tcsetattr = rffi.llexternal('tcsetattr', [lltype.Signed, lltype.Signed, - TERMIOSP], lltype.Signed, includes=includes) -c_cfgetispeed = rffi.llexternal('cfgetispeed', [TERMIOSP], SPEED_T, - includes=includes) -c_cfgetospeed = rffi.llexternal('cfgetospeed', [TERMIOSP], SPEED_T, - includes=includes) -c_cfsetispeed = rffi.llexternal('cfsetispeed', [TERMIOSP, SPEED_T], - lltype.Signed, includes=includes) -c_cfsetospeed = rffi.llexternal('cfsetospeed', [TERMIOSP, SPEED_T], - lltype.Signed, includes=includes) +def c_external(name, args, result): + return rffi.llexternal(name, args, result, includes=includes) +c_tcgetattr = c_external('tcgetattr', [INT, TERMIOSP], INT) +c_tcsetattr = c_external('tcsetattr', [INT, INT, TERMIOSP], INT) +c_cfgetispeed = c_external('cfgetispeed', [TERMIOSP], SPEED_T) +c_cfgetospeed = c_external('cfgetospeed', [TERMIOSP], SPEED_T) +c_cfsetispeed = c_external('cfsetispeed', [TERMIOSP, SPEED_T], INT) +c_cfsetospeed = c_external('cfsetospeed', [TERMIOSP, SPEED_T], INT) +c_tcsendbreak = c_external('tcsendbreak', [INT, INT], INT) +c_tcdrain = c_external('tcdrain', [INT], INT) +c_tcflush = c_external('tcflush', [INT, INT], INT) +c_tcflow = c_external('tcflow', [INT, INT], INT) class termios_error(termios.error): def __init__(self, num, msg): @@ -88,4 +89,33 @@ r_uint, r_uint, r_uint, [str])], llimpl=tcsetattr_llimpl, export_name='termios.tcsetattr') - +# a bit C-c C-v code follows... + +def tcsendbreak_llimpl(fd, duration): + error = c_tcsendbreak(fd, duration) + if error == -1: + raise termios_error(error, 'tcsendbreak failed') +register_external(termios.tcsendbreak, [int, int], + llimpl=tcsendbreak_llimpl, + export_name='termios.tcsendbreak') + +def tcdrain_llimpl(fd): + error = c_tcdrain(fd) + if error == -1: + raise termios_error(error, 'tcdrain failed') +register_external(termios.tcdrain, [int], llimpl=tcdrain_llimpl, + export_name='termios.tcdrain') + +def tcflush_llimpl(fd, queue_selector): + error = c_tcflush(fd, queue_selector) + if error == -1: + raise termios_error(error, 'tcflush failed') +register_external(termios.tcflush, [int, int], llimpl=tcflush_llimpl, + export_name='termios.tcflush') + +def tcflow_llimpl(fd, action): + error = c_tcflow(fd, action) + if error == -1: + raise termios_error(error, 'tcflow failed') +register_external(termios.tcflow, [int, int], llimpl=tcflow_llimpl, + export_name='termios.tcflow') Modified: pypy/branch/kill-ctypes/pypy/rpython/module/test/test_ll_termios.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/rpython/module/test/test_ll_termios.py (original) +++ pypy/branch/kill-ctypes/pypy/rpython/module/test/test_ll_termios.py Thu May 31 17:02:11 2007 @@ -110,4 +110,25 @@ child = self.spawn([str(f)]) child.expect("OK!") - + def test_tcrest(self): + source = py.code.Source(""" + import sys + sys.path.insert(0, '%s') + from pypy.translator.c.test.test_genc import compile + from pypy.rpython.module import ll_termios + import termios, time + def runs_tcall(): + termios.tcsendbreak(2, 0) + termios.tcdrain(2) + termios.tcflush(2, termios.TCIOFLUSH) + termios.tcflow(2, termios.TCOON) + + fn = compile(runs_tcall, [], backendopt=False) + fn() + print 'OK!' + """ % os.path.dirname(pypydir)) + f = udir.join("test_tcall.py") + f.write(source) + child = self.spawn([str(f)]) + child.expect("OK!") + From pdg at codespeak.net Thu May 31 17:43:55 2007 From: pdg at codespeak.net (pdg at codespeak.net) Date: Thu, 31 May 2007 17:43:55 +0200 (CEST) Subject: [pypy-svn] r43943 - pypy/dist/pypy/translator/jvm/test Message-ID: <20070531154355.9A47480A6@code0.codespeak.net> Author: pdg Date: Thu May 31 17:43:54 2007 New Revision: 43943 Added: pypy/dist/pypy/translator/jvm/test/test_streamio.py Modified: pypy/dist/pypy/translator/jvm/test/test_int.py Log: translator/jvm/test - removed the tab from test_int and added test_streamio Modified: pypy/dist/pypy/translator/jvm/test/test_int.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_int.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_int.py Thu May 31 17:43:54 2007 @@ -18,8 +18,8 @@ pass # does this make more sense in jvm def test_specializing_int_functions(self): - py.test.skip("Error with longlong precision results in 2 == 1") - + py.test.skip("Error with longlong precision results in 2 == 1") + def test_float_conversion(self): py.test.skip("Unknown opcode cast_longlong_to_float") Added: pypy/dist/pypy/translator/jvm/test/test_streamio.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/jvm/test/test_streamio.py Thu May 31 17:43:54 2007 @@ -0,0 +1,48 @@ +import py +from pypy.translator.jvm.test.runtest import JvmTest +from pypy.rlib.test.test_streamio import BaseTestBufferingInputStreamTests,\ + BaseTestBufferingOutputStream, BaseTestLineBufferingOutputStream,\ + BaseTestCRLFFilter, BaseTestBufferingInputOutputStreamTests,\ + BaseTestTextInputFilter, BaseTestTextOutputFilter + +class TestBufferingInputStreamTests(JvmTest, BaseTestBufferingInputStreamTests): + pass + +class TestBufferingOutputStream(JvmTest, BaseTestBufferingOutputStream): + pass + +class TestLineBufferingOutputStream(JvmTest, BaseTestLineBufferingOutputStream): + pass + +class TestCRLFFilter(JvmTest, BaseTestCRLFFilter): + pass + +class TestBufferingInputOutputStreamTests(JvmTest, BaseTestBufferingInputOutputStreamTests): + pass + +class TestTextInputFilter(JvmTest, BaseTestTextInputFilter): + pass + +class TestTextOutputFilter(JvmTest, BaseTestTextOutputFilter): + def test_write_nl(self): + py.test.skip("VerifyError - Incompatible object arguments") + + def test_write_cr(self): + py.test.skip("VerifyError - Incompatible object arguments") + + def test_write_crnl(self): + py.test.skip("VerifyError - Incompatible object arguments") + + def test_write_tell_nl(self): + py.test.skip("VerifyError - Incompatible object arguments") + + def test_write_tell_cr(self): + py.test.skip("VerifyError - Incompatible object arguments") + + def test_write_tell_crnl(self): + py.test.skip("VerifyError - Incompatible object arguments") + + def test_write_seek(self): + py.test.skip("VerifyError - Incompatible object arguments") + + From fijal at codespeak.net Thu May 31 17:49:18 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 31 May 2007 17:49:18 +0200 (CEST) Subject: [pypy-svn] r43944 - in pypy/branch/kill-ctypes/pypy/module/termios: . test Message-ID: <20070531154918.B73BA80A6@code0.codespeak.net> Author: fijal Date: Thu May 31 17:49:18 2007 New Revision: 43944 Modified: pypy/branch/kill-ctypes/pypy/module/termios/interp_termios.py pypy/branch/kill-ctypes/pypy/module/termios/test/test_termios.py Log: Finish termios module on applevel. Modified: pypy/branch/kill-ctypes/pypy/module/termios/interp_termios.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/module/termios/interp_termios.py (original) +++ pypy/branch/kill-ctypes/pypy/module/termios/interp_termios.py Thu May 31 17:49:18 2007 @@ -6,6 +6,7 @@ from pypy.interpreter.baseobjspace import ObjSpace, W_Root from pypy.interpreter.error import OperationError from pypy.rpython.module import ll_termios +from pypy.rlib.objectmodel import we_are_translated import os import termios @@ -26,7 +27,29 @@ return OperationError(w_exception_class, w_exception) def tcsetattr(space, fd, when, w_attributes): - pass + try: + w_cc = space.getitem(w_attributes, space.wrap(-1)) + tup_w = space.getitem(w_attributes, space.newslice( + space.wrap(0), space.wrap(-1), space.wrap(1))) + w_iflag, w_oflag, w_cflag, w_lflag, w_ispeed, w_ospeed = \ + space.unpackiterable(tup_w) + cc = [space.str_w(c) for c in space.unpackiterable(w_cc)] + tup = (space.int_w(w_iflag), space.int_w(w_oflag), + space.int_w(w_cflag), space.int_w(w_lflag), + space.int_w(w_ispeed), space.int_w(w_ospeed), cc) + try: + if we_are_translated(): + termios.tcsetattr(fd, when, tup) + else: + termios.tcsetattr(fd, when, list(tup)) + except termios.error, e: + e.errno = e.args[0] + raise convert_error(space, e) + except OperationError: + if not e.match(space, space.w_TypeError): + raise + msg = "tcsetattr must be called with int, int and 7-arg tuple" + raise OperationError(space.w_TypeError, space.wrap(str(msg))) tcsetattr.unwrap_spec = [ObjSpace, int, int, W_Root] def tcgetattr(space, fd): @@ -44,17 +67,33 @@ tcgetattr.unwrap_spec = [ObjSpace, int] def tcsendbreak(space, fd, duration): - pass + try: + termios.tcsendbreak(fd, duration) + except termios.error, e: + e.errno = e.args[0] + raise convert_error(space, e) tcsendbreak.unwrap_spec = [ObjSpace, int, int] -def tcdrain(space, fd, duration): - pass -tcdrain.unwrap_spec = [ObjSpace, int, int] +def tcdrain(space, fd): + try: + termios.tcdrain(fd) + except termios.error, e: + e.errno = e.args[0] + raise convert_error(space, e) +tcdrain.unwrap_spec = [ObjSpace, int] def tcflush(space, fd, queue): - pass + try: + termios.tcflush(fd, queue) + except termios.error, e: + e.errno = e.args[0] + raise convert_error(space, e) tcflush.unwrap_spec = [ObjSpace, int, int] def tcflow(space, fd, action): - pass + try: + termios.tcflow(fd, action) + except termios.error, e: + e.errno = e.args[0] + raise convert_error(space, e) tcflow.unwrap_spec = [ObjSpace, int, int] Modified: pypy/branch/kill-ctypes/pypy/module/termios/test/test_termios.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/module/termios/test/test_termios.py (original) +++ pypy/branch/kill-ctypes/pypy/module/termios/test/test_termios.py Thu May 31 17:49:18 2007 @@ -3,6 +3,7 @@ import sys from pypy.conftest import gettestobjspace from pypy.tool.autopath import pypydir +from pypy.tool.udir import udir def setup_module(mod): try: @@ -41,6 +42,24 @@ assert len(lst) == 7 assert len(lst[-1]) == 32 # XXX is this portable??? + def test_tcall(self): + """ Again - a test that doesnt really test anything + """ + source = py.code.Source(""" + import termios + f = termios.tcgetattr(2) + termios.tcsetattr(2, termios.TCSANOW, f) + termios.tcsendbreak(2, 0) + termios.tcdrain(2) + termios.tcflush(2, termios.TCIOFLUSH) + termios.tcflow(2, termios.TCOON) + print 'ok!' + """) + f = udir.join("test_tcall.py") + f.write(source) + child = self.spawn(['--withmod-termios', str(f)]) + child.expect('ok!') + class AppTestTermios(object): def setup_class(cls): cls.space = gettestobjspace(usemodules=['termios']) @@ -64,3 +83,4 @@ # XXX not always true, but good assumption import termios raises(termios.error, "termios.tcgetattr(334)") + From fijal at codespeak.net Thu May 31 17:52:57 2007 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 31 May 2007 17:52:57 +0200 (CEST) Subject: [pypy-svn] r43945 - pypy/branch/kill-ctypes/pypy/module/termios Message-ID: <20070531155257.5A2AB80A8@code0.codespeak.net> Author: fijal Date: Thu May 31 17:52:57 2007 New Revision: 43945 Modified: pypy/branch/kill-ctypes/pypy/module/termios/interp_termios.py Log: typo Modified: pypy/branch/kill-ctypes/pypy/module/termios/interp_termios.py ============================================================================== --- pypy/branch/kill-ctypes/pypy/module/termios/interp_termios.py (original) +++ pypy/branch/kill-ctypes/pypy/module/termios/interp_termios.py Thu May 31 17:52:57 2007 @@ -45,7 +45,7 @@ except termios.error, e: e.errno = e.args[0] raise convert_error(space, e) - except OperationError: + except OperationError, e: if not e.match(space, space.w_TypeError): raise msg = "tcsetattr must be called with int, int and 7-arg tuple" From pdg at codespeak.net Thu May 31 17:54:34 2007 From: pdg at codespeak.net (pdg at codespeak.net) Date: Thu, 31 May 2007 17:54:34 +0200 (CEST) Subject: [pypy-svn] r43946 - pypy/dist/pypy/translator/jvm/test Message-ID: <20070531155434.D772180A8@code0.codespeak.net> Author: pdg Date: Thu May 31 17:54:34 2007 New Revision: 43946 Added: pypy/dist/pypy/translator/jvm/test/test_tuple.py Log: translator/jvm/test - added test_tuple Added: pypy/dist/pypy/translator/jvm/test/test_tuple.py ============================================================================== --- (empty file) +++ pypy/dist/pypy/translator/jvm/test/test_tuple.py Thu May 31 17:54:34 2007 @@ -0,0 +1,21 @@ +import py +from pypy.translator.jvm.test.runtest import JvmTest +from pypy.rpython.test.test_rtuple import BaseTestRtuple + +class TestJvmTuple(JvmTest, BaseTestRtuple): + def test_builtin_records(self): + def fn(x, y): + return x, y + res = self.interpret(fn, [1.0, 1]) + assert res.item0 == 1.0 and res.item1 == 1 + res = self.interpret(fn, [1.0, 1.0]) + assert res.item0 == 1.0 and res.item1 == 1.0 + + def test_constant_tuple_contains(self): + py.test.skip("VerifyError - Incompatible object argumemt") + + def test_constant_tuple_contains2(self): + py.test.skip("VerifyError - Incompatible object argumemt") + + def test_constant_unichar_tuple_contains(self): + py.test.skip("VerifyError - Incompatible object argumemt") From pdg at codespeak.net Thu May 31 18:56:22 2007 From: pdg at codespeak.net (pdg at codespeak.net) Date: Thu, 31 May 2007 18:56:22 +0200 (CEST) Subject: [pypy-svn] r43947 - pypy/dist/pypy/translator/jvm/test Message-ID: <20070531165622.8C50D809C@code0.codespeak.net> Author: pdg Date: Thu May 31 18:56:21 2007 New Revision: 43947 Modified: pypy/dist/pypy/translator/jvm/test/test_builtin.py pypy/dist/pypy/translator/jvm/test/test_pbc.py Log: translator/jvm/test - fixed typos Modified: pypy/dist/pypy/translator/jvm/test/test_builtin.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_builtin.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_builtin.py Thu May 31 18:56:21 2007 @@ -29,14 +29,14 @@ py.test.skip("ll_os_stat is not currently implemented in the Jvm backed") def test_builtin_math_floor(self): - py.test.skip("metavm.py needs to be updated to handle this math op") + py.test.skip("metavm.py needs to be updated to handle this math op; graphless extrernal") def test_builtin_math_fmod(self): - py.test.skip("metavm.py needs to be updated to handle this math op") + py.test.skip("metavm.py needs to be updated to handle this math op; graphless extrernal") def test_builtin_math_frexp(self): - py.test.skip("metavm.py needs to be updated to handle this math op") + py.test.skip("metavm.py needs to be updated to handle this math op; graphless extrernal") def test_builtin_math_modf(self): - py.test.skip("metavm.py needs to be updated to handle this math op") + py.test.skip("metavm.py needs to be updated to handle this math op; graphless extrernal") \ No newline at end of file Modified: pypy/dist/pypy/translator/jvm/test/test_pbc.py ============================================================================== --- pypy/dist/pypy/translator/jvm/test/test_pbc.py (original) +++ pypy/dist/pypy/translator/jvm/test/test_pbc.py Thu May 31 18:56:21 2007 @@ -2,5 +2,5 @@ from pypy.translator.jvm.test.runtest import JvmTest from pypy.rpython.test.test_rpbc import BaseTestRPBC -class TestCliPBC(JvmTest, BaseTestRPBC): +class TestJvmPBC(JvmTest, BaseTestRPBC): pass From pypy-svn at codespeak.net Thu May 31 23:47:58 2007 From: pypy-svn at codespeak.net (Doctor Taylor) Date: Thu, 31 May 2007 23:47:58 +0200 (CEST) Subject: [pypy-svn] Online MedHelp Message-ID: <20070531104818.20011.qmail@host81-132-127-138.range81-132.btcentralplus.com> An HTML attachment was scrubbed... URL: